discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

migration, auth, refactoring, formatting

Browse Source
This commit is contained in:
tonyrewin 2022-09-17 21:12:14 +03:00
parent 6b4c00d9e7
commit 3136eecd7e
68 changed files with 968 additions and 930 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.editorconfig
View File

@ -1,8 +1,7 @@
root = true root = true
[*] [*]
indent_style = tabs indent_size = 4
indent_size = 2
end_of_line = lf end_of_line = lf
charset = utf-8 charset = utf-8
trim_trailing_whitespace=true trim_trailing_whitespace=true

2
.flake8
View File

@ -1,5 +1,5 @@
[flake8] [flake8]
ignore = E203,W504,W191 ignore = E203,W504,W191,W503
exclude = .git,__pycache__,orm/rbac.py exclude = .git,__pycache__,orm/rbac.py
max-complexity = 10 max-complexity = 10
max-line-length = 108 max-line-length = 108

3
auth/__init__.py
View File

@ -1,3 +0,0 @@
from auth.email import load_email_templates
load_email_templates()

86
auth/authenticate.py
View File

@ -1,21 +1,20 @@
from functools import wraps from functools import wraps
from typing import Optional, Tuple from typing import Optional, Tuple
from datetime import datetime, timedelta
from graphql import GraphQLResolveInfo from graphql.type import GraphQLResolveInfo
from jwt import DecodeError, ExpiredSignatureError from jwt import DecodeError, ExpiredSignatureError
from starlette.authentication import AuthenticationBackend from starlette.authentication import AuthenticationBackend
from starlette.requests import HTTPConnection from starlette.requests import HTTPConnection
from auth.credentials import AuthCredentials, AuthUser from auth.credentials import AuthCredentials, AuthUser
from auth.jwtcodec import JWTCodec from auth.jwtcodec import JWTCodec
from auth.authorize import Authorize, TokenStorage from auth.tokenstorage import TokenStorage
from base.exceptions import InvalidToken from base.exceptions import InvalidToken
from orm.user import User
from services.auth.users import UserStorage from services.auth.users import UserStorage
from base.orm import local_session from settings import SESSION_TOKEN_HEADER
from settings import JWT_AUTH_HEADER, EMAIL_TOKEN_LIFE_SPAN
class _Authenticate: class SessionToken:
@classmethod @classmethod
async def verify(cls, token: str): async def verify(cls, token: str):
""" """
@ -32,33 +31,30 @@ class _Authenticate:
payload = JWTCodec.decode(token) payload = JWTCodec.decode(token)
except ExpiredSignatureError: except ExpiredSignatureError:
payload = JWTCodec.decode(token, verify_exp=False) payload = JWTCodec.decode(token, verify_exp=False)
if not await cls.exists(payload.user_id, token): if not await cls.get(payload.user_id, token):
raise InvalidToken("Login expired, please login again") raise InvalidToken("Session token has expired, please try again")
if payload.device == "mobile": # noqa
"we cat set mobile token to be valid forever"
return payload
except DecodeError as e: except DecodeError as e:
raise InvalidToken("token format error") from e raise InvalidToken("token format error") from e
else: else:
if not await cls.exists(payload.user_id, token): if not await cls.get(payload.user_id, token):
raise InvalidToken("Login expired, please login again") raise InvalidToken("Session token has expired, please login again")
return payload return payload
@classmethod @classmethod
async def exists(cls, user_id, token): async def get(cls, uid, token):
return await TokenStorage.exist(f"{user_id}-{token}") return await TokenStorage.get(f"{uid}-{token}")
class JWTAuthenticate(AuthenticationBackend): class JWTAuthenticate(AuthenticationBackend):
async def authenticate( async def authenticate(
self, request: HTTPConnection self, request: HTTPConnection
) -> Optional[Tuple[AuthCredentials, AuthUser]]: ) -> Optional[Tuple[AuthCredentials, AuthUser]]:
if JWT_AUTH_HEADER not in request.headers: if SESSION_TOKEN_HEADER not in request.headers:
return AuthCredentials(scopes=[]), AuthUser(user_id=None) return AuthCredentials(scopes=[]), AuthUser(user_id=None)
token = request.headers[JWT_AUTH_HEADER] token = request.headers[SESSION_TOKEN_HEADER]
try: try:
payload = await _Authenticate.verify(token) payload = await SessionToken.verify(token)
except Exception as exc: except Exception as exc:
return AuthCredentials(scopes=[], error_message=str(exc)), AuthUser( return AuthCredentials(scopes=[], error_message=str(exc)), AuthUser(
user_id=None user_id=None
@ -67,9 +63,6 @@ class JWTAuthenticate(AuthenticationBackend):
if payload is None: if payload is None:
return AuthCredentials(scopes=[]), AuthUser(user_id=None) return AuthCredentials(scopes=[]), AuthUser(user_id=None)
if payload.device not in ("pc", "mobile"):
return AuthCredentials(scopes=[]), AuthUser(user_id=None)
user = await UserStorage.get_user(payload.user_id) user = await UserStorage.get_user(payload.user_id)
if not user: if not user:
return AuthCredentials(scopes=[]), AuthUser(user_id=None) return AuthCredentials(scopes=[]), AuthUser(user_id=None)
@ -81,55 +74,6 @@ class JWTAuthenticate(AuthenticationBackend):
) )
class EmailAuthenticate:
@staticmethod
async def get_email_token(user):
token = await Authorize.authorize(
user, device="email", life_span=EMAIL_TOKEN_LIFE_SPAN
)
return token
@staticmethod
async def authenticate(token):
payload = await _Authenticate.verify(token)
if payload is None:
raise InvalidToken("invalid token")
if payload.device != "email":
raise InvalidToken("invalid token")
with local_session() as session:
user = session.query(User).filter_by(id=payload.user_id).first()
if not user:
raise Exception("user not exist")
if not user.emailConfirmed:
user.emailConfirmed = True
session.commit()
auth_token = await Authorize.authorize(user)
return (auth_token, user)
class ResetPassword:
@staticmethod
async def get_reset_token(user):
exp = datetime.utcnow() + timedelta(seconds=EMAIL_TOKEN_LIFE_SPAN)
token = JWTCodec.encode(user, exp=exp, device="pc")
await TokenStorage.save(f"{user.id}-reset-{token}", EMAIL_TOKEN_LIFE_SPAN, True)
return token
@staticmethod
async def verify(token):
try:
payload = JWTCodec.decode(token)
except ExpiredSignatureError:
raise InvalidToken("Login expired, please login again")
except DecodeError as e:
raise InvalidToken("token format error") from e
else:
if not await TokenStorage.exist(f"{payload.user_id}-reset-{token}"):
raise InvalidToken("Login expired, please login again")
return payload.user_id
def login_required(func): def login_required(func):
@wraps(func) @wraps(func)
async def wrap(parent, info: GraphQLResolveInfo, *args, **kwargs): async def wrap(parent, info: GraphQLResolveInfo, *args, **kwargs):

45
auth/authorize.py
View File

@ -1,45 +0,0 @@
from datetime import datetime, timedelta
from auth.jwtcodec import JWTCodec
from base.redis import redis
from settings import JWT_LIFE_SPAN
from auth.validations import User
class TokenStorage:
@staticmethod
async def save(token_key, life_span, auto_delete=True):
await redis.execute("SET", token_key, "True")
if auto_delete:
expire_at = (datetime.now() + timedelta(seconds=life_span)).timestamp()
await redis.execute("EXPIREAT", token_key, int(expire_at))
@staticmethod
async def exist(token_key):
return await redis.execute("GET", token_key)
class Authorize:
@staticmethod
async def authorize(
user: User, device: str = "pc", life_span=JWT_LIFE_SPAN, auto_delete=True
) -> str:
exp = datetime.utcnow() + timedelta(seconds=life_span)
token = JWTCodec.encode(user, exp=exp, device=device)
await TokenStorage.save(f"{user.id}-{token}", life_span, auto_delete)
return token
@staticmethod
async def revoke(token: str) -> bool:
try:
payload = JWTCodec.decode(token)
except: # noqa
pass
else:
await redis.execute("DEL", f"{payload.user_id}-{token}")
return True
@staticmethod
async def revoke_all(user: User):
tokens = await redis.execute("KEYS", f"{user.id}-*")
await redis.execute("DEL", *tokens)

6
auth/credentials.py
View File

@ -1,6 +1,9 @@
from typing import List, Optional, Text from typing import List, Optional, Text
from pydantic import BaseModel from pydantic import BaseModel
from base.exceptions import OperationNotAllowed
class Permission(BaseModel): class Permission(BaseModel):
name: Text name: Text
@ -17,7 +20,8 @@ class AuthCredentials(BaseModel):
return True return True
async def permissions(self) -> List[Permission]: async def permissions(self) -> List[Permission]:
assert self.user_id is not None, "Please login first" if self.user_id is not None:
raise OperationNotAllowed("Please login first")
return NotImplemented() return NotImplemented()

112
auth/email.py
View File

@ -1,84 +1,28 @@
import requests import requests
from starlette.responses import RedirectResponse
from auth.authenticate import EmailAuthenticate, ResetPassword from settings import BACKEND_URL, MAILGUN_API_KEY, MAILGUN_DOMAIN
from base.orm import local_session
from settings import ( MAILGUN_API_URL = "https://api.mailgun.net/v3/%s/messages" % MAILGUN_DOMAIN
BACKEND_URL, MAILGUN_FROM = "discours.io <noreply@%s>" % MAILGUN_DOMAIN
MAILGUN_API_KEY,
MAILGUN_DOMAIN,
RESET_PWD_URL, async def send_auth_email(user, token):
CONFIRM_EMAIL_URL, text = """<html><body>
ERROR_URL_ON_FRONTEND, Follow the <a href='%s'>link</link> to authorize
) </body></html>
"""
MAILGUN_API_URL = "https://api.mailgun.net/v3/%s/messages" % (MAILGUN_DOMAIN) url = "%s/confirm_email" % BACKEND_URL
MAILGUN_FROM = "discours.io <noreply@%s>" % (MAILGUN_DOMAIN) to = "%s <%s>" % (user.username, user.email)
url_with_token = "%s?token=%s" % (url, token)
AUTH_URL = "%s/email_authorize" % (BACKEND_URL) text = text % url_with_token
response = requests.post(
email_templates = {"confirm_email": "", "auth_email": "", "reset_password_email": ""} MAILGUN_API_URL,
auth=("api", MAILGUN_API_KEY),
data={
def load_email_templates(): "from": MAILGUN_FROM,
for name in email_templates: "to": to,
filename = "auth/templates/%s.tmpl" % name "subject": "Confirm email",
with open(filename) as f: "html": text,
email_templates[name] = f.read() },
print("[auth.email] templates loaded") )
response.raise_for_status()
async def send_confirm_email(user):
text = email_templates["confirm_email"]
token = await EmailAuthenticate.get_email_token(user)
await send_email(user, AUTH_URL, text, token)
async def send_auth_email(user):
text = email_templates["auth_email"]
token = await EmailAuthenticate.get_email_token(user)
await send_email(user, AUTH_URL, text, token)
async def send_reset_password_email(user):
text = email_templates["reset_password_email"]
token = await ResetPassword.get_reset_token(user)
await send_email(user, RESET_PWD_URL, text, token)
async def send_email(user, url, text, token):
to = "%s <%s>" % (user.username, user.email)
url_with_token = "%s?token=%s" % (url, token)
text = text % (url_with_token)
response = requests.post(
MAILGUN_API_URL,
auth=("api", MAILGUN_API_KEY),
data={
"from": MAILGUN_FROM,
"to": to,
"subject": "authorize log in",
"html": text,
},
)
response.raise_for_status()
async def email_authorize(request):
token = request.query_params.get("token")
if not token:
url_with_error = "%s?error=%s" % (ERROR_URL_ON_FRONTEND, "INVALID_TOKEN")
return RedirectResponse(url=url_with_error)
try:
auth_token, user = await EmailAuthenticate.authenticate(token)
except:
url_with_error = "%s?error=%s" % (ERROR_URL_ON_FRONTEND, "INVALID_TOKEN")
return RedirectResponse(url=url_with_error)
if not user.emailConfirmed:
with local_session() as session:
user.emailConfirmed = True
session.commit()
response = RedirectResponse(url=CONFIRM_EMAIL_URL)
response.set_cookie("token", auth_token)
return response

63
auth/identity.py
View File

@ -1,16 +1,30 @@
from auth.password import Password from jwt import DecodeError, ExpiredSignatureError
from base.exceptions import InvalidPassword
from orm import User as OrmUser
from base.orm import local_session
from auth.validations import User
from sqlalchemy import or_ from sqlalchemy import or_
from auth.jwtcodec import JWTCodec
from auth.tokenstorage import TokenStorage
from validations.auth import AuthInput
from base.exceptions import InvalidPassword
from base.exceptions import InvalidToken
from base.orm import local_session
from orm import User
from passlib.hash import bcrypt
class Password:
@staticmethod
def encode(password: str) -> str:
return bcrypt.hash(password)
@staticmethod
def verify(password: str, other: str) -> bool:
return bcrypt.verify(password, other)
class Identity: class Identity:
@staticmethod @staticmethod
def identity(orm_user: OrmUser, password: str) -> User: def password(orm_user: User, password: str) -> User:
user = User(**orm_user.dict()) user = AuthInput(**orm_user.dict())
if not user.password: if not user.password:
raise InvalidPassword("User password is empty") raise InvalidPassword("User password is empty")
if not Password.verify(password, user.password): if not Password.verify(password, user.password):
@ -18,22 +32,37 @@ class Identity:
return user return user
@staticmethod @staticmethod
def identity_oauth(input) -> User: def oauth(inp: AuthInput) -> User:
with local_session() as session: with local_session() as session:
user = ( user = (
session.query(OrmUser) session.query(User)
.filter( .filter(or_(User.oauth == inp["oauth"], User.email == inp["email"]))
or_(
OrmUser.oauth == input["oauth"], OrmUser.email == input["email"]
)
)
.first() .first()
) )
if not user: if not user:
user = OrmUser.create(**input) user = User.create(**inp)
if not user.oauth: if not user.oauth:
user.oauth = input["oauth"] user.oauth = inp["oauth"]
session.commit() session.commit()
user = User(**user.dict()) user = User(**user.dict())
return user return user
@staticmethod
async def onetime(token: str) -> User:
try:
payload = JWTCodec.decode(token)
if not await TokenStorage.exist(f"{payload.user_id}-{token}"):
raise InvalidToken("Login token has expired, please login again")
except ExpiredSignatureError:
raise InvalidToken("Login token has expired, please try again")
except DecodeError as e:
raise InvalidToken("token format error") from e
with local_session() as session:
user = session.query(User).filter_by(id=payload.user_id).first()
if not user:
raise Exception("user not exist")
if not user.emailConfirmed:
user.emailConfirmed = True
session.commit()
return user

13
auth/jwtcodec.py
View File

@ -1,26 +1,29 @@
from datetime import datetime from datetime import datetime
import jwt import jwt
from validations.auth import TokenPayload, AuthInput
from settings import JWT_ALGORITHM, JWT_SECRET_KEY from settings import JWT_ALGORITHM, JWT_SECRET_KEY
from auth.validations import PayLoad, User
class JWTCodec: class JWTCodec:
@staticmethod @staticmethod
def encode(user: User, exp: datetime, device: str = "pc") -> str: def encode(user: AuthInput, exp: datetime) -> str:
payload = { payload = {
"user_id": user.id, "user_id": user.id,
"device": device, # "user_email": user.email, # less secure
# "device": device, # no use cases
"exp": exp, "exp": exp,
"iat": datetime.utcnow(), "iat": datetime.utcnow(),
} }
return jwt.encode(payload, JWT_SECRET_KEY, JWT_ALGORITHM) return jwt.encode(payload, JWT_SECRET_KEY, JWT_ALGORITHM)
@staticmethod @staticmethod
def decode(token: str, verify_exp: bool = True) -> PayLoad: def decode(token: str, verify_exp: bool = True) -> TokenPayload:
payload = jwt.decode( payload = jwt.decode(
token, token,
key=JWT_SECRET_KEY, key=JWT_SECRET_KEY,
options={"verify_exp": verify_exp}, options={"verify_exp": verify_exp},
algorithms=[JWT_ALGORITHM], algorithms=[JWT_ALGORITHM],
) )
return PayLoad(**payload) return TokenPayload(**payload)

180
auth/oauth.py
View File

@ -1,91 +1,89 @@
from authlib.integrations.starlette_client import OAuth from authlib.integrations.starlette_client import OAuth
from starlette.responses import RedirectResponse from starlette.responses import RedirectResponse
from auth.authorize import Authorize from auth.identity import Identity
from auth.identity import Identity from auth.tokenstorage import TokenStorage
from settings import OAUTH_CLIENTS, BACKEND_URL, OAUTH_CALLBACK_URL
from settings import OAUTH_CLIENTS, BACKEND_URL, OAUTH_CALLBACK_URL
oauth = OAuth()
oauth = OAuth()
oauth.register(
oauth.register( name="facebook",
name="facebook", client_id=OAUTH_CLIENTS["FACEBOOK"]["id"],
client_id=OAUTH_CLIENTS["FACEBOOK"]["id"], client_secret=OAUTH_CLIENTS["FACEBOOK"]["key"],
client_secret=OAUTH_CLIENTS["FACEBOOK"]["key"], access_token_url="https://graph.facebook.com/v11.0/oauth/access_token",
access_token_url="https://graph.facebook.com/v11.0/oauth/access_token", access_token_params=None,
access_token_params=None, authorize_url="https://www.facebook.com/v11.0/dialog/oauth",
authorize_url="https://www.facebook.com/v11.0/dialog/oauth", authorize_params=None,
authorize_params=None, api_base_url="https://graph.facebook.com/",
api_base_url="https://graph.facebook.com/", client_kwargs={"scope": "public_profile email"},
client_kwargs={"scope": "public_profile email"}, )
)
oauth.register(
oauth.register( name="github",
name="github", client_id=OAUTH_CLIENTS["GITHUB"]["id"],
client_id=OAUTH_CLIENTS["GITHUB"]["id"], client_secret=OAUTH_CLIENTS["GITHUB"]["key"],
client_secret=OAUTH_CLIENTS["GITHUB"]["key"], access_token_url="https://github.com/login/oauth/access_token",
access_token_url="https://github.com/login/oauth/access_token", access_token_params=None,
access_token_params=None, authorize_url="https://github.com/login/oauth/authorize",
authorize_url="https://github.com/login/oauth/authorize", authorize_params=None,
authorize_params=None, api_base_url="https://api.github.com/",
api_base_url="https://api.github.com/", client_kwargs={"scope": "user:email"},
client_kwargs={"scope": "user:email"}, )
)
oauth.register(
oauth.register( name="google",
name="google", client_id=OAUTH_CLIENTS["GOOGLE"]["id"],
client_id=OAUTH_CLIENTS["GOOGLE"]["id"], client_secret=OAUTH_CLIENTS["GOOGLE"]["key"],
client_secret=OAUTH_CLIENTS["GOOGLE"]["key"], server_metadata_url="https://accounts.google.com/.well-known/openid-configuration",
server_metadata_url="https://accounts.google.com/.well-known/openid-configuration", client_kwargs={"scope": "openid email profile"},
client_kwargs={"scope": "openid email profile"}, )
)
async def google_profile(client, request, token):
async def google_profile(client, request, token): profile = await client.parse_id_token(request, token)
profile = await client.parse_id_token(request, token) profile["id"] = profile["sub"]
profile["id"] = profile["sub"] return profile
return profile
async def facebook_profile(client, request, token):
async def facebook_profile(client, request, token): profile = await client.get("me?fields=name,id,email", token=token)
profile = await client.get("me?fields=name,id,email", token=token) return profile.json()
return profile.json()
async def github_profile(client, request, token):
async def github_profile(client, request, token): profile = await client.get("user", token=token)
profile = await client.get("user", token=token) return profile.json()
return profile.json()
profile_callbacks = {
profile_callbacks = { "google": google_profile,
"google": google_profile, "facebook": facebook_profile,
"facebook": facebook_profile, "github": github_profile,
"github": github_profile, }
}
async def oauth_login(request):
async def oauth_login(request): provider = request.path_params["provider"]
provider = request.path_params["provider"] request.session["provider"] = provider
request.session["provider"] = provider client = oauth.create_client(provider)
client = oauth.create_client(provider) redirect_uri = "%s/%s" % (BACKEND_URL, "oauth_authorize")
redirect_uri = "%s/%s" % (BACKEND_URL, "oauth_authorize") return await client.authorize_redirect(request, redirect_uri)
return await client.authorize_redirect(request, redirect_uri)
async def oauth_authorize(request):
async def oauth_authorize(request): provider = request.session["provider"]
provider = request.session["provider"] client = oauth.create_client(provider)
client = oauth.create_client(provider) token = await client.authorize_access_token(request)
token = await client.authorize_access_token(request) get_profile = profile_callbacks[provider]
get_profile = profile_callbacks[provider] profile = await get_profile(client, request, token)
profile = await get_profile(client, request, token) user_oauth_info = "%s:%s" % (provider, profile["id"])
user_oauth_info = "%s:%s" % (provider, profile["id"]) user_input = {
user_input = { "oauth": user_oauth_info,
"oauth": user_oauth_info, "email": profile["email"],
"email": profile["email"], "username": profile["name"],
"username": profile["name"], }
} user = Identity.oauth(user_input)
user = Identity.identity_oauth(user_input) session_token = await TokenStorage.create_session(user)
token = await Authorize.authorize(user, device="pc") response = RedirectResponse(url=OAUTH_CALLBACK_URL)
response.set_cookie("token", session_token)
response = RedirectResponse(url=OAUTH_CALLBACK_URL) return response
response.set_cookie("token", token)
return response

11
auth/password.py
View File

@ -1,11 +0,0 @@
from passlib.hash import bcrypt
class Password:
@staticmethod
def encode(password: str) -> str:
return bcrypt.hash(password)
@staticmethod
def verify(password: str, other: str) -> bool:
return bcrypt.verify(password, other)

1
auth/templates/auth_email.tmpl
View File

@ -1 +0,0 @@
<html><body>To enter the site follow the <a href='%s'>link</link></body></html>

1
auth/templates/confirm_email.tmpl
View File

@ -1 +0,0 @@
<html><body>To confirm registration follow the <a href='%s'>link</link></body></html>

1
auth/templates/reset_password_email.tmpl
View File

@ -1 +0,0 @@
<html><body>To reset password follow the <a href='%s'>link</link></body></html>

50
auth/tokenstorage.py Normal file
View File

@ -0,0 +1,50 @@
from datetime import datetime, timedelta
from auth.jwtcodec import JWTCodec
from validations.auth import AuthInput
from base.redis import redis
from settings import SESSION_TOKEN_LIFE_SPAN, ONETIME_TOKEN_LIFE_SPAN
async def save(token_key, life_span, auto_delete=True):
await redis.execute("SET", token_key, "True")
if auto_delete:
expire_at = (datetime.now() + timedelta(seconds=life_span)).timestamp()
await redis.execute("EXPIREAT", token_key, int(expire_at))
class TokenStorage:
@staticmethod
async def get(token_key):
return await redis.execute("GET", token_key)
@staticmethod
async def create_onetime(user: AuthInput) -> str:
life_span = ONETIME_TOKEN_LIFE_SPAN
exp = datetime.utcnow() + timedelta(seconds=life_span)
one_time_token = JWTCodec.encode(user, exp=exp)
await save(f"{user.id}-{one_time_token}", life_span)
return one_time_token
@staticmethod
async def create_session(user: AuthInput) -> str:
life_span = SESSION_TOKEN_LIFE_SPAN
exp = datetime.utcnow() + timedelta(seconds=life_span)
session_token = JWTCodec.encode(user, exp=exp)
await save(f"{user.id}-{session_token}", life_span)
return session_token
@staticmethod
async def revoke(token: str) -> bool:
try:
payload = JWTCodec.decode(token)
except: # noqa
pass
else:
await redis.execute("DEL", f"{payload.user_id}-{token}")
return True
@staticmethod
async def revoke_all(user: AuthInput):
tokens = await redis.execute("KEYS", f"{user.id}-*")
await redis.execute("DEL", *tokens)

27
auth/validations.py
View File

@ -1,27 +0,0 @@
from datetime import datetime
from typing import Optional, Text
from pydantic import BaseModel
class User(BaseModel):
id: Optional[int]
# age: Optional[int]
username: Optional[Text]
# phone: Optional[Text]
password: Optional[Text]
class PayLoad(BaseModel):
user_id: int
device: Text
exp: datetime
iat: datetime
class CreateUser(BaseModel):
email: Text
username: Optional[Text]
# age: Optional[int]
# phone: Optional[Text]
password: Optional[Text]

2
base/exceptions.py
View File

@ -1,4 +1,4 @@
from graphql import GraphQLError from graphql.error import GraphQLError
class BaseHttpException(GraphQLError): class BaseHttpException(GraphQLError):

2
base/orm.py
View File

@ -1,8 +1,10 @@
from typing import TypeVar, Any, Dict, Generic, Callable from typing import TypeVar, Any, Dict, Generic, Callable
from sqlalchemy import create_engine, Column, Integer from sqlalchemy import create_engine, Column, Integer
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from sqlalchemy.sql.schema import Table from sqlalchemy.sql.schema import Table
from settings import DB_URL from settings import DB_URL
if DB_URL.startswith("sqlite"): if DB_URL.startswith("sqlite"):

69
base/redis.py
View File

@ -1,34 +1,35 @@
import aioredis import aioredis
from settings import REDIS_URL
from settings import REDIS_URL
class Redis:
def __init__(self, uri=REDIS_URL): class Redis:
self._uri: str = uri def __init__(self, uri=REDIS_URL):
self._instance = None self._uri: str = uri
self._instance = None
async def connect(self):
if self._instance is not None: async def connect(self):
return if self._instance is not None:
self._instance = aioredis.from_url(self._uri, encoding="utf-8") return
self._instance = aioredis.from_url(self._uri, encoding="utf-8")
async def disconnect(self):
if self._instance is None: async def disconnect(self):
return if self._instance is None:
self._instance.close() return
await self._instance.wait_closed() self._instance.close()
self._instance = None await self._instance.wait_closed()
self._instance = None
async def execute(self, command, *args, **kwargs):
return await self._instance.execute_command(command, *args, **kwargs) async def execute(self, command, *args, **kwargs):
return await self._instance.execute_command(command, *args, **kwargs)
async def lrange(self, key, start, stop):
return await self._instance.lrange(key, start, stop) async def lrange(self, key, start, stop):
return await self._instance.lrange(key, start, stop)
async def mget(self, key, *keys):
return await self._instance.mget(key, *keys) async def mget(self, key, *keys):
return await self._instance.mget(key, *keys)
redis = Redis()
redis = Redis()
__all__ = ["redis"]
__all__ = ["redis"]

13
main.py
View File

@ -1,4 +1,6 @@
import asyncio
from importlib import import_module from importlib import import_module
from ariadne import load_schema_from_path, make_executable_schema from ariadne import load_schema_from_path, make_executable_schema
from ariadne.asgi import GraphQL from ariadne.asgi import GraphQL
from starlette.applications import Starlette from starlette.applications import Starlette
@ -6,19 +8,18 @@ from starlette.middleware import Middleware
from starlette.middleware.authentication import AuthenticationMiddleware from starlette.middleware.authentication import AuthenticationMiddleware
from starlette.middleware.sessions import SessionMiddleware from starlette.middleware.sessions import SessionMiddleware
from starlette.routing import Route from starlette.routing import Route
from auth.authenticate import JWTAuthenticate from auth.authenticate import JWTAuthenticate
from auth.oauth import oauth_login, oauth_authorize from auth.oauth import oauth_login, oauth_authorize
from auth.email import email_authorize
from base.redis import redis from base.redis import redis
from base.resolvers import resolvers from base.resolvers import resolvers
from resolvers.zine import ShoutsCache from resolvers.zine import ShoutsCache
from services.main import storages_init
from services.stat.reacted import ReactedStorage from services.stat.reacted import ReactedStorage
from services.stat.topicstat import TopicStat
from services.stat.viewed import ViewedStorage from services.stat.viewed import ViewedStorage
from services.zine.gittask import GitTask from services.zine.gittask import GitTask
from services.stat.topicstat import TopicStat
from services.zine.shoutauthor import ShoutAuthorStorage from services.zine.shoutauthor import ShoutAuthorStorage
import asyncio
import_module("resolvers") import_module("resolvers")
schema = make_executable_schema(load_schema_from_path("schema.graphql"), resolvers) # type: ignore schema = make_executable_schema(load_schema_from_path("schema.graphql"), resolvers) # type: ignore
@ -42,6 +43,8 @@ async def start_up():
print(topic_stat_task) print(topic_stat_task)
git_task = asyncio.create_task(GitTask.git_task_worker()) git_task = asyncio.create_task(GitTask.git_task_worker())
print(git_task) print(git_task)
await storages_init()
print()
async def shutdown(): async def shutdown():
@ -51,7 +54,7 @@ async def shutdown():
routes = [ routes = [
Route("/oauth/{provider}", endpoint=oauth_login), Route("/oauth/{provider}", endpoint=oauth_login),
Route("/oauth_authorize", endpoint=oauth_authorize), Route("/oauth_authorize", endpoint=oauth_authorize),
Route("/email_authorize", endpoint=email_authorize), # Route("/confirm_email", endpoint=), # should be called on client
] ]
app = Starlette( app = Starlette(

77
migration/__init__.py
View File

@ -1,33 +1,31 @@
""" cmd managed migration """ """ cmd managed migration """
import csv
import asyncio import asyncio
from datetime import datetime
import json import json
import os
import subprocess import subprocess
import sys import sys
import os from datetime import datetime
import bs4 import bs4
import numpy as np
from migration.tables.comments import migrate as migrateComment
from migration.tables.comments import migrate_2stage as migrateComment_2stage
from migration.tables.content_items import get_shout_slug, migrate as migrateShout
from migration.tables.topics import migrate as migrateTopic
from migration.tables.users import migrate as migrateUser
from migration.tables.users import migrate_2stage as migrateUser_2stage
from orm.reaction import Reaction
from settings import DB_URL
# from export import export_email_subscriptions # from export import export_email_subscriptions
from .export import export_mdx, export_slug from .export import export_mdx, export_slug
from orm.reaction import Reaction
from .tables.users import migrate as migrateUser
from .tables.users import migrate_2stage as migrateUser_2stage
from .tables.content_items import get_shout_slug, migrate as migrateShout
from .tables.topics import migrate as migrateTopic
from .tables.comments import migrate as migrateComment
from .tables.comments import migrate_2stage as migrateComment_2stage
from settings import DB_URL
TODAY = datetime.strftime(datetime.now(), "%Y%m%d") TODAY = datetime.strftime(datetime.now(), "%Y%m%d")
OLD_DATE = "2016-03-05 22:22:00.350000" OLD_DATE = "2016-03-05 22:22:00.350000"
def users_handle(storage): async def users_handle(storage):
"""migrating users first""" """migrating users first"""
counter = 0 counter = 0
id_map = {} id_map = {}
@ -47,10 +45,9 @@ def users_handle(storage):
ce = 0 ce = 0
for entry in storage["users"]["data"]: for entry in storage["users"]["data"]:
ce += migrateUser_2stage(entry, id_map) ce += migrateUser_2stage(entry, id_map)
return storage
def topics_handle(storage): async def topics_handle(storage):
"""topics from categories and tags""" """topics from categories and tags"""
counter = 0 counter = 0
for t in storage["topics"]["tags"] + storage["topics"]["cats"]: for t in storage["topics"]["tags"] + storage["topics"]["cats"]:
@ -78,8 +75,6 @@ def topics_handle(storage):
+ str(len(storage["topics"]["by_slug"].values())) + str(len(storage["topics"]["by_slug"].values()))
+ " topics by slug" + " topics by slug"
) )
# raise Exception
return storage
async def shouts_handle(storage, args): async def shouts_handle(storage, args):
@ -105,9 +100,9 @@ async def shouts_handle(storage, args):
if not shout["topics"]: if not shout["topics"]:
print("[migration] no topics!") print("[migration] no topics!")
# wuth author # with author
author = shout["authors"][0].slug author: str = shout["authors"][0].dict()
if author == "discours": if author["slug"] == "discours":
discours_author += 1 discours_author += 1
# print('[migration] ' + shout['slug'] + ' with author ' + author) # print('[migration] ' + shout['slug'] + ' with author ' + author)
@ -118,21 +113,21 @@ async def shouts_handle(storage, args):
# print main counter # print main counter
counter += 1 counter += 1
line = str(counter + 1) + ": " + shout["slug"] + " @" + author line = str(counter + 1) + ": " + shout["slug"] + " @" + author["slug"]
print(line) print(line)
b = bs4.BeautifulSoup(shout["body"], "html.parser") b = bs4.BeautifulSoup(shout["body"], "html.parser")
texts = [] texts = [shout["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")]
texts.append(shout["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")) texts = texts + b.findAll(text=True)
texts = b.findAll(text=True)
topics_dataset_bodies.append(" ".join([x.strip().lower() for x in texts])) topics_dataset_bodies.append(" ".join([x.strip().lower() for x in texts]))
topics_dataset_tlist.append(shout["topics"]) topics_dataset_tlist.append(shout["topics"])
# np.savetxt('topics_dataset.csv', (topics_dataset_bodies, topics_dataset_tlist), delimiter=',', fmt='%s') # np.savetxt('topics_dataset.csv', (topics_dataset_bodies, topics_dataset_tlist), delimiter=',
# ', fmt='%s')
print("[migration] " + str(counter) + " content items were migrated") print("[migration] " + str(counter) + " content items were migrated")
print("[migration] " + str(pub_counter) + " have been published") print("[migration] " + str(pub_counter) + " have been published")
print("[migration] " + str(discours_author) + " authored by @discours") print("[migration] " + str(discours_author) + " authored by @discours")
return storage
async def comments_handle(storage): async def comments_handle(storage):
@ -146,9 +141,9 @@ async def comments_handle(storage):
missed_shouts[reaction] = oldcomment missed_shouts[reaction] = oldcomment
elif type(reaction) == Reaction: elif type(reaction) == Reaction:
reaction = reaction.dict() reaction = reaction.dict()
id = reaction["id"] rid = reaction["id"]
oid = reaction["oid"] oid = reaction["oid"]
id_map[oid] = id id_map[oid] = rid
else: else:
ignored_counter += 1 ignored_counter += 1
@ -161,7 +156,6 @@ async def comments_handle(storage):
for missed in missed_shouts.values(): for missed in missed_shouts.values():
missed_counter += len(missed) missed_counter += len(missed)
print("[migration] " + str(missed_counter) + " comments dropped") print("[migration] " + str(missed_counter) + " comments dropped")
return storage
def bson_handle(): def bson_handle():
@ -180,8 +174,8 @@ def export_one(slug, storage, args=None):
async def all_handle(storage, args): async def all_handle(storage, args):
print("[migration] handle everything") print("[migration] handle everything")
users_handle(storage) await users_handle(storage)
topics_handle(storage) await topics_handle(storage)
await shouts_handle(storage, args) await shouts_handle(storage, args)
await comments_handle(storage) await comments_handle(storage)
# export_email_subscriptions() # export_email_subscriptions()
@ -205,11 +199,6 @@ def data_load():
"users": {"by_oid": {}, "by_slug": {}, "data": []}, "users": {"by_oid": {}, "by_slug": {}, "data": []},
"replacements": json.loads(open("migration/tables/replacements.json").read()), "replacements": json.loads(open("migration/tables/replacements.json").read()),
} }
users_data = []
tags_data = []
cats_data = []
comments_data = []
content_data = []
try: try:
users_data = json.loads(open("migration/data/users.json").read()) users_data = json.loads(open("migration/data/users.json").read())
print("[migration.load] " + str(len(users_data)) + " users ") print("[migration.load] " + str(len(users_data)) + " users ")
@ -265,13 +254,13 @@ def data_load():
+ str(len(storage["reactions"]["by_content"].keys())) + str(len(storage["reactions"]["by_content"].keys()))
+ " with comments" + " with comments"
) )
storage["users"]["data"] = users_data
storage["topics"]["tags"] = tags_data
storage["topics"]["cats"] = cats_data
storage["shouts"]["data"] = content_data
storage["reactions"]["data"] = comments_data
except Exception as e: except Exception as e:
raise e raise e
storage["users"]["data"] = users_data
storage["topics"]["tags"] = tags_data
storage["topics"]["cats"] = cats_data
storage["shouts"]["data"] = content_data
storage["reactions"]["data"] = comments_data
return storage return storage
@ -301,7 +290,7 @@ def create_pgdump():
async def handle_auto(): async def handle_auto():
print("[migration] no command given, auto mode") print("[migration] no option given, auto mode")
url = os.getenv("MONGODB_URL") url = os.getenv("MONGODB_URL")
if url: if url:
mongo_download(url) mongo_download(url)

5
migration/bson2json.py
View File

@ -1,6 +1,7 @@
import os
import bson
import json import json
import os
import bson
from .utils import DateTimeEncoder from .utils import DateTimeEncoder

40
migration/export.py
View File

@ -1,7 +1,9 @@
from datetime import datetime
import json import json
import os import os
from datetime import datetime
import frontmatter import frontmatter
from .extract import extract_html, prepare_html_body from .extract import extract_html, prepare_html_body
from .utils import DateTimeEncoder from .utils import DateTimeEncoder
@ -67,22 +69,40 @@ def export_slug(slug, storage):
def export_email_subscriptions(): def export_email_subscriptions():
email_subscriptions_data = json.loads(open("migration/data/email_subscriptions.json").read()) email_subscriptions_data = json.loads(
open("migration/data/email_subscriptions.json").read()
)
for data in email_subscriptions_data: for data in email_subscriptions_data:
# TODO: migrate to mailgun list manually # TODO: migrate to mailgun list manually
# migrate_email_subscription(data) # migrate_email_subscription(data)
pass pass
print("[migration] " + str(len(email_subscriptions_data)) + " email subscriptions exported") print(
"[migration] "
+ str(len(email_subscriptions_data))
+ " email subscriptions exported"
)
def export_shouts(storage): def export_shouts(storage):
# update what was just migrated or load json again # update what was just migrated or load json again
if len(storage["users"]["by_slugs"].keys()) == 0: if len(storage["users"]["by_slugs"].keys()) == 0:
storage["users"]["by_slugs"] = json.loads(open(EXPORT_DEST + "authors.json").read()) storage["users"]["by_slugs"] = json.loads(
print("[migration] " + str(len(storage["users"]["by_slugs"].keys())) + " exported authors ") open(EXPORT_DEST + "authors.json").read()
)
print(
"[migration] "
+ str(len(storage["users"]["by_slugs"].keys()))
+ " exported authors "
)
if len(storage["shouts"]["by_slugs"].keys()) == 0: if len(storage["shouts"]["by_slugs"].keys()) == 0:
storage["shouts"]["by_slugs"] = json.loads(open(EXPORT_DEST + "articles.json").read()) storage["shouts"]["by_slugs"] = json.loads(
print("[migration] " + str(len(storage["shouts"]["by_slugs"].keys())) + " exported articles ") open(EXPORT_DEST + "articles.json").read()
)
print(
"[migration] "
+ str(len(storage["shouts"]["by_slugs"].keys()))
+ " exported articles "
)
for slug in storage["shouts"]["by_slugs"].keys(): for slug in storage["shouts"]["by_slugs"].keys():
export_slug(slug, storage) export_slug(slug, storage)
@ -130,4 +150,8 @@ def export_json(
ensure_ascii=False, ensure_ascii=False,
) )
) )
print("[migration] " + str(len(export_comments.items())) + " exported articles with comments") print(
"[migration] "
+ str(len(export_comments.items()))
+ " exported articles with comments"
)

3
migration/extract.py
View File

@ -1,6 +1,7 @@
import base64
import os import os
import re import re
import base64
from .html2text import html2text from .html2text import html2text
TOOLTIP_REGEX = r"(\/\/\/(.+)\/\/\/)" TOOLTIP_REGEX = r"(\/\/\/(.+)\/\/\/)"

10
migration/html2text/__init__.py
View File

@ -379,16 +379,16 @@ class HTML2Text(html.parser.HTMLParser):
if start: if start:
if ( if (
self.current_class == "highlight" self.current_class == "highlight"
and self.inheader == False and not self.inheader
and self.span_lead == False and not self.span_lead
and self.astack == False and not self.astack
): ):
self.o("`") # NOTE: same as <code> self.o("`") # NOTE: same as <code>
self.span_highlight = True self.span_highlight = True
elif ( elif (
self.current_class == "lead" self.current_class == "lead"
and self.inheader == False and not self.inheader
and self.span_highlight == False and not self.span_highlight
): ):
# self.o("==") # NOTE: CriticMarkup {== # self.o("==") # NOTE: CriticMarkup {==
self.span_lead = True self.span_lead = True

1
migration/html2text/cli.py
View File

@ -4,6 +4,7 @@ import sys
from . import HTML2Text, __version__, config from . import HTML2Text, __version__, config
# noinspection DuplicatedCode
def main() -> None: def main() -> None:
baseurl = "" baseurl = ""

9
migration/html2text/utils.py
View File

@ -68,13 +68,11 @@ def element_style(
:rtype: dict :rtype: dict
""" """
style = parent_style.copy() style = parent_style.copy()
if "class" in attrs: if attrs.get("class"):
assert attrs["class"] is not None
for css_class in attrs["class"].split(): for css_class in attrs["class"].split():
css_style = style_def.get("." + css_class, {}) css_style = style_def.get("." + css_class, {})
style.update(css_style) style.update(css_style)
if "style" in attrs: if attrs.get("style"):
assert attrs["style"] is not None
immediate_style = dumb_property_dict(attrs["style"]) immediate_style = dumb_property_dict(attrs["style"])
style.update(immediate_style) style.update(immediate_style)
@ -149,8 +147,7 @@ def list_numbering_start(attrs: Dict[str, Optional[str]]) -> int:
:rtype: int or None :rtype: int or None
""" """
if "start" in attrs: if attrs.get("start"):
assert attrs["start"] is not None
try: try:
return int(attrs["start"]) - 1 return int(attrs["start"]) - 1
except ValueError: except ValueError:

23
migration/tables/comments.py
View File

@ -1,8 +1,10 @@
from datetime import datetime from datetime import datetime
from dateutil.parser import parse as date_parse from dateutil.parser import parse as date_parse
from orm import Reaction, User
from base.orm import local_session from base.orm import local_session
from migration.html2text import html2text from migration.html2text import html2text
from orm import Reaction, User
from orm.reaction import ReactionKind from orm.reaction import ReactionKind
from services.stat.reacted import ReactedStorage from services.stat.reacted import ReactedStorage
@ -46,16 +48,13 @@ async def migrate(entry, storage):
old_thread: String old_thread: String
} }
""" """
reaction_dict = {} reaction_dict = {
reaction_dict["createdAt"] = ( "createdAt": (
ts if not entry.get("createdAt") else date_parse(entry.get("createdAt")) ts if not entry.get("createdAt") else date_parse(entry.get("createdAt"))
) ),
print("[migration] reaction original date %r" % entry.get("createdAt")) "body": html2text(entry.get("body", "")),
# print('[migration] comment date %r ' % comment_dict['createdAt']) "oid": entry["_id"],
reaction_dict["body"] = html2text(entry.get("body", "")) }
reaction_dict["oid"] = entry["_id"]
if entry.get("createdAt"):
reaction_dict["createdAt"] = date_parse(entry.get("createdAt"))
shout_oid = entry.get("contentItem") shout_oid = entry.get("contentItem")
if shout_oid not in storage["shouts"]["by_oid"]: if shout_oid not in storage["shouts"]["by_oid"]:
if len(storage["shouts"]["by_oid"]) > 0: if len(storage["shouts"]["by_oid"]) > 0:
@ -126,7 +125,7 @@ def migrate_2stage(rr, old_new_id):
with local_session() as session: with local_session() as session:
comment = session.query(Reaction).filter(Reaction.id == new_id).first() comment = session.query(Reaction).filter(Reaction.id == new_id).first()
comment.replyTo = old_new_id.get(reply_oid) comment.replyTo = old_new_id.get(reply_oid)
comment.save() session.add(comment)
session.commit() session.commit()
if not rr["body"]: if not rr["body"]:
raise Exception(rr) raise Exception(rr)

51
migration/tables/content_items.py
View File

@ -1,14 +1,18 @@
from dateutil.parser import parse as date_parse
import sqlalchemy
from orm.shout import Shout, ShoutTopic, User
from services.stat.reacted import ReactedStorage
from services.stat.viewed import ViewedByDay
from transliterate import translit
from datetime import datetime from datetime import datetime
from dateutil.parser import parse as date_parse
from sqlalchemy.exc import IntegrityError
from transliterate import translit
from base.orm import local_session from base.orm import local_session
from migration.extract import prepare_html_body from migration.extract import prepare_html_body
from orm.community import Community from orm.community import Community
from orm.reaction import Reaction, ReactionKind from orm.reaction import Reaction, ReactionKind
from orm.shout import Shout, ShoutTopic, User
from orm.topic import TopicFollower
from services.stat.reacted import ReactedStorage
from services.stat.viewed import ViewedByDay
from services.zine.topics import TopicStorage
OLD_DATE = "2016-03-05 22:22:00.350000" OLD_DATE = "2016-03-05 22:22:00.350000"
ts = datetime.now() ts = datetime.now()
@ -72,7 +76,10 @@ async def migrate(entry, storage):
} }
else: else:
userdata = User.default_user.dict() userdata = User.default_user.dict()
assert userdata, "no user found for %s from %d" % [oid, len(users_by_oid.keys())] if not userdata:
raise Exception(
"no user found for %s from %d" % [oid, len(users_by_oid.keys())]
)
r["authors"] = [ r["authors"] = [
userdata, userdata,
] ]
@ -139,32 +146,40 @@ async def migrate(entry, storage):
# del shout_dict['rating'] # NOTE: TypeError: 'rating' is an invalid keyword argument for Shout # del shout_dict['rating'] # NOTE: TypeError: 'rating' is an invalid keyword argument for Shout
# del shout_dict['ratings'] # del shout_dict['ratings']
email = userdata.get("email") email = userdata.get("email")
slug = userdata.get("slug") userslug = userdata.get("slug")
if not slug: if not userslug:
raise Exception raise Exception
with local_session() as session: with local_session() as session:
# c = session.query(Community).all().pop() # c = session.query(Community).all().pop()
if email: if email:
user = session.query(User).filter(User.email == email).first() user = session.query(User).filter(User.email == email).first()
if not user and slug: if not user and userslug:
user = session.query(User).filter(User.slug == slug).first() user = session.query(User).filter(User.slug == userslug).first()
if not user and userdata: if not user and userdata:
try: try:
userdata["slug"] = userdata["slug"].lower().strip().replace(" ", "-") userdata["slug"] = userdata["slug"].lower().strip().replace(" ", "-")
user = User.create(**userdata) user = User.create(**userdata)
except sqlalchemy.exc.IntegrityError: except IntegrityError:
print("[migration] user error: " + userdata) print("[migration] user error: " + userdata)
userdata["id"] = user.id userdata["id"] = user.id
userdata["createdAt"] = user.createdAt userdata["createdAt"] = user.createdAt
storage["users"]["by_slug"][userdata["slug"]] = userdata storage["users"]["by_slug"][userdata["slug"]] = userdata
storage["users"]["by_oid"][entry["_id"]] = userdata storage["users"]["by_oid"][entry["_id"]] = userdata
assert user, "could not get a user" if not user:
shout_dict["authors"] = [user, ] raise Exception("could not get a user")
shout_dict["authors"] = [
user,
]
# TODO: subscribe shout user on shout topics # TODO: subscribe shout user on shout topics
try: try:
s = Shout.create(**shout_dict) s = Shout.create(**shout_dict)
except sqlalchemy.exc.IntegrityError as e: with local_session() as session:
topics = session.query(ShoutTopic).where(ShoutTopic.shout == s.slug).all()
for tpc in topics:
TopicFollower.create(topic=tpc.slug, follower=userslug)
await TopicStorage.update_topic(tpc.slug)
except IntegrityError as e:
with local_session() as session: with local_session() as session:
s = session.query(Shout).filter(Shout.slug == shout_dict["slug"]).first() s = session.query(Shout).filter(Shout.slug == shout_dict["slug"]).first()
bump = False bump = False
@ -267,9 +282,9 @@ async def migrate(entry, storage):
) )
reaction.update(reaction_dict) reaction.update(reaction_dict)
else: else:
reaction_dict["day"] = ( # day = (
reaction_dict.get("createdAt") or ts # reaction_dict.get("createdAt") or ts
).replace(hour=0, minute=0, second=0, microsecond=0) # ).replace(hour=0, minute=0, second=0, microsecond=0)
rea = Reaction.create(**reaction_dict) rea = Reaction.create(**reaction_dict)
await ReactedStorage.react(rea) await ReactedStorage.react(rea)
# shout_dict['ratings'].append(reaction_dict) # shout_dict['ratings'].append(reaction_dict)

36
migration/tables/replacements.json
View File

@ -764,5 +764,37 @@
"blocked-in-russia": "blocked-in-russia", "blocked-in-russia": "blocked-in-russia",
"kavarga": "kavarga", "kavarga": "kavarga",
"galereya-anna-nova": "gallery-anna-nova", "galereya-anna-nova": "gallery-anna-nova",
"derrida": "derrida" "derrida": "derrida",
} "dinozavry": "dinosaurs",
"beecake": "beecake",
"literaturnyykaver": "literature-cover",
"dialog": "dialogue",
"dozhd": "rain",
"pomosch": "help",
"igra": "game",
"reportazh-1": "reportage",
"armiya-1": "army",
"ukraina-2": "ukraine",
"nasilie-1": "violence",
"smert-1": "death",
"dnevnik-1": "dairy",
"voyna-na-ukraine": "war-in-ukraine",
"zabota": "care",
"ango": "ango",
"hayku": "haiku",
"utrata": "loss",
"pokoy": "peace",
"kladbische": "cemetery",
"lomonosov": "lomonosov",
"istoriya-nauki": "history-of-sceince",
"sud": "court",
"russkaya-toska": "russian-toska",
"duh": "spirit",
"devyanostye": "90s",
"seksualnoe-nasilie": "sexual-violence",
"gruziya-2": "georgia",
"dokumentalnaya-poeziya": "documentary-poetry",
"kriptovalyuty": "cryptocurrencies",
"magiya": "magic",
"yazychestvo": "paganism"
}

2
migration/tables/topics.py
View File

@ -1,5 +1,5 @@
from migration.extract import extract_md, html2text
from base.orm import local_session from base.orm import local_session
from migration.extract import extract_md, html2text
from orm import Topic, Community from orm import Topic, Community

36
migration/tables/users.py
View File

@ -1,8 +1,9 @@
import sqlalchemy from dateutil.parser import parse
from sqlalchemy.exc import IntegrityError
from base.orm import local_session
from migration.html2text import html2text from migration.html2text import html2text
from orm import User, UserRating from orm import User, UserRating
from dateutil.parser import parse
from base.orm import local_session
def migrate(entry): def migrate(entry):
@ -21,9 +22,6 @@ def migrate(entry):
"muted": False, # amnesty "muted": False, # amnesty
"bio": entry["profile"].get("bio", ""), "bio": entry["profile"].get("bio", ""),
"notifications": [], "notifications": [],
"createdAt": parse(entry["createdAt"]),
"roles": [], # entry['roles'] # roles by community
"ratings": [], # entry['ratings']
"links": [], "links": [],
"name": "anonymous", "name": "anonymous",
} }
@ -86,7 +84,7 @@ def migrate(entry):
user_dict["slug"] = user_dict["slug"].lower().strip().replace(" ", "-") user_dict["slug"] = user_dict["slug"].lower().strip().replace(" ", "-")
try: try:
user = User.create(**user_dict.copy()) user = User.create(**user_dict.copy())
except sqlalchemy.exc.IntegrityError: except IntegrityError:
print("[migration] cannot create user " + user_dict["slug"]) print("[migration] cannot create user " + user_dict["slug"])
with local_session() as session: with local_session() as session:
old_user = ( old_user = (
@ -120,28 +118,10 @@ def migrate_2stage(entry, id_map):
with local_session() as session: with local_session() as session:
try: try:
user_rating = UserRating.create(**user_rating_dict) user_rating = UserRating.create(**user_rating_dict)
except sqlalchemy.exc.IntegrityError: session.add(user_rating)
old_rating = (
session.query(UserRating)
.filter(UserRating.rater == rater_slug)
.first()
)
print(
"[migration] cannot create "
+ author_slug
+ "`s rate from "
+ rater_slug
)
print(
"[migration] concat rating value %d+%d=%d"
% (
old_rating.value,
rating_entry["value"],
old_rating.value + rating_entry["value"],
)
)
old_rating.update({"value": old_rating.value + rating_entry["value"]})
session.commit() session.commit()
except IntegrityError:
print("[migration] cannot rate " + author_slug + "`s by " + rater_slug)
except Exception as e: except Exception as e:
print(e) print(e)
return ce return ce

70
orm/__init__.py
View File

@ -1,41 +1,29 @@
from orm.rbac import Operation, Resource, Permission, Role from base.orm import Base, engine
from services.auth.roles import RoleStorage from orm.community import Community
from orm.community import Community from orm.notification import Notification
from orm.user import User, UserRating from orm.rbac import Operation, Resource, Permission, Role
from orm.topic import Topic, TopicFollower from orm.reaction import Reaction
from orm.notification import Notification from orm.shout import Shout
from orm.shout import Shout from orm.topic import Topic, TopicFollower
from orm.reaction import Reaction from orm.user import User, UserRating
from services.stat.reacted import ReactedStorage
from services.zine.topics import TopicStorage __all__ = [
from services.auth.users import UserStorage "User",
from services.stat.viewed import ViewedStorage "Role",
from base.orm import Base, engine, local_session "Operation",
"Permission",
__all__ = [ "Community",
"User", "Shout",
"Role", "Topic",
"Operation", "TopicFollower",
"Permission", "Notification",
"Community", "Reaction",
"Shout", "UserRating",
"Topic", ]
"TopicFollower",
"Notification", Base.metadata.create_all(engine)
"Reaction", Operation.init_table()
"UserRating", Resource.init_table()
] User.init_table()
Community.init_table()
Base.metadata.create_all(engine) Role.init_table()
Operation.init_table()
Resource.init_table()
User.init_table()
Community.init_table()
Role.init_table()
with local_session() as session:
ViewedStorage.init(session)
ReactedStorage.init(session)
RoleStorage.init(session)
UserStorage.init(session)
TopicStorage.init(session)

2
orm/collab.py
View File

@ -1,5 +1,7 @@
from datetime import datetime from datetime import datetime
from sqlalchemy import Boolean, Column, String, ForeignKey, DateTime from sqlalchemy import Boolean, Column, String, ForeignKey, DateTime
from base.orm import Base from base.orm import Base

2
orm/collection.py
View File

@ -1,5 +1,7 @@
from datetime import datetime from datetime import datetime
from sqlalchemy import Column, String, ForeignKey, DateTime from sqlalchemy import Column, String, ForeignKey, DateTime
from base.orm import Base from base.orm import Base

2
orm/community.py
View File

@ -1,5 +1,7 @@
from datetime import datetime from datetime import datetime
from sqlalchemy import Column, String, ForeignKey, DateTime from sqlalchemy import Column, String, ForeignKey, DateTime
from base.orm import Base, local_session from base.orm import Base, local_session

1
orm/notification.py
View File

@ -1,4 +1,5 @@
from sqlalchemy import Column, String, JSON as JSONType from sqlalchemy import Column, String, JSON as JSONType
from base.orm import Base from base.orm import Base

2
orm/rbac.py
View File

@ -1,6 +1,8 @@
import warnings import warnings
from sqlalchemy import String, Column, ForeignKey, UniqueConstraint, TypeDecorator from sqlalchemy import String, Column, ForeignKey, UniqueConstraint, TypeDecorator
from sqlalchemy.orm import relationship from sqlalchemy.orm import relationship
from base.orm import Base, REGISTRY, engine, local_session from base.orm import Base, REGISTRY, engine, local_session
from orm.community import Community from orm.community import Community

4
orm/reaction.py
View File

@ -1,7 +1,9 @@
from datetime import datetime from datetime import datetime
from sqlalchemy import Column, String, ForeignKey, DateTime from sqlalchemy import Column, String, ForeignKey, DateTime
from base.orm import Base
from sqlalchemy import Enum from sqlalchemy import Enum
from base.orm import Base
from services.stat.reacted import ReactedStorage, ReactionKind from services.stat.reacted import ReactedStorage, ReactionKind
from services.stat.viewed import ViewedStorage from services.stat.viewed import ViewedStorage

8
orm/shout.py
View File

@ -1,12 +1,14 @@
from datetime import datetime from datetime import datetime
from sqlalchemy import Column, Integer, String, ForeignKey, DateTime, Boolean from sqlalchemy import Column, Integer, String, ForeignKey, DateTime, Boolean
from sqlalchemy.orm import relationship from sqlalchemy.orm import relationship
from orm.user import User
from orm.topic import Topic, ShoutTopic from base.orm import Base
from orm.reaction import Reaction from orm.reaction import Reaction
from orm.topic import Topic, ShoutTopic
from orm.user import User
from services.stat.reacted import ReactedStorage from services.stat.reacted import ReactedStorage
from services.stat.viewed import ViewedStorage from services.stat.viewed import ViewedStorage
from base.orm import Base
class ShoutReactionsFollower(Base): class ShoutReactionsFollower(Base):

2
orm/topic.py
View File

@ -1,5 +1,7 @@
from datetime import datetime from datetime import datetime
from sqlalchemy import Column, String, ForeignKey, DateTime, JSON as JSONType from sqlalchemy import Column, String, ForeignKey, DateTime, JSON as JSONType
from base.orm import Base from base.orm import Base

2
orm/user.py
View File

@ -1,4 +1,5 @@
from datetime import datetime from datetime import datetime
from sqlalchemy import ( from sqlalchemy import (
Column, Column,
Integer, Integer,
@ -9,6 +10,7 @@ from sqlalchemy import (
JSON as JSONType, JSON as JSONType,
) )
from sqlalchemy.orm import relationship from sqlalchemy.orm import relationship
from base.orm import Base, local_session from base.orm import Base, local_session
from orm.rbac import Role from orm.rbac import Role
from services.auth.roles import RoleStorage from services.auth.roles import RoleStorage

33
requirements.txt
View File

@ -1,20 +1,25 @@
frontmatter python-frontmatter~=1.0.0
numpy aioredis~=2.0.1
aioredis ariadne>=0.16.0
ariadne PyYAML>=5.4
pyjwt>=2.0.0 pyjwt>=2.0.0
starlette starlette~=0.20.4
sqlalchemy sqlalchemy>=1.4.41
uvicorn graphql-core
pydantic uvicorn>=0.18.3
passlib pydantic>=1.10.2
passlib~=1.7.4
itsdangerous itsdangerous
authlib==0.15.5 authlib>=1.1.0
httpx>=0.23.0 httpx>=0.23.0
psycopg2-binary psycopg2-binary
transliterate transliterate~=1.10.2
requests requests~=2.28.1
bcrypt bcrypt>=4.0.0
websockets websockets
bson bson~=0.5.10
flake8 flake8
DateTime~=4.7
asyncio~=3.4.3
python-dateutil~=2.8.2
beautifulsoup4~=4.11.1

78
resolvers/__init__.py
View File

@ -3,9 +3,42 @@ from resolvers.auth import (
sign_out, sign_out,
is_email_used, is_email_used,
register, register,
confirm, confirm_email,
auth_forget, auth_send_link,
auth_reset, )
from resolvers.collab import remove_author, invite_author
from resolvers.community import (
create_community,
delete_community,
get_community,
get_communities,
)
# from resolvers.collab import invite_author, remove_author
from resolvers.editor import create_shout, delete_shout, update_shout
from resolvers.profile import (
get_users_by_slugs,
get_current_user,
get_user_reacted_shouts,
get_user_roles,
get_top_authors,
)
# from resolvers.feed import shouts_for_feed, my_candidates
from resolvers.reactions import (
create_reaction,
delete_reaction,
update_reaction,
reactions_unfollow,
reactions_follow,
get_shout_reactions,
)
from resolvers.topics import (
topic_follow,
topic_unfollow,
topics_by_author,
topics_by_community,
topics_all,
) )
from resolvers.zine import ( from resolvers.zine import (
get_shout_by_slug, get_shout_by_slug,
@ -21,36 +54,6 @@ from resolvers.zine import (
shouts_by_topics, shouts_by_topics,
shouts_by_communities, shouts_by_communities,
) )
from resolvers.profile import (
get_users_by_slugs,
get_current_user,
get_user_reacted_shouts,
get_user_roles,
get_top_authors
)
from resolvers.topics import (
topic_follow,
topic_unfollow,
topics_by_author,
topics_by_community,
topics_all,
)
# from resolvers.feed import shouts_for_feed, my_candidates
from resolvers.reactions import (
create_reaction,
delete_reaction,
update_reaction,
)
# from resolvers.collab import invite_author, remove_author
from resolvers.editor import create_shout, delete_shout, update_shout
from resolvers.community import (
create_community,
delete_community,
get_community,
get_communities,
)
__all__ = [ __all__ = [
"follow", "follow",
@ -59,9 +62,8 @@ __all__ = [
"login", "login",
"register", "register",
"is_email_used", "is_email_used",
"confirm", "confirm_email",
"auth_forget", "auth_send_link",
"auth_reset",
"sign_out", "sign_out",
# profile # profile
"get_current_user", "get_current_user",
@ -69,10 +71,7 @@ __all__ = [
"get_user_roles", "get_user_roles",
"get_top_authors", "get_top_authors",
# zine # zine
"shouts_for_feed",
"my_candidates",
"recent_published", "recent_published",
"recent_reacted",
"recent_all", "recent_all",
"shouts_by_topics", "shouts_by_topics",
"shouts_by_authors", "shouts_by_authors",
@ -82,7 +81,6 @@ __all__ = [
"top_overall", "top_overall",
"top_viewed", "top_viewed",
"view_shout", "view_shout",
"view_reaction",
"get_shout_by_slug", "get_shout_by_slug",
# editor # editor
"create_shout", "create_shout",

148
resolvers/auth.py
View File

@ -1,29 +1,42 @@
from graphql import GraphQLResolveInfo
from transliterate import translit
from urllib.parse import quote_plus from urllib.parse import quote_plus
from auth.authenticate import login_required, ResetPassword
from auth.authorize import Authorize from auth.tokenstorage import TokenStorage
from auth.identity import Identity from graphql.type import GraphQLResolveInfo
from auth.password import Password from transliterate import translit
from auth.email import send_confirm_email, send_auth_email, send_reset_password_email
from orm import User, Role from auth.authenticate import login_required
from auth.email import send_auth_email
from auth.identity import Identity, Password
from base.exceptions import (
InvalidPassword,
InvalidToken,
ObjectNotExist,
OperationNotAllowed,
)
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation, query from base.resolvers import mutation, query
from orm import User, Role
from resolvers.profile import get_user_info from resolvers.profile import get_user_info
from base.exceptions import InvalidPassword, InvalidToken, ObjectNotExist, OperationNotAllowed from settings import SESSION_TOKEN_HEADER
from settings import JWT_AUTH_HEADER
@mutation.field("confirmEmail") @mutation.field("confirmEmail")
async def confirm(*_, confirm_token): async def confirm_email(*_, confirm_token):
"""confirm owning email address""" """confirm owning email address"""
auth_token, user = await Authorize.confirm(confirm_token) user_id = None
if auth_token: try:
user.emailConfirmed = True user_id = await TokenStorage.get(confirm_token)
user.save() with local_session() as session:
return {"token": auth_token, "user": user} user = session.query(User).where(User.id == user_id).first()
else: session_token = TokenStorage.create_session(user)
# not an error, warns user user.emailConfirmed = True
session.add(user)
session.commit()
return {"token": session_token, "user": user}
except InvalidToken as e:
raise InvalidToken(e.message)
except Exception as e:
print(e) # FIXME: debug only
return {"error": "email not confirmed"} return {"error": "email not confirmed"}
@ -50,40 +63,21 @@ async def register(*_, email: str, password: str = ""):
session.add(user) session.add(user)
session.commit() session.commit()
await send_confirm_email(user) token = await TokenStorage.create_onetime(user)
await send_auth_email(user, token)
return {"user": user} return {"user": user}
@mutation.field("requestPasswordUpdate") @mutation.field("sendLink")
async def auth_forget(_, info, email): async def auth_send_link(_, info, email):
"""send email to recover account""" """send link with confirm code to email"""
with local_session() as session: with local_session() as session:
user = session.query(User).filter(User.email == email).first() user = session.query(User).filter(User.email == email).first()
if not user: if not user:
raise ObjectNotExist("User not found") raise ObjectNotExist("User not found")
token = await TokenStorage.create_onetime(user)
await send_reset_password_email(user) await send_auth_email(user, token)
return {}
@mutation.field("updatePassword")
async def auth_reset(_, info, password, resetToken):
"""set the new password"""
try:
user_id = await ResetPassword.verify(resetToken)
except InvalidToken as e:
raise InvalidToken(e.message)
# return {"error": e.message}
with local_session() as session:
user = session.query(User).filter_by(id=user_id).first()
if not user:
raise ObjectNotExist("User not found")
user.password = Password.encode(password)
session.commit()
return {} return {}
@ -92,48 +86,44 @@ async def login(_, info: GraphQLResolveInfo, email: str, password: str = ""):
with local_session() as session: with local_session() as session:
orm_user = session.query(User).filter(User.email == email).first() orm_user = session.query(User).filter(User.email == email).first()
if orm_user is None: if orm_user is None:
print(f"signIn {email}: email not found") print(f"[auth] {email}: email not found")
# return {"error": "email not found"} # return {"error": "email not found"}
raise ObjectNotExist("User not found") raise ObjectNotExist("User not found") # contains webserver status
if not password: if not password:
print(f"signIn {email}: send auth email") print(f"[auth] send confirm link to {email}")
await send_auth_email(orm_user) token = await TokenStorage.create_onetime(orm_user)
return {""} await send_auth_email(orm_user, token)
# FIXME: not an error, warning
return {"error": "no password, email link was sent"}
if not orm_user.emailConfirmed: else:
# not an error, warns users # sign in using password
return {"error": "email not confirmed"} if not orm_user.emailConfirmed:
# not an error, warns users
try: return {"error": "please, confirm email"}
device = info.context["request"].headers["device"] else:
except KeyError: try:
device = "pc" user = Identity.password(orm_user, password)
auto_delete = False if device == "mobile" else True # why autodelete with mobile? session_token = await TokenStorage.create_session(user)
print(f"[auth] user {email} authorized")
try: return {
user = Identity.identity(orm_user, password) "token": session_token,
except InvalidPassword: "user": user,
print(f"signIn {email}: invalid password") "info": await get_user_info(user.slug),
raise InvalidPassword("invalid passoword") }
# return {"error": "invalid password"} except InvalidPassword:
print(f"[auth] {email}: invalid password")
token = await Authorize.authorize(user, device=device, auto_delete=auto_delete) raise InvalidPassword("invalid passoword") # contains webserver status
print(f"signIn {email}: OK") # return {"error": "invalid password"}
return {
"token": token,
"user": orm_user,
"info": await get_user_info(orm_user.slug),
}
@query.field("signOut") @query.field("signOut")
@login_required @login_required
async def sign_out(_, info: GraphQLResolveInfo): async def sign_out(_, info: GraphQLResolveInfo):
token = info.context["request"].headers[JWT_AUTH_HEADER] token = info.context["request"].headers[SESSION_TOKEN_HEADER]
status = await Authorize.revoke(token) status = await TokenStorage.revoke(token)
return status return status

12
resolvers/collab.py
View File

@ -1,10 +1,11 @@
from datetime import datetime from datetime import datetime
from auth.authenticate import login_required
from base.orm import local_session from base.orm import local_session
from base.resolvers import query, mutation
from orm.collab import Collab from orm.collab import Collab
from orm.shout import Shout from orm.shout import Shout
from orm.user import User from orm.user import User
from base.resolvers import query, mutation
from auth.authenticate import login_required
@query.field("getCollabs") @query.field("getCollabs")
@ -12,11 +13,10 @@ from auth.authenticate import login_required
async def get_collabs(_, info): async def get_collabs(_, info):
auth = info.context["request"].auth auth = info.context["request"].auth
user_id = auth.user_id user_id = auth.user_id
collabs = []
with local_session() as session: with local_session() as session:
user = session.query(User).where(User.id == user_id).first() user = session.query(User).where(User.id == user_id).first()
collabs = session.query(Collab).filter(user.slug in Collab.authors) collabs = session.query(Collab).filter(user.slug in Collab.authors)
return collabs return collabs
@mutation.field("inviteAuthor") @mutation.field("inviteAuthor")
@ -37,7 +37,7 @@ async def invite_author(_, info, author, shout):
return {"error": "already added"} return {"error": "already added"}
shout.authors.append(author) shout.authors.append(author)
shout.updated_at = datetime.now() shout.updated_at = datetime.now()
shout.save() session.add(shout)
session.commit() session.commit()
# TODO: email notify # TODO: email notify
@ -63,7 +63,7 @@ async def remove_author(_, info, author, shout):
return {"error": "not in authors"} return {"error": "not in authors"}
shout.authors.remove(author) shout.authors.remove(author)
shout.updated_at = datetime.now() shout.updated_at = datetime.now()
shout.save() session.add(shout)
session.commit() session.commit()
# result = Result("INVITED") # result = Result("INVITED")

15
resolvers/collection.py
View File

@ -1,11 +1,13 @@
from orm.collection import Collection, ShoutCollection
from base.orm import local_session
from orm.user import User
from base.resolvers import mutation, query
from auth.authenticate import login_required
from datetime import datetime from datetime import datetime
from sqlalchemy import and_ from sqlalchemy import and_
from auth.authenticate import login_required
from base.orm import local_session
from base.resolvers import mutation, query
from orm.collection import Collection, ShoutCollection
from orm.user import User
@mutation.field("createCollection") @mutation.field("createCollection")
@login_required @login_required
@ -27,7 +29,7 @@ async def create_collection(_, _info, inp):
async def update_collection(_, info, inp): async def update_collection(_, info, inp):
auth = info.context["request"].auth auth = info.context["request"].auth
user_id = auth.user_id user_id = auth.user_id
collection_slug = input.get("slug", "") collection_slug = inp.get("slug", "")
with local_session() as session: with local_session() as session:
owner = session.query(User).filter(User.id == user_id) # note list here owner = session.query(User).filter(User.id == user_id) # note list here
collection = ( collection = (
@ -57,6 +59,7 @@ async def delete_collection(_, info, slug):
if collection.owner != user_id: if collection.owner != user_id:
return {"error": "access denied"} return {"error": "access denied"}
collection.deletedAt = datetime.now() collection.deletedAt = datetime.now()
session.add(collection)
session.commit() session.commit()
return {} return {}

16
resolvers/community.py
View File

@ -1,12 +1,14 @@
from orm.community import Community, CommunityFollower
from base.orm import local_session
from orm.user import User
from base.resolvers import mutation, query
from auth.authenticate import login_required
from datetime import datetime from datetime import datetime
from typing import List from typing import List
from sqlalchemy import and_ from sqlalchemy import and_
from auth.authenticate import login_required
from base.orm import local_session
from base.resolvers import mutation, query
from orm.community import Community, CommunityFollower
from orm.user import User
@mutation.field("createCommunity") @mutation.field("createCommunity")
@login_required @login_required
@ -23,6 +25,8 @@ async def create_community(_, info, input):
createdBy=user.slug, createdBy=user.slug,
createdAt=datetime.now(), createdAt=datetime.now(),
) )
session.add(community)
session.commit()
return {"community": community} return {"community": community}
@ -48,6 +52,7 @@ async def update_community(_, info, input):
community.desc = input.get("desc", "") community.desc = input.get("desc", "")
community.pic = input.get("pic", "") community.pic = input.get("pic", "")
community.updatedAt = datetime.now() community.updatedAt = datetime.now()
session.add(community)
session.commit() session.commit()
@ -64,6 +69,7 @@ async def delete_community(_, info, slug):
if community.owner != user_id: if community.owner != user_id:
return {"error": "access denied"} return {"error": "access denied"}
community.deletedAt = datetime.now() community.deletedAt = datetime.now()
session.add(community)
session.commit() session.commit()
return {} return {}

35
resolvers/editor.py
View File

@ -1,37 +1,38 @@
from orm import Shout from datetime import datetime
from auth.authenticate import login_required
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation
from orm import Shout
from orm.rbac import Resource from orm.rbac import Resource
from orm.shout import ShoutAuthor, ShoutTopic from orm.shout import ShoutAuthor, ShoutTopic
from orm.user import User from orm.user import User
from base.resolvers import mutation
from resolvers.reactions import reactions_follow, reactions_unfollow from resolvers.reactions import reactions_follow, reactions_unfollow
from auth.authenticate import login_required
from datetime import datetime
from services.zine.gittask import GitTask from services.zine.gittask import GitTask
@mutation.field("createShout") @mutation.field("createShout")
@login_required @login_required
async def create_shout(_, info, input): async def create_shout(_, info, inp):
user = info.context["request"].user user = info.context["request"].user
topic_slugs = input.get("topic_slugs", []) topic_slugs = inp.get("topic_slugs", [])
if topic_slugs: if topic_slugs:
del input["topic_slugs"] del inp["topic_slugs"]
new_shout = Shout.create(**input) new_shout = Shout.create(**inp)
ShoutAuthor.create(shout=new_shout.slug, user=user.slug) ShoutAuthor.create(shout=new_shout.slug, user=user.slug)
reactions_follow(user, new_shout.slug, True) reactions_follow(user, new_shout.slug, True)
if "mainTopic" in input: if "mainTopic" in inp:
topic_slugs.append(input["mainTopic"]) topic_slugs.append(inp["mainTopic"])
for slug in topic_slugs: for slug in topic_slugs:
ShoutTopic.create(shout=new_shout.slug, topic=slug) ShoutTopic.create(shout=new_shout.slug, topic=slug)
new_shout.topic_slugs = topic_slugs new_shout.topic_slugs = topic_slugs
GitTask(input, user.username, user.email, "new shout %s" % (new_shout.slug)) GitTask(inp, user.username, user.email, "new shout %s" % (new_shout.slug))
# await ShoutCommentsStorage.send_shout(new_shout) # await ShoutCommentsStorage.send_shout(new_shout)
@ -40,11 +41,11 @@ async def create_shout(_, info, input):
@mutation.field("updateShout") @mutation.field("updateShout")
@login_required @login_required
async def update_shout(_, info, input): async def update_shout(_, info, inp):
auth = info.context["request"].auth auth = info.context["request"].auth
user_id = auth.user_id user_id = auth.user_id
slug = input["slug"] slug = inp["slug"]
session = local_session() session = local_session()
user = session.query(User).filter(User.id == user_id).first() user = session.query(User).filter(User.id == user_id).first()
@ -60,15 +61,16 @@ async def update_shout(_, info, input):
if Resource.shout_id not in scopes: if Resource.shout_id not in scopes:
return {"error": "access denied"} return {"error": "access denied"}
shout.update(input) shout.update(inp)
shout.updatedAt = datetime.now() shout.updatedAt = datetime.now()
session.add(shout)
session.commit() session.commit()
session.close() session.close()
for topic in input.get("topic_slugs", []): for topic in inp.get("topic_slugs", []):
ShoutTopic.create(shout=slug, topic=topic) ShoutTopic.create(shout=slug, topic=topic)
GitTask(input, user.username, user.email, "update shout %s" % (slug)) GitTask(inp, user.username, user.email, "update shout %s" % (slug))
return {"shout": shout} return {"shout": shout}
@ -89,6 +91,7 @@ async def delete_shout(_, info, slug):
for a in authors: for a in authors:
reactions_unfollow(a.slug, slug, True) reactions_unfollow(a.slug, slug, True)
shout.deletedAt = datetime.now() shout.deletedAt = datetime.now()
session.add(shout)
session.commit() session.commit()
return {} return {}

13
resolvers/feed.py
View File

@ -1,11 +1,13 @@
from typing import List
from sqlalchemy import and_, desc
from auth.authenticate import login_required from auth.authenticate import login_required
from base.orm import local_session from base.orm import local_session
from base.resolvers import query from base.resolvers import query
from sqlalchemy import and_, desc
from orm.shout import Shout, ShoutAuthor, ShoutTopic from orm.shout import Shout, ShoutAuthor, ShoutTopic
from orm.topic import TopicFollower from orm.topic import TopicFollower
from orm.user import AuthorFollower from orm.user import AuthorFollower
from typing import List
from services.zine.shoutscache import prepare_shouts from services.zine.shoutscache import prepare_shouts
@ -22,14 +24,14 @@ def get_user_feed(_, info, offset, limit) -> List[Shout]:
.where(AuthorFollower.follower == user.slug) .where(AuthorFollower.follower == user.slug)
.order_by(desc(Shout.createdAt)) .order_by(desc(Shout.createdAt))
) )
topicrows = ( topic_rows = (
session.query(Shout) session.query(Shout)
.join(ShoutTopic) .join(ShoutTopic)
.join(TopicFollower) .join(TopicFollower)
.where(TopicFollower.follower == user.slug) .where(TopicFollower.follower == user.slug)
.order_by(desc(Shout.createdAt)) .order_by(desc(Shout.createdAt))
) )
shouts = shouts.union(topicrows).limit(limit).offset(offset).all() shouts = shouts.union(topic_rows).limit(limit).offset(offset).all()
return shouts return shouts
@ -37,7 +39,6 @@ def get_user_feed(_, info, offset, limit) -> List[Shout]:
@login_required @login_required
async def user_unpublished_shouts(_, info, offset, limit) -> List[Shout]: async def user_unpublished_shouts(_, info, offset, limit) -> List[Shout]:
user = info.context["request"].user user = info.context["request"].user
shouts = []
with local_session() as session: with local_session() as session:
shouts = prepare_shouts( shouts = prepare_shouts(
session.query(Shout) session.query(Shout)
@ -48,4 +49,4 @@ async def user_unpublished_shouts(_, info, offset, limit) -> List[Shout]:
.offset(offset) .offset(offset)
.all() .all()
) )
return shouts return shouts

7
resolvers/inbox.py
View File

@ -1,10 +1,11 @@
from base.resolvers import mutation, query, subscription
from auth.authenticate import login_required
import asyncio import asyncio
import uuid
import json import json
import uuid
from datetime import datetime from datetime import datetime
from auth.authenticate import login_required
from base.redis import redis from base.redis import redis
from base.resolvers import mutation, query, subscription
class ChatFollowing: class ChatFollowing:

37
resolvers/profile.py
View File

@ -1,18 +1,21 @@
from datetime import datetime from datetime import datetime
from orm.user import User, UserRole, Role, UserRating, AuthorFollower from typing import List
from services.auth.users import UserStorage
from orm.shout import Shout
from orm.reaction import Reaction
from base.orm import local_session
from orm.topic import Topic, TopicFollower
from base.resolvers import mutation, query
from resolvers.community import get_followed_communities
from resolvers.reactions import get_shout_reactions
from auth.authenticate import login_required
from resolvers.inbox import get_unread_counter
from sqlalchemy import and_, desc from sqlalchemy import and_, desc
from sqlalchemy.orm import selectinload from sqlalchemy.orm import selectinload
from typing import List
from auth.authenticate import login_required
from auth.tokenstorage import TokenStorage
from base.orm import local_session
from base.resolvers import mutation, query
from orm.reaction import Reaction
from orm.shout import Shout
from orm.topic import Topic, TopicFollower
from orm.user import User, UserRole, Role, UserRating, AuthorFollower
from resolvers.community import get_followed_communities
from resolvers.inbox import get_unread_counter
from resolvers.reactions import get_shout_reactions
from services.auth.users import UserStorage
@query.field("userReactedShouts") @query.field("userReactedShouts")
@ -87,12 +90,13 @@ async def get_user_info(slug):
@login_required @login_required
async def get_current_user(_, info): async def get_current_user(_, info):
user = info.context["request"].user user = info.context["request"].user
user.lastSeen = datetime.now()
with local_session() as session: with local_session() as session:
user.lastSeen = datetime.now() session.add(user)
user.save()
session.commit() session.commit()
token = await TokenStorage.create_session(user)
return { return {
"token": "", # same token? "token": token,
"user": user, "user": user,
"info": await get_user_info(user.slug), "info": await get_user_info(user.slug),
} }
@ -133,7 +137,8 @@ async def update_profile(_, info, profile):
user = session.query(User).filter(User.id == user_id).first() user = session.query(User).filter(User.id == user_id).first()
if user: if user:
User.update(user, **profile) User.update(user, **profile)
session.commit() session.add(user)
session.commit()
return {} return {}

10
resolvers/reactions.py
View File

@ -1,11 +1,13 @@
from datetime import datetime
from sqlalchemy import desc from sqlalchemy import desc
from orm.reaction import Reaction
from auth.authenticate import login_required
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation, query
from orm.reaction import Reaction
from orm.shout import ShoutReactionsFollower from orm.shout import ShoutReactionsFollower
from orm.user import User from orm.user import User
from base.resolvers import mutation, query
from auth.authenticate import login_required
from datetime import datetime
from services.auth.users import UserStorage from services.auth.users import UserStorage
from services.stat.reacted import ReactedStorage from services.stat.reacted import ReactedStorage

16
resolvers/topics.py
View File

@ -1,12 +1,14 @@
from orm.topic import Topic, TopicFollower import random
from services.zine.topics import TopicStorage
from services.stat.topicstat import TopicStat from sqlalchemy import and_
from auth.authenticate import login_required
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation, query from base.resolvers import mutation, query
from auth.authenticate import login_required from orm.topic import Topic, TopicFollower
from sqlalchemy import and_ from services.stat.topicstat import TopicStat
import random
from services.zine.shoutscache import ShoutsCache from services.zine.shoutscache import ShoutsCache
from services.zine.topics import TopicStorage
@query.field("topicsAll") @query.field("topicsAll")
@ -60,7 +62,7 @@ async def update_topic(_, _info, inp):
async def topic_follow(user, slug): async def topic_follow(user, slug):
TopicFollower.create(follower=user.slug, topic=slug) TopicFollower.create(topic=slug, follower=user.slug)
await TopicStorage.update_topic(slug) await TopicStorage.update_topic(slug)

37
resolvers/zine.py
View File

@ -1,18 +1,19 @@
from sqlalchemy.orm import selectinload
from sqlalchemy.sql.expression import and_, select, desc
from auth.authenticate import login_required
from base.orm import local_session
from base.resolvers import mutation, query
from orm.collection import ShoutCollection from orm.collection import ShoutCollection
from orm.shout import Shout, ShoutAuthor, ShoutTopic from orm.shout import Shout, ShoutAuthor, ShoutTopic
from orm.topic import Topic from orm.topic import Topic
from base.orm import local_session from resolvers.community import community_follow, community_unfollow
from base.resolvers import mutation, query from resolvers.profile import author_follow, author_unfollow
from resolvers.reactions import reactions_follow, reactions_unfollow
from resolvers.topics import topic_follow, topic_unfollow
from services.stat.viewed import ViewedStorage
from services.zine.shoutauthor import ShoutAuthorStorage from services.zine.shoutauthor import ShoutAuthorStorage
from services.zine.shoutscache import ShoutsCache from services.zine.shoutscache import ShoutsCache
from services.stat.viewed import ViewedStorage
from resolvers.profile import author_follow, author_unfollow
from resolvers.topics import topic_follow, topic_unfollow
from resolvers.community import community_follow, community_unfollow
from resolvers.reactions import reactions_follow, reactions_unfollow
from auth.authenticate import login_required
from sqlalchemy import select, desc, asc, and_
from sqlalchemy.orm import selectinload
@mutation.field("incrementView") @mutation.field("incrementView")
@ -33,6 +34,12 @@ async def top_month(_, _info, offset, limit):
return ShoutsCache.top_month[offset : offset + limit] return ShoutsCache.top_month[offset : offset + limit]
@query.field("topCommented")
async def top_commented(_, _info, offset, limit):
async with ShoutsCache.lock:
return ShoutsCache.top_commented[offset : offset + limit]
@query.field("topOverall") @query.field("topOverall")
async def top_overall(_, _info, offset, limit): async def top_overall(_, _info, offset, limit):
async with ShoutsCache.lock: async with ShoutsCache.lock:
@ -105,7 +112,7 @@ async def get_search_results(_, _info, query, offset, limit):
for s in shouts: for s in shouts:
for a in s.authors: for a in s.authors:
a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug) a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug)
s.stat.search = 1 # FIXME s.stat.relevance = 1 # FIXME
return shouts return shouts
@ -116,7 +123,7 @@ async def shouts_by_topics(_, _info, slugs, offset, limit):
session.query(Shout) session.query(Shout)
.join(ShoutTopic) .join(ShoutTopic)
.where(and_(ShoutTopic.topic.in_(slugs), bool(Shout.publishedAt))) .where(and_(ShoutTopic.topic.in_(slugs), bool(Shout.publishedAt)))
.order_by(asc(Shout.publishedAt)) .order_by(desc(Shout.publishedAt))
.limit(limit) .limit(limit)
.offset(offset) .offset(offset)
) )
@ -134,7 +141,7 @@ async def shouts_by_collection(_, _info, collection, offset, limit):
session.query(Shout) session.query(Shout)
.join(ShoutCollection, ShoutCollection.collection == collection) .join(ShoutCollection, ShoutCollection.collection == collection)
.where(and_(ShoutCollection.shout == Shout.slug, bool(Shout.publishedAt))) .where(and_(ShoutCollection.shout == Shout.slug, bool(Shout.publishedAt)))
.order_by(asc(Shout.publishedAt)) .order_by(desc(Shout.publishedAt))
.limit(limit) .limit(limit)
.offset(offset) .offset(offset)
) )
@ -151,7 +158,7 @@ async def shouts_by_authors(_, _info, slugs, offset, limit):
session.query(Shout) session.query(Shout)
.join(ShoutAuthor) .join(ShoutAuthor)
.where(and_(ShoutAuthor.user.in_(slugs), bool(Shout.publishedAt))) .where(and_(ShoutAuthor.user.in_(slugs), bool(Shout.publishedAt)))
.order_by(asc(Shout.publishedAt)) .order_by(desc(Shout.publishedAt))
.limit(limit) .limit(limit)
.offset(offset) .offset(offset)
) )
@ -184,7 +191,7 @@ async def shouts_by_communities(_, info, slugs, offset, limit):
), ),
) )
) )
.order_by(desc(Shout.publishedAt)) .order_by(desc("publishedAt"))
.limit(limit) .limit(limit)
.offset(offset) .offset(offset)
) )

6
schema.graphql
View File

@ -148,11 +148,10 @@ type Mutation {
markAsRead(chatId: String!, ids: [Int]!): Result! markAsRead(chatId: String!, ids: [Int]!): Result!
# auth # auth
confirmEmail(token: String!): AuthResult!
refreshSession: AuthResult! refreshSession: AuthResult!
registerUser(email: String!, password: String): AuthResult! registerUser(email: String!, password: String): AuthResult!
requestPasswordUpdate(email: String!): Result! sendLink(email: String!): Result!
updatePassword(password: String!, token: String!): Result! confirmEmail(code: String!): AuthResult!
# shout # shout
createShout(input: ShoutInput!): Result! createShout(input: ShoutInput!): Result!
@ -237,6 +236,7 @@ type Query {
topAuthors(offset: Int!, limit: Int!): [Author]! topAuthors(offset: Int!, limit: Int!): [Author]!
topMonth(offset: Int!, limit: Int!): [Shout]! topMonth(offset: Int!, limit: Int!): [Shout]!
topOverall(offset: Int!, limit: Int!): [Shout]! topOverall(offset: Int!, limit: Int!): [Shout]!
topCommented(offset: Int!, limit: Int!): [Shout]!
recentPublished(offset: Int!, limit: Int!): [Shout]! # homepage recentPublished(offset: Int!, limit: Int!): [Shout]! # homepage
recentReacted(offset: Int!, limit: Int!): [Shout]! # test recentReacted(offset: Int!, limit: Int!): [Shout]! # test
recentAll(offset: Int!, limit: Int!): [Shout]! recentAll(offset: Int!, limit: Int!): [Shout]!

59
server.py
View File

@ -1,30 +1,29 @@
import uvicorn import sys
from settings import PORT import uvicorn
from settings import PORT
import sys
if __name__ == "__main__":
if __name__ == "__main__": x = ""
x = "" if len(sys.argv) > 1:
if len(sys.argv) > 1: x = sys.argv[1]
x = sys.argv[1] if x == "dev":
if x == "dev": print("DEV MODE")
print("DEV MODE") headers = [
headers = [ ("Access-Control-Allow-Methods", "GET, POST, OPTIONS, HEAD"),
("Access-Control-Allow-Methods", "GET, POST, OPTIONS, HEAD"), ("Access-Control-Allow-Origin", "http://localhost:3000"),
("Access-Control-Allow-Origin", "http://localhost:3000"), (
( "Access-Control-Allow-Headers",
"Access-Control-Allow-Headers", "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range",
"DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range", ),
), ("Access-Control-Expose-Headers", "Content-Length,Content-Range"),
("Access-Control-Expose-Headers", "Content-Length,Content-Range"), ("Access-Control-Allow-Credentials", "true"),
("Access-Control-Allow-Credentials", "true"), ]
] uvicorn.run(
uvicorn.run( "main:app", host="localhost", port=8080, headers=headers
"main:app", host="localhost", port=8080, headers=headers ) # , ssl_keyfile="discours.key", ssl_certfile="discours.crt", reload=True)
) # , ssl_keyfile="discours.key", ssl_certfile="discours.crt", reload=True) elif x == "migrate":
elif x == "migrate": from migration import migrate
from migration import migrate
migrate()
migrate() else:
else: uvicorn.run("main:app", host="0.0.0.0", port=PORT)
uvicorn.run("main:app", host="0.0.0.0", port=PORT)

2
services/auth/roles.py
View File

@ -1,5 +1,7 @@
import asyncio import asyncio
from sqlalchemy.orm import selectinload from sqlalchemy.orm import selectinload
from orm.rbac import Role from orm.rbac import Role

2
services/auth/users.py
View File

@ -1,5 +1,7 @@
import asyncio import asyncio
from sqlalchemy.orm import selectinload from sqlalchemy.orm import selectinload
from orm.user import User from orm.user import User

17
services/main.py Normal file
View File

@ -0,0 +1,17 @@
from services.stat.viewed import ViewedStorage
from services.stat.reacted import ReactedStorage
from services.auth.roles import RoleStorage
from services.auth.users import UserStorage
from services.zine.topics import TopicStorage
from base.orm import local_session
async def storages_init():
with local_session() as session:
print('[main] initialize storages')
ViewedStorage.init(session)
ReactedStorage.init(session)
RoleStorage.init(session)
UserStorage.init(session)
TopicStorage.init(session)
session.commit()

58
services/stat/reacted.py
View File

@ -1,11 +1,13 @@
import asyncio import asyncio
from datetime import datetime from datetime import datetime
from enum import Enum as Enumeration
from sqlalchemy import Column, DateTime, ForeignKey, Boolean from sqlalchemy import Column, DateTime, ForeignKey, Boolean
from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.orm.attributes import flag_modified
from sqlalchemy.types import Enum as ColumnEnum
from base.orm import Base, local_session from base.orm import Base, local_session
from orm.topic import ShoutTopic from orm.topic import ShoutTopic
from enum import Enum as Enumeration
from sqlalchemy.types import Enum as ColumnEnum
class ReactionKind(Enumeration): class ReactionKind(Enumeration):
@ -139,26 +141,23 @@ class ReactedStorage:
self = ReactedStorage self = ReactedStorage
async with self.lock: async with self.lock:
reactions = self.reacted["shouts"].get(reaction.shout) reactions = {}
if reaction.replyTo:
reactions = self.reacted["reactions"].get(reaction.id) # iterate sibling reactions
for r in reactions.values(): reactions = self.reacted["shouts"].get(reaction.shout, {})
r = { for r in reactions.values():
"day": datetime.now().replace( reaction = ReactedByDay.create({
hour=0, minute=0, second=0, microsecond=0 "day": datetime.now().replace(
), hour=0, minute=0, second=0, microsecond=0
"reaction": reaction.id, ),
"kind": reaction.kind, "reaction": r.id,
"shout": reaction.shout, "kind": r.kind,
} "shout": r.shout,
if reaction.replyTo: "comment": bool(r.body),
r["replyTo"] = reaction.replyTo "replyTo": r.replyTo
if reaction.body: })
r["comment"] = True # renew sorted by shouts store
reaction: ReactedByDay = ReactedByDay.create(**r) # type: ignore self.reacted["shouts"][reaction.shout] = self.reacted["shouts"].get(reaction.shout, [])
self.reacted["shouts"][reaction.shout] = self.reacted["shouts"].get(
reaction.shout, []
)
self.reacted["shouts"][reaction.shout].append(reaction) self.reacted["shouts"][reaction.shout].append(reaction)
if reaction.replyTo: if reaction.replyTo:
self.reacted["reaction"][reaction.replyTo] = self.reacted[ self.reacted["reaction"][reaction.replyTo] = self.reacted[
@ -169,11 +168,12 @@ class ReactedStorage:
"reactions" "reactions"
].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind) ].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind)
else: else:
# rate only by root reactions on shout
self.rating["shouts"][reaction.replyTo] = self.rating["shouts"].get( self.rating["shouts"][reaction.replyTo] = self.rating["shouts"].get(
reaction.shout, 0 reaction.shout, 0
) + kind_to_rate(reaction.kind) ) + kind_to_rate(reaction.kind)
flag_modified(r, "value") flag_modified(reaction, "value")
@staticmethod @staticmethod
def init(session): def init(session):
@ -218,16 +218,20 @@ class ReactedStorage:
async def flush_changes(session): async def flush_changes(session):
self = ReactedStorage self = ReactedStorage
async with self.lock: async with self.lock:
for slug in dict(self.reacted['shouts']).keys(): for slug in dict(self.reacted["shouts"]).keys():
topics = session.query(ShoutTopic.topic).where(ShoutTopic.shout == slug).all() topics = (
reactions = self.reacted['shouts'].get(slug, []) session.query(ShoutTopic.topic)
.where(ShoutTopic.shout == slug)
.all()
)
reactions = self.reacted["shouts"].get(slug, [])
# print('[stat.reacted] shout {' + str(slug) + "}: " + str(len(reactions))) # print('[stat.reacted] shout {' + str(slug) + "}: " + str(len(reactions)))
for ts in list(topics): for ts in list(topics):
tslug = ts[0] tslug = ts[0]
topic_reactions = self.reacted["topics"].get(tslug, []) topic_reactions = self.reacted["topics"].get(tslug, [])
topic_reactions += reactions topic_reactions += reactions
# print('[stat.reacted] topic {' + str(tslug) + "}: " + str(len(topic_reactions))) # print('[stat.reacted] topic {' + str(tslug) + "}: " + str(len(topic_reactions)))
reactions += list(self.reacted['reactions'].values()) reactions += list(self.reacted["reactions"].values())
for reaction in reactions: for reaction in reactions:
if getattr(reaction, "modified", False): if getattr(reaction, "modified", False):
session.add(reaction) session.add(reaction)

20
services/stat/topicstat.py
View File

@ -1,19 +1,11 @@
import asyncio import asyncio
from base.orm import local_session from base.orm import local_session
from orm.shout import Shout from orm.shout import Shout
from orm.topic import ShoutTopic, TopicFollower
from services.stat.reacted import ReactedStorage from services.stat.reacted import ReactedStorage
from services.stat.viewed import ViewedStorage from services.stat.viewed import ViewedStorage
from services.zine.shoutauthor import ShoutAuthorStorage from services.zine.shoutauthor import ShoutAuthorStorage
from orm.topic import ShoutTopic, TopicFollower
def unique(list1):
# insert the list to the set
list_set = set(list1)
# convert the set to the list
unique_list = (list(list_set))
return unique_list
class TopicStat: class TopicStat:
@ -27,7 +19,7 @@ class TopicStat:
async def load_stat(session): async def load_stat(session):
self = TopicStat self = TopicStat
shout_topics = session.query(ShoutTopic).all() shout_topics = session.query(ShoutTopic).all()
print('[stat.topics] shout topics amount', len(shout_topics)) print("[stat.topics] shout topics amount", len(shout_topics))
for shout_topic in shout_topics: for shout_topic in shout_topics:
# shouts by topics # shouts by topics
@ -35,7 +27,11 @@ class TopicStat:
shout = shout_topic.shout shout = shout_topic.shout
sss = set(self.shouts_by_topic.get(topic, [])) sss = set(self.shouts_by_topic.get(topic, []))
shout = session.query(Shout).where(Shout.slug == shout).first() shout = session.query(Shout).where(Shout.slug == shout).first()
sss.union([shout, ]) sss.union(
[
shout,
]
)
self.shouts_by_topic[topic] = list(sss) self.shouts_by_topic[topic] = list(sss)
# authors by topics # authors by topics

2
services/stat/viewed.py
View File

@ -1,7 +1,9 @@
import asyncio import asyncio
from datetime import datetime from datetime import datetime
from sqlalchemy import Column, DateTime, ForeignKey, Integer from sqlalchemy import Column, DateTime, ForeignKey, Integer
from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.orm.attributes import flag_modified
from base.orm import Base, local_session from base.orm import Base, local_session
from orm.topic import ShoutTopic from orm.topic import ShoutTopic

3
services/zine/gittask.py
View File

@ -1,6 +1,7 @@
import asyncio
import subprocess import subprocess
from pathlib import Path from pathlib import Path
import asyncio
from settings import SHOUTS_REPO from settings import SHOUTS_REPO

1
services/zine/shoutauthor.py
View File

@ -1,4 +1,5 @@
import asyncio import asyncio
from base.orm import local_session from base.orm import local_session
from orm.shout import ShoutAuthor from orm.shout import ShoutAuthor

168
services/zine/shoutscache.py
View File

@ -1,7 +1,9 @@
import asyncio import asyncio
from datetime import datetime, timedelta from datetime import datetime, timedelta
from sqlalchemy import and_, desc, func, select from sqlalchemy import and_, desc, func, select
from sqlalchemy.orm import selectinload from sqlalchemy.orm import selectinload
from base.orm import local_session from base.orm import local_session
from orm.reaction import Reaction from orm.reaction import Reaction
from orm.shout import Shout, ShoutAuthor, ShoutTopic from orm.shout import Shout, ShoutAuthor, ShoutTopic
@ -27,6 +29,7 @@ class ShoutsCache:
top_month = [] top_month = []
top_overall = [] top_overall = []
top_viewed = [] top_viewed = []
top_commented = []
by_author = {} by_author = {}
by_topic = {} by_topic = {}
@ -34,14 +37,17 @@ class ShoutsCache:
@staticmethod @staticmethod
async def prepare_recent_published(): async def prepare_recent_published():
with local_session() as session: with local_session() as session:
shouts = await prepare_shouts(session, ( shouts = await prepare_shouts(
select(Shout) session,
.options(selectinload(Shout.authors), selectinload(Shout.topics)) (
.where(bool(Shout.publishedAt)) select(Shout)
.group_by(Shout.slug) .options(selectinload(Shout.authors), selectinload(Shout.topics))
.order_by(desc("publishedAt")) .where(bool(Shout.publishedAt))
.limit(ShoutsCache.limit) .group_by(Shout.slug)
)) .order_by(desc("publishedAt"))
.limit(ShoutsCache.limit)
),
)
async with ShoutsCache.lock: async with ShoutsCache.lock:
ShoutsCache.recent_published = shouts ShoutsCache.recent_published = shouts
print("[zine.cache] %d recently published shouts " % len(shouts)) print("[zine.cache] %d recently published shouts " % len(shouts))
@ -49,14 +55,17 @@ class ShoutsCache:
@staticmethod @staticmethod
async def prepare_recent_all(): async def prepare_recent_all():
with local_session() as session: with local_session() as session:
shouts = await prepare_shouts(session, ( shouts = await prepare_shouts(
select(Shout) session,
.options(selectinload(Shout.authors), selectinload(Shout.topics)) (
.where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt))) select(Shout)
.group_by(Shout.slug) .options(selectinload(Shout.authors), selectinload(Shout.topics))
.order_by(desc("createdAt")) .where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt)))
.limit(ShoutsCache.limit) .group_by(Shout.slug)
)) .order_by(desc("createdAt"))
.limit(ShoutsCache.limit)
),
)
async with ShoutsCache.lock: async with ShoutsCache.lock:
ShoutsCache.recent_all = shouts ShoutsCache.recent_all = shouts
print("[zine.cache] %d recently created shouts " % len(shouts)) print("[zine.cache] %d recently created shouts " % len(shouts))
@ -64,18 +73,23 @@ class ShoutsCache:
@staticmethod @staticmethod
async def prepare_recent_reacted(): async def prepare_recent_reacted():
with local_session() as session: with local_session() as session:
shouts = await prepare_shouts(session, ( shouts = await prepare_shouts(
select(Shout, func.max(Reaction.createdAt).label("reactionCreatedAt")) session,
.options( (
selectinload(Shout.authors), select(
selectinload(Shout.topics), Shout, func.max(Reaction.createdAt).label("reactionCreatedAt")
) )
.join(Reaction, Reaction.shout == Shout.slug) .options(
.where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt))) selectinload(Shout.authors),
.group_by(Shout.slug) selectinload(Shout.topics),
.order_by(desc("reactionCreatedAt")) )
.limit(ShoutsCache.limit) .join(Reaction, Reaction.shout == Shout.slug)
)) .where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("reactionCreatedAt"))
.limit(ShoutsCache.limit)
),
)
async with ShoutsCache.lock: async with ShoutsCache.lock:
ShoutsCache.recent_reacted = shouts ShoutsCache.recent_reacted = shouts
print("[zine.cache] %d recently reacted shouts " % len(shouts)) print("[zine.cache] %d recently reacted shouts " % len(shouts))
@ -84,20 +98,23 @@ class ShoutsCache:
async def prepare_top_overall(): async def prepare_top_overall():
with local_session() as session: with local_session() as session:
# with reacted times counter # with reacted times counter
shouts = await prepare_shouts(session, ( shouts = await prepare_shouts(
select(Shout, func.count(Reaction.id).label("reacted")) session,
.options( (
selectinload(Shout.authors), select(Shout, func.count(Reaction.id).label("reacted"))
selectinload(Shout.topics), .options(
selectinload(Shout.reactions), selectinload(Shout.authors),
) selectinload(Shout.topics),
.join(Reaction) selectinload(Shout.reactions),
.where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt))) )
.group_by(Shout.slug) .join(Reaction)
.order_by(desc("reacted")) .where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt)))
.limit(ShoutsCache.limit) .group_by(Shout.slug)
)) .order_by(desc("reacted"))
shouts.sort(key=lambda s: s.stats['rating'], reverse=True) .limit(ShoutsCache.limit)
),
)
shouts.sort(key=lambda s: s.stats["rating"], reverse=True)
async with ShoutsCache.lock: async with ShoutsCache.lock:
print("[zine.cache] %d top shouts " % len(shouts)) print("[zine.cache] %d top shouts " % len(shouts))
ShoutsCache.top_overall = shouts ShoutsCache.top_overall = shouts
@ -106,34 +123,61 @@ class ShoutsCache:
async def prepare_top_month(): async def prepare_top_month():
month_ago = datetime.now() - timedelta(days=30) month_ago = datetime.now() - timedelta(days=30)
with local_session() as session: with local_session() as session:
shouts = await prepare_shouts(session, ( shouts = await prepare_shouts(
select(Shout, func.count(Reaction.id).label("reacted")) session,
.options(selectinload(Shout.authors), selectinload(Shout.topics)) (
.join(Reaction) select(Shout, func.count(Reaction.id).label("reacted"))
.where(and_(Shout.createdAt > month_ago, bool(Reaction.deletedAt))) .options(selectinload(Shout.authors), selectinload(Shout.topics))
.group_by(Shout.slug) .join(Reaction)
.order_by(desc("reacted")) .where(and_(Shout.createdAt > month_ago, bool(Reaction.deletedAt)))
.limit(ShoutsCache.limit) .group_by(Shout.slug)
)) .order_by(desc("reacted"))
shouts.sort(key=lambda s: s.stats['rating'], reverse=True) .limit(ShoutsCache.limit)
),
)
shouts.sort(key=lambda s: s.stats["rating"], reverse=True)
async with ShoutsCache.lock: async with ShoutsCache.lock:
print("[zine.cache] %d top month shouts " % len(shouts)) print("[zine.cache] %d top month shouts " % len(shouts))
ShoutsCache.top_month = shouts ShoutsCache.top_month = shouts
@staticmethod
async def prepare_top_commented():
month_ago = datetime.now() - timedelta(days=30)
with local_session() as session:
shouts = await prepare_shouts(
session,
(
select(Shout, Reaction)
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.join(Reaction)
.where(and_(Shout.createdAt > month_ago, bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("commented"))
.limit(ShoutsCache.limit)
),
)
shouts.sort(key=lambda s: s.stats["commented"], reverse=True)
async with ShoutsCache.lock:
print("[zine.cache] %d top commented shouts " % len(shouts))
ShoutsCache.top_viewed = shouts
@staticmethod @staticmethod
async def prepare_top_viewed(): async def prepare_top_viewed():
month_ago = datetime.now() - timedelta(days=30) month_ago = datetime.now() - timedelta(days=30)
with local_session() as session: with local_session() as session:
shouts = await prepare_shouts(session, ( shouts = await prepare_shouts(
select(Shout, func.sum(ViewedByDay.value).label("viewed")) session,
.options(selectinload(Shout.authors), selectinload(Shout.topics)) (
.join(ViewedByDay) select(Shout, func.sum(ViewedByDay.value).label("viewed"))
.where(and_(Shout.createdAt > month_ago, bool(Reaction.deletedAt))) .options(selectinload(Shout.authors), selectinload(Shout.topics))
.group_by(Shout.slug) .join(ViewedByDay)
.order_by(desc("viewed")) .where(and_(Shout.createdAt > month_ago, bool(Reaction.deletedAt)))
.limit(ShoutsCache.limit) .group_by(Shout.slug)
)) .order_by(desc("viewed"))
shouts.sort(key=lambda s: s.stats['viewed'], reverse=True) .limit(ShoutsCache.limit)
),
)
shouts.sort(key=lambda s: s.stats["viewed"], reverse=True)
async with ShoutsCache.lock: async with ShoutsCache.lock:
print("[zine.cache] %d top viewed shouts " % len(shouts)) print("[zine.cache] %d top viewed shouts " % len(shouts))
ShoutsCache.top_viewed = shouts ShoutsCache.top_viewed = shouts

9
settings.py
View File

@ -5,8 +5,7 @@ INBOX_SERVICE_PORT = 8081
BACKEND_URL = environ.get("BACKEND_URL") or "https://localhost:8080" BACKEND_URL = environ.get("BACKEND_URL") or "https://localhost:8080"
OAUTH_CALLBACK_URL = environ.get("OAUTH_CALLBACK_URL") or "https://localhost:8080" OAUTH_CALLBACK_URL = environ.get("OAUTH_CALLBACK_URL") or "https://localhost:8080"
RESET_PWD_URL = environ.get("RESET_PWD_URL") or "https://localhost:8080/reset_pwd" CONFIRM_EMAIL_URL = environ.get("AUTH_CONFIRM_URL") or BACKEND_URL + "/confirm"
CONFIRM_EMAIL_URL = environ.get("CONFIRM_EMAIL_URL") or "https://new.discours.io"
ERROR_URL_ON_FRONTEND = ( ERROR_URL_ON_FRONTEND = (
environ.get("ERROR_URL_ON_FRONTEND") or "https://new.discours.io" environ.get("ERROR_URL_ON_FRONTEND") or "https://new.discours.io"
) )
@ -17,9 +16,9 @@ DB_URL = (
) )
JWT_ALGORITHM = "HS256" JWT_ALGORITHM = "HS256"
JWT_SECRET_KEY = "8f1bd7696ffb482d8486dfbc6e7d16dd-secret-key" JWT_SECRET_KEY = "8f1bd7696ffb482d8486dfbc6e7d16dd-secret-key"
JWT_LIFE_SPAN = 24 * 60 * 60 # seconds SESSION_TOKEN_HEADER = "Auth"
JWT_AUTH_HEADER = "Auth" SESSION_TOKEN_LIFE_SPAN = 24 * 60 * 60 # seconds
EMAIL_TOKEN_LIFE_SPAN = 1 * 60 * 60 # seconds ONETIME_TOKEN_LIFE_SPAN = 1 * 60 * 60 # seconds
REDIS_URL = environ.get("REDIS_URL") or "redis://127.0.0.1" REDIS_URL = environ.get("REDIS_URL") or "redis://127.0.0.1"
MAILGUN_API_KEY = environ.get("MAILGUN_API_KEY") MAILGUN_API_KEY = environ.get("MAILGUN_API_KEY")

16
validations/auth.py Normal file
View File

@ -0,0 +1,16 @@
from datetime import datetime
from typing import Optional, Text
from pydantic import BaseModel
class AuthInput(BaseModel):
id: Optional[int]
username: Optional[Text]
password: Optional[Text]
class TokenPayload(BaseModel):
user_id: int
exp: datetime
iat: datetime