discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

lint

Browse Source
This commit is contained in:
Igor Lobanov 2023-10-26 22:38:31 +02:00
parent 1c49780cd4
commit c2cc428abe
64 changed files with 631 additions and 626 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.flake8
View File

@ -1,6 +1,6 @@
[flake8] [flake8]
ignore = E203,W504,W191,W503 ignore = E203
exclude = .git,__pycache__,orm/rbac.py exclude = .git,__pycache__,orm/rbac.py
max-complexity = 10 max-complexity = 15
max-line-length = 108 max-line-length = 100
indent-string = ' ' indent-string = ' '

5
.pre-commit-config.yaml
View File

@ -17,7 +17,6 @@ repos:
- id: check-docstring-first - id: check-docstring-first
- id: check-json - id: check-json
- id: check-merge-conflict - id: check-merge-conflict
- id: check-toml
- id: check-yaml - id: check-yaml
- id: end-of-file-fixer - id: end-of-file-fixer
- id: trailing-whitespace - id: trailing-whitespace
@ -33,12 +32,8 @@ repos:
- id: black - id: black
args: args:
- --line-length=100 - --line-length=100
- --skip-string-normalization
- repo: https://github.com/PyCQA/flake8 - repo: https://github.com/PyCQA/flake8
rev: 6.1.0 rev: 6.1.0
hooks: hooks:
- id: flake8 - id: flake8
args:
- --max-line-length=100
- --disable=protected-access

21
ai/preprocess.py
View File

@ -1,29 +1,28 @@
import re
from string import punctuation
import nltk
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from nltk.corpus import stopwords from nltk.corpus import stopwords
from pymystem3 import Mystem from pymystem3 import Mystem
from transformers import BertTokenizer from string import punctuation
import nltk
import re
nltk.download("stopwords") nltk.download("stopwords")
def get_clear_text(text): def get_clear_text(text):
soup = BeautifulSoup(text, 'html.parser') soup = BeautifulSoup(text, "html.parser")
# extract the plain text from the HTML document without tags # extract the plain text from the HTML document without tags
clear_text = '' clear_text = ""
for tag in soup.find_all(): for tag in soup.find_all():
clear_text += tag.string or '' clear_text += tag.string or ""
clear_text = re.sub(pattern='[\u202F\u00A0\n]+', repl=' ', string=clear_text) clear_text = re.sub(pattern="[\u202F\u00A0\n]+", repl=" ", string=clear_text)
# only words # only words
clear_text = re.sub(pattern='[^A-ZА-ЯЁ -]', repl='', string=clear_text, flags=re.IGNORECASE) clear_text = re.sub(pattern="[^A-ZА-ЯЁ -]", repl="", string=clear_text, flags=re.IGNORECASE)
clear_text = re.sub(pattern='\s+', repl=' ', string=clear_text) clear_text = re.sub(pattern=r"\s+", repl=" ", string=clear_text)
clear_text = clear_text.lower() clear_text = clear_text.lower()

9
alembic/env.py
View File

@ -1,9 +1,8 @@
from logging.config import fileConfig
from sqlalchemy import engine_from_config, pool
from alembic import context from alembic import context
from base.orm import Base
from logging.config import fileConfig
from settings import DB_URL from settings import DB_URL
from sqlalchemy import engine_from_config, pool
# this is the Alembic Config object, which provides # this is the Alembic Config object, which provides
# access to the values within the .ini file in use. # access to the values within the .ini file in use.
@ -17,8 +16,6 @@ config.set_section_option(config.config_ini_section, "DB_URL", DB_URL)
if config.config_file_name is not None: if config.config_file_name is not None:
fileConfig(config.config_file_name) fileConfig(config.config_file_name)
from base.orm import Base
target_metadata = [Base.metadata] target_metadata = [Base.metadata]
# other values from the config, defined by the needs of env.py, # other values from the config, defined by the needs of env.py,

6
alembic/versions/fe943b098418_init_alembic.py
View File

@ -7,12 +7,12 @@ Create Date: 2023-08-19 01:37:57.031933
""" """
from typing import Sequence, Union from typing import Sequence, Union
import sqlalchemy as sa # import sqlalchemy as sa
from alembic import op # from alembic import op
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision: str = 'fe943b098418' revision: str = "fe943b098418"
down_revision: Union[str, None] = None down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None

31
auth/authenticate.py
View File

@ -1,17 +1,15 @@
from functools import wraps
from typing import Optional, Tuple
from graphql.type import GraphQLResolveInfo
from sqlalchemy.orm import exc, joinedload
from starlette.authentication import AuthenticationBackend
from starlette.requests import HTTPConnection
from auth.credentials import AuthCredentials, AuthUser from auth.credentials import AuthCredentials, AuthUser
from auth.tokenstorage import SessionToken from auth.tokenstorage import SessionToken
from base.exceptions import OperationNotAllowed from base.exceptions import OperationNotAllowed
from base.orm import local_session from base.orm import local_session
from functools import wraps
from graphql.type import GraphQLResolveInfo
from orm.user import Role, User from orm.user import Role, User
from settings import SESSION_TOKEN_HEADER from settings import SESSION_TOKEN_HEADER
from sqlalchemy.orm import exc, joinedload
from starlette.authentication import AuthenticationBackend
from starlette.requests import HTTPConnection
from typing import Optional, Tuple
class JWTAuthenticate(AuthenticationBackend): class JWTAuthenticate(AuthenticationBackend):
@ -19,16 +17,16 @@ class JWTAuthenticate(AuthenticationBackend):
self, request: HTTPConnection self, request: HTTPConnection
) -> Optional[Tuple[AuthCredentials, AuthUser]]: ) -> Optional[Tuple[AuthCredentials, AuthUser]]:
if SESSION_TOKEN_HEADER not in request.headers: if SESSION_TOKEN_HEADER not in request.headers:
return AuthCredentials(scopes={}), AuthUser(user_id=None, username='') return AuthCredentials(scopes={}), AuthUser(user_id=None, username="")
token = request.headers.get(SESSION_TOKEN_HEADER) token = request.headers.get(SESSION_TOKEN_HEADER)
if not token: if not token:
print("[auth.authenticate] no token in header %s" % SESSION_TOKEN_HEADER) print("[auth.authenticate] no token in header %s" % SESSION_TOKEN_HEADER)
return AuthCredentials(scopes={}, error_message=str("no token")), AuthUser( return AuthCredentials(scopes={}, error_message=str("no token")), AuthUser(
user_id=None, username='' user_id=None, username=""
) )
if len(token.split('.')) > 1: if len(token.split(".")) > 1:
payload = await SessionToken.verify(token) payload = await SessionToken.verify(token)
with local_session() as session: with local_session() as session:
@ -47,20 +45,21 @@ class JWTAuthenticate(AuthenticationBackend):
return ( return (
AuthCredentials(user_id=payload.user_id, scopes=scopes, logged_in=True), AuthCredentials(user_id=payload.user_id, scopes=scopes, logged_in=True),
AuthUser(user_id=user.id, username=''), AuthUser(user_id=user.id, username=""),
) )
except exc.NoResultFound: except exc.NoResultFound:
pass pass
return AuthCredentials(scopes={}, error_message=str('Invalid token')), AuthUser( return AuthCredentials(scopes={}, error_message=str("Invalid token")), AuthUser(
user_id=None, username='' user_id=None, username=""
) )
def login_required(func): def login_required(func):
@wraps(func) @wraps(func)
async def wrap(parent, info: GraphQLResolveInfo, *args, **kwargs): async def wrap(parent, info: GraphQLResolveInfo, *args, **kwargs):
# print('[auth.authenticate] login required for %r with info %r' % (func, info)) # debug only # debug only
# print('[auth.authenticate] login required for %r with info %r' % (func, info))
auth: AuthCredentials = info.context["request"].auth auth: AuthCredentials = info.context["request"].auth
# print(auth) # print(auth)
if not auth or not auth.logged_in: if not auth or not auth.logged_in:
@ -75,7 +74,7 @@ def permission_required(resource, operation, func):
@wraps(func) @wraps(func)
async def wrap(parent, info: GraphQLResolveInfo, *args, **kwargs): async def wrap(parent, info: GraphQLResolveInfo, *args, **kwargs):
print( print(
'[auth.authenticate] permission_required for %r with info %r' % (func, info) "[auth.authenticate] permission_required for %r with info %r" % (func, info)
) # debug only ) # debug only
auth: AuthCredentials = info.context["request"].auth auth: AuthCredentials = info.context["request"].auth
if not auth.logged_in: if not auth.logged_in:

3
auth/credentials.py
View File

@ -1,6 +1,5 @@
from typing import List, Optional, Text
from pydantic import BaseModel from pydantic import BaseModel
from typing import List, Optional, Text
# from base.exceptions import Unauthorized # from base.exceptions import Unauthorized

16
auth/email.py
View File

@ -1,17 +1,17 @@
import requests
from settings import MAILGUN_API_KEY, MAILGUN_DOMAIN from settings import MAILGUN_API_KEY, MAILGUN_DOMAIN
api_url = "https://api.mailgun.net/v3/%s/messages" % (MAILGUN_DOMAIN or 'discours.io') import requests
noreply = "discours.io <noreply@%s>" % (MAILGUN_DOMAIN or 'discours.io')
api_url = "https://api.mailgun.net/v3/%s/messages" % (MAILGUN_DOMAIN or "discours.io")
noreply = "discours.io <noreply@%s>" % (MAILGUN_DOMAIN or "discours.io")
lang_subject = {"ru": "Подтверждение почты", "en": "Confirm email"} lang_subject = {"ru": "Подтверждение почты", "en": "Confirm email"}
async def send_auth_email(user, token, lang="ru", template="email_confirmation"): async def send_auth_email(user, token, lang="ru", template="email_confirmation"):
try: try:
to = "%s <%s>" % (user.name, user.email) to = "%s <%s>" % (user.name, user.email)
if lang not in ['ru', 'en']: if lang not in ["ru", "en"]:
lang = 'ru' lang = "ru"
subject = lang_subject.get(lang, lang_subject["en"]) subject = lang_subject.get(lang, lang_subject["en"])
template = template + "_" + lang template = template + "_" + lang
payload = { payload = {
@ -19,9 +19,9 @@ async def send_auth_email(user, token, lang="ru", template="email_confirmation")
"to": to, "to": to,
"subject": subject, "subject": subject,
"template": template, "template": template,
"h:X-Mailgun-Variables": "{ \"token\": \"%s\" }" % token, "h:X-Mailgun-Variables": '{ "token": "%s" }' % token,
} }
print('[auth.email] payload: %r' % payload) print("[auth.email] payload: %r" % payload)
# debug # debug
# print('http://localhost:3000/?modal=auth&mode=confirm-email&token=%s' % token) # print('http://localhost:3000/?modal=auth&mode=confirm-email&token=%s' % token)
response = requests.post(api_url, auth=("api", MAILGUN_API_KEY), data=payload) response = requests.post(api_url, auth=("api", MAILGUN_API_KEY), data=payload)

15
auth/identity.py
View File

@ -1,16 +1,14 @@
from binascii import hexlify
from hashlib import sha256
from jwt import DecodeError, ExpiredSignatureError
from passlib.hash import bcrypt
from sqlalchemy import or_
from auth.jwtcodec import JWTCodec from auth.jwtcodec import JWTCodec
from auth.tokenstorage import TokenStorage from auth.tokenstorage import TokenStorage
# from base.exceptions import InvalidPassword, InvalidToken # from base.exceptions import InvalidPassword, InvalidToken
from base.orm import local_session from base.orm import local_session
from binascii import hexlify
from hashlib import sha256
from jwt import DecodeError, ExpiredSignatureError
from orm import User from orm import User
from passlib.hash import bcrypt
from sqlalchemy import or_
from validations.auth import AuthInput from validations.auth import AuthInput
@ -35,6 +33,7 @@ class Password:
Verify that password hash is equal to specified hash. Hash format: Verify that password hash is equal to specified hash. Hash format:
$2a$10$Ro0CUfOqk6cXEKf3dyaM7OhSCvnwM9s4wIX9JeLapehKK5YdLxKcm $2a$10$Ro0CUfOqk6cXEKf3dyaM7OhSCvnwM9s4wIX9JeLapehKK5YdLxKcm
# noqa: W605
\__/\/ \____________________/\_____________________________/ \__/\/ \____________________/\_____________________________/
| | Salt Hash | | Salt Hash
| Cost | Cost
@ -84,7 +83,7 @@ class Identity:
@staticmethod @staticmethod
async def onetime(token: str) -> User: async def onetime(token: str) -> User:
try: try:
print('[auth.identity] using one time token') print("[auth.identity] using one time token")
payload = JWTCodec.decode(token) payload = JWTCodec.decode(token)
if not await TokenStorage.exist(f"{payload.user_id}-{payload.username}-{token}"): if not await TokenStorage.exist(f"{payload.user_id}-{payload.username}-{token}"):
# raise InvalidToken("Login token has expired, please login again") # raise InvalidToken("Login token has expired, please login again")

21
auth/jwtcodec.py
View File

@ -1,11 +1,10 @@
from datetime import datetime, timezone
import jwt
from base.exceptions import ExpiredToken, InvalidToken from base.exceptions import ExpiredToken, InvalidToken
from datetime import datetime, timezone
from settings import JWT_ALGORITHM, JWT_SECRET_KEY from settings import JWT_ALGORITHM, JWT_SECRET_KEY
from validations.auth import AuthInput, TokenPayload from validations.auth import AuthInput, TokenPayload
import jwt
class JWTCodec: class JWTCodec:
@staticmethod @staticmethod
@ -20,7 +19,7 @@ class JWTCodec:
try: try:
return jwt.encode(payload, JWT_SECRET_KEY, JWT_ALGORITHM) return jwt.encode(payload, JWT_SECRET_KEY, JWT_ALGORITHM)
except Exception as e: except Exception as e:
print('[auth.jwtcodec] JWT encode error %r' % e) print("[auth.jwtcodec] JWT encode error %r" % e)
@staticmethod @staticmethod
def decode(token: str, verify_exp: bool = True) -> TokenPayload: def decode(token: str, verify_exp: bool = True) -> TokenPayload:
@ -41,12 +40,12 @@ class JWTCodec:
# print('[auth.jwtcodec] debug token %r' % r) # print('[auth.jwtcodec] debug token %r' % r)
return r return r
except jwt.InvalidIssuedAtError: except jwt.InvalidIssuedAtError:
print('[auth.jwtcodec] invalid issued at: %r' % payload) print("[auth.jwtcodec] invalid issued at: %r" % payload)
raise ExpiredToken('check token issued time') raise ExpiredToken("check token issued time")
except jwt.ExpiredSignatureError: except jwt.ExpiredSignatureError:
print('[auth.jwtcodec] expired signature %r' % payload) print("[auth.jwtcodec] expired signature %r" % payload)
raise ExpiredToken('check token lifetime') raise ExpiredToken("check token lifetime")
except jwt.InvalidTokenError: except jwt.InvalidTokenError:
raise InvalidToken('token is not valid') raise InvalidToken("token is not valid")
except jwt.InvalidSignatureError: except jwt.InvalidSignatureError:
raise InvalidToken('token is not valid') raise InvalidToken("token is not valid")

5
auth/oauth.py
View File

@ -1,9 +1,8 @@
from authlib.integrations.starlette_client import OAuth
from starlette.responses import RedirectResponse
from auth.identity import Identity from auth.identity import Identity
from auth.tokenstorage import TokenStorage from auth.tokenstorage import TokenStorage
from authlib.integrations.starlette_client import OAuth
from settings import FRONTEND_URL, OAUTH_CLIENTS from settings import FRONTEND_URL, OAUTH_CLIENTS
from starlette.responses import RedirectResponse
oauth = OAuth() oauth = OAuth()

5
auth/tokenstorage.py
View File

@ -1,7 +1,6 @@
from datetime import datetime, timedelta, timezone
from auth.jwtcodec import JWTCodec from auth.jwtcodec import JWTCodec
from base.redis import redis from base.redis import redis
from datetime import datetime, timedelta, timezone
from settings import ONETIME_TOKEN_LIFE_SPAN, SESSION_TOKEN_LIFE_SPAN from settings import ONETIME_TOKEN_LIFE_SPAN, SESSION_TOKEN_LIFE_SPAN
from validations.auth import AuthInput from validations.auth import AuthInput
@ -35,7 +34,7 @@ class SessionToken:
class TokenStorage: class TokenStorage:
@staticmethod @staticmethod
async def get(token_key): async def get(token_key):
print('[tokenstorage.get] ' + token_key) print("[tokenstorage.get] " + token_key)
# 2041-user@domain.zn-eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoyMDQxLCJ1c2VybmFtZSI6ImFudG9uLnJld2luK3Rlc3QtbG9hZGNoYXRAZ21haWwuY29tIiwiZXhwIjoxNjcxNzgwNjE2LCJpYXQiOjE2NjkxODg2MTYsImlzcyI6ImRpc2NvdXJzIn0.Nml4oV6iMjMmc6xwM7lTKEZJKBXvJFEIZ-Up1C1rITQ # 2041-user@domain.zn-eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoyMDQxLCJ1c2VybmFtZSI6ImFudG9uLnJld2luK3Rlc3QtbG9hZGNoYXRAZ21haWwuY29tIiwiZXhwIjoxNjcxNzgwNjE2LCJpYXQiOjE2NjkxODg2MTYsImlzcyI6ImRpc2NvdXJzIn0.Nml4oV6iMjMmc6xwM7lTKEZJKBXvJFEIZ-Up1C1rITQ
return await redis.execute("GET", token_key) return await redis.execute("GET", token_key)

8
base/orm.py
View File

@ -1,11 +1,9 @@
from typing import Any, Callable, Dict, Generic, TypeVar from settings import DB_URL
from sqlalchemy import Column, create_engine, Integer
from sqlalchemy import Column, Integer, create_engine
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from sqlalchemy.sql.schema import Table from sqlalchemy.sql.schema import Table
from typing import Any, Callable, Dict, Generic, TypeVar
from settings import DB_URL
engine = create_engine(DB_URL, echo=False, pool_size=10, max_overflow=20) engine = create_engine(DB_URL, echo=False, pool_size=10, max_overflow=20)

4
base/redis.py
View File

@ -1,7 +1,5 @@
from asyncio import sleep
from aioredis import from_url from aioredis import from_url
from asyncio import sleep
from settings import REDIS_URL from settings import REDIS_URL

27
main.py
View File

@ -1,21 +1,12 @@
import asyncio
import os
from importlib import import_module
from os.path import exists
from ariadne import load_schema_from_path, make_executable_schema from ariadne import load_schema_from_path, make_executable_schema
from ariadne.asgi import GraphQL from ariadne.asgi import GraphQL
from starlette.applications import Starlette
from starlette.middleware import Middleware
from starlette.middleware.authentication import AuthenticationMiddleware
from starlette.middleware.sessions import SessionMiddleware
from starlette.routing import Route
from auth.authenticate import JWTAuthenticate from auth.authenticate import JWTAuthenticate
from auth.oauth import oauth_authorize, oauth_login from auth.oauth import oauth_authorize, oauth_login
from base.redis import redis from base.redis import redis
from base.resolvers import resolvers from base.resolvers import resolvers
from importlib import import_module
from orm import init_tables from orm import init_tables
from os.path import exists
from resolvers.auth import confirm_email_handler from resolvers.auth import confirm_email_handler
from resolvers.upload import upload_handler from resolvers.upload import upload_handler
from services.main import storages_init from services.main import storages_init
@ -25,6 +16,14 @@ from services.stat.viewed import ViewedStorage
# from services.zine.gittask import GitTask # from services.zine.gittask import GitTask
from settings import DEV_SERVER_PID_FILE_NAME, SENTRY_DSN, SESSION_SECRET_KEY from settings import DEV_SERVER_PID_FILE_NAME, SENTRY_DSN, SESSION_SECRET_KEY
from starlette.applications import Starlette
from starlette.middleware import Middleware
from starlette.middleware.authentication import AuthenticationMiddleware
from starlette.middleware.sessions import SessionMiddleware
from starlette.routing import Route
import asyncio
import os
import_module("resolvers") import_module("resolvers")
schema = make_executable_schema(load_schema_from_path("schema.graphql"), resolvers) # type: ignore schema = make_executable_schema(load_schema_from_path("schema.graphql"), resolvers) # type: ignore
@ -51,7 +50,7 @@ async def start_up():
sentry_sdk.init(SENTRY_DSN) sentry_sdk.init(SENTRY_DSN)
except Exception as e: except Exception as e:
print('[sentry] init error') print("[sentry] init error")
print(e) print(e)
@ -60,7 +59,7 @@ async def dev_start_up():
await redis.connect() await redis.connect()
return return
else: else:
with open(DEV_SERVER_PID_FILE_NAME, 'w', encoding='utf-8') as f: with open(DEV_SERVER_PID_FILE_NAME, "w", encoding="utf-8") as f:
f.write(str(os.getpid())) f.write(str(os.getpid()))
await start_up() await start_up()
@ -75,7 +74,7 @@ routes = [
Route("/oauth/{provider}", endpoint=oauth_login), Route("/oauth/{provider}", endpoint=oauth_login),
Route("/oauth-authorize", endpoint=oauth_authorize), Route("/oauth-authorize", endpoint=oauth_authorize),
Route("/confirm/{token}", endpoint=confirm_email_handler), Route("/confirm/{token}", endpoint=confirm_email_handler),
Route("/upload", endpoint=upload_handler, methods=['POST']), Route("/upload", endpoint=upload_handler, methods=["POST"]),
Route("/subscribe/{user_id}", endpoint=sse_subscribe_handler), Route("/subscribe/{user_id}", endpoint=sse_subscribe_handler),
] ]

36
migration/__init__.py
View File

@ -1,18 +1,12 @@
""" cmd managed migration """ """ cmd managed migration """
import asyncio
import gc
import json
import sys
from datetime import datetime, timezone from datetime import datetime, timezone
import bs4
from migration.export import export_mdx from migration.export import export_mdx
from migration.tables.comments import migrate as migrateComment from migration.tables.comments import migrate as migrateComment
from migration.tables.comments import migrate_2stage as migrateComment_2stage from migration.tables.comments import migrate_2stage as migrateComment_2stage
from migration.tables.content_items import get_shout_slug from migration.tables.content_items import get_shout_slug
from migration.tables.content_items import migrate as migrateShout from migration.tables.content_items import migrate as migrateShout
from migration.tables.remarks import migrate as migrateRemark
# from migration.tables.remarks import migrate as migrateRemark
from migration.tables.topics import migrate as migrateTopic from migration.tables.topics import migrate as migrateTopic
from migration.tables.users import migrate as migrateUser from migration.tables.users import migrate as migrateUser
from migration.tables.users import migrate_2stage as migrateUser_2stage from migration.tables.users import migrate_2stage as migrateUser_2stage
@ -20,6 +14,12 @@ from migration.tables.users import post_migrate as users_post_migrate
from orm import init_tables from orm import init_tables
from orm.reaction import Reaction from orm.reaction import Reaction
import asyncio
import bs4
import gc
import json
import sys
TODAY = datetime.strftime(datetime.now(tz=timezone.utc), "%Y%m%d") TODAY = datetime.strftime(datetime.now(tz=timezone.utc), "%Y%m%d")
OLD_DATE = "2016-03-05 22:22:00.350000" OLD_DATE = "2016-03-05 22:22:00.350000"
@ -111,7 +111,7 @@ async def shouts_handle(storage, args):
# print main counter # print main counter
counter += 1 counter += 1
print( print(
'[migration] shouts_handle %d: %s @%s' "[migration] shouts_handle %d: %s @%s"
% ((counter + 1), shout_dict["slug"], author["slug"]) % ((counter + 1), shout_dict["slug"], author["slug"])
) )
@ -132,13 +132,13 @@ async def shouts_handle(storage, args):
print("[migration] " + str(anonymous_author) + " authored by @anonymous") print("[migration] " + str(anonymous_author) + " authored by @anonymous")
async def remarks_handle(storage): # async def remarks_handle(storage):
print("[migration] comments") # print("[migration] comments")
c = 0 # c = 0
for entry_remark in storage["remarks"]["data"]: # for entry_remark in storage["remarks"]["data"]:
remark = await migrateRemark(entry_remark, storage) # remark = await migrateRemark(entry_remark, storage)
c += 1 # c += 1
print("[migration] " + str(c) + " remarks migrated") # print("[migration] " + str(c) + " remarks migrated")
async def comments_handle(storage): async def comments_handle(storage):
@ -149,9 +149,9 @@ async def comments_handle(storage):
for oldcomment in storage["reactions"]["data"]: for oldcomment in storage["reactions"]["data"]:
if not oldcomment.get("deleted"): if not oldcomment.get("deleted"):
reaction = await migrateComment(oldcomment, storage) reaction = await migrateComment(oldcomment, storage)
if type(reaction) == str: if isinstance(reaction, str):
missed_shouts[reaction] = oldcomment missed_shouts[reaction] = oldcomment
elif type(reaction) == Reaction: elif isinstance(reaction, Reaction):
reaction = reaction.dict() reaction = reaction.dict()
rid = reaction["id"] rid = reaction["id"]
oid = reaction["oid"] oid = reaction["oid"]

9
migration/bson2json.py
View File

@ -1,11 +1,10 @@
from .utils import DateTimeEncoder
import bson
import gc import gc
import json import json
import os import os
import bson
from .utils import DateTimeEncoder
def json_tables(): def json_tables():
print("[migration] unpack dump/discours/*.bson to migration/data/*.json") print("[migration] unpack dump/discours/*.bson to migration/data/*.json")
@ -19,7 +18,7 @@ def json_tables():
"remarks": [], "remarks": [],
} }
for table in data.keys(): for table in data.keys():
print('[migration] bson2json for ' + table) print("[migration] bson2json for " + table)
gc.collect() gc.collect()
lc = [] lc = []
bs = open("dump/discours/" + table + ".bson", "rb").read() bs = open("dump/discours/" + table + ".bson", "rb").read()

9
migration/export.py
View File

@ -1,11 +1,10 @@
import json from .extract import extract_html, extract_media
import os from .utils import DateTimeEncoder
from datetime import datetime, timezone from datetime import datetime, timezone
import frontmatter import frontmatter
import json
from .extract import extract_html, extract_media import os
from .utils import DateTimeEncoder
OLD_DATE = "2016-03-05 22:22:00.350000" OLD_DATE = "2016-03-05 22:22:00.350000"
EXPORT_DEST = "../discoursio-web/data/" EXPORT_DEST = "../discoursio-web/data/"

125
migration/extract.py
View File

@ -1,9 +1,11 @@
from bs4 import BeautifulSoup
import base64 import base64
import os import os
import re import re
import uuid
from bs4 import BeautifulSoup # import uuid
TOOLTIP_REGEX = r"(\/\/\/(.+)\/\/\/)" TOOLTIP_REGEX = r"(\/\/\/(.+)\/\/\/)"
contentDir = os.path.join( contentDir = os.path.join(
@ -26,40 +28,40 @@ def replace_tooltips(body):
return newbody return newbody
def extract_footnotes(body, shout_dict): # def extract_footnotes(body, shout_dict):
parts = body.split("&&&") # parts = body.split("&&&")
lll = len(parts) # lll = len(parts)
newparts = list(parts) # newparts = list(parts)
placed = False # placed = False
if lll & 1: # if lll & 1:
if lll > 1: # if lll > 1:
i = 1 # i = 1
print("[extract] found %d footnotes in body" % (lll - 1)) # print("[extract] found %d footnotes in body" % (lll - 1))
for part in parts[1:]: # for part in parts[1:]:
if i & 1: # if i & 1:
placed = True # placed = True
if 'a class="footnote-url" href=' in part: # if 'a class="footnote-url" href=' in part:
print("[extract] footnote: " + part) # print("[extract] footnote: " + part)
fn = 'a class="footnote-url" href="' # fn = 'a class="footnote-url" href="'
exxtracted_link = part.split(fn, 1)[1].split('"', 1)[0] # # exxtracted_link = part.split(fn, 1)[1].split('"', 1)[0]
extracted_body = part.split(fn, 1)[1].split('>', 1)[1].split('</a>', 1)[0] # extracted_body = part.split(fn, 1)[1].split(">", 1)[1].split("</a>", 1)[0]
print("[extract] footnote link: " + extracted_link) # print("[extract] footnote link: " + extracted_link)
with local_session() as session: # with local_session() as session:
Reaction.create( # Reaction.create(
{ # {
"shout": shout_dict['id'], # "shout": shout_dict["id"],
"kind": ReactionKind.FOOTNOTE, # "kind": ReactionKind.FOOTNOTE,
"body": extracted_body, # "body": extracted_body,
"range": str(body.index(fn + link) - len('<')) # "range": str(body.index(fn + link) - len("<"))
+ ':' # + ":"
+ str(body.index(extracted_body) + len('</a>')), # + str(body.index(extracted_body) + len("</a>")),
} # }
) # )
newparts[i] = "<a href='#'>ℹ️</a>" # newparts[i] = "<a href='#'>ℹ️</a>"
else: # else:
newparts[i] = part # newparts[i] = part
i += 1 # i += 1
return ("".join(newparts), placed) # return ("".join(newparts), placed)
def place_tooltips(body): def place_tooltips(body):
@ -228,7 +230,6 @@ di = "data:image"
def extract_md_images(body, prefix): def extract_md_images(body, prefix):
newbody = ""
body = ( body = (
body.replace("\n! [](" + di, "\n ![](" + di) body.replace("\n! [](" + di, "\n ![](" + di)
.replace("\n[](" + di, "\n![](" + di) .replace("\n[](" + di, "\n![](" + di)
@ -236,10 +237,10 @@ def extract_md_images(body, prefix):
) )
parts = body.split(di) parts = body.split(di)
if len(parts) > 1: if len(parts) > 1:
newbody = extract_dataimages(parts, prefix) new_body = extract_dataimages(parts, prefix)
else: else:
newbody = body new_body = body
return newbody return new_body
def cleanup_md(body): def cleanup_md(body):
@ -262,28 +263,28 @@ def cleanup_md(body):
return newbody return newbody
def extract_md(body, shout_dict=None): # def extract_md(body, shout_dict=None):
newbody = body # newbody = body
if newbody: # if newbody:
newbody = cleanup_md(newbody) # newbody = cleanup_md(newbody)
if not newbody: # if not newbody:
raise Exception("cleanup error") # raise Exception("cleanup error")
#
if shout_dict: # if shout_dict:
uid = shout_dict['id'] or uuid.uuid4() # uid = shout_dict["id"] or uuid.uuid4()
newbody = extract_md_images(newbody, uid) # newbody = extract_md_images(newbody, uid)
if not newbody: # if not newbody:
raise Exception("extract_images error") # raise Exception("extract_images error")
#
newbody, placed = extract_footnotes(body, shout_dict) # newbody, placed = extract_footnotes(body, shout_dict)
if not newbody: # if not newbody:
raise Exception("extract_footnotes error") # raise Exception("extract_footnotes error")
#
return newbody # return newbody
def extract_media(entry): def extract_media(entry):
'''normalized media extraction method''' """normalized media extraction method"""
# media [ { title pic url body } ]} # media [ { title pic url body } ]}
kind = entry.get("type") kind = entry.get("type")
if not kind: if not kind:
@ -398,16 +399,14 @@ def cleanup_html(body: str) -> str:
return new_body return new_body
def extract_html(entry, shout_id=None, cleanup=False): def extract_html(entry, cleanup=False):
body_orig = (entry.get("body") or "").replace('\(', '(').replace('\)', ')') body_orig = (entry.get("body") or "").replace(r"\(", "(").replace(r"\)", ")")
if cleanup: if cleanup:
# we do that before bs parsing to catch the invalid html # we do that before bs parsing to catch the invalid html
body_clean = cleanup_html(body_orig) body_clean = cleanup_html(body_orig)
if body_clean != body_orig: if body_clean != body_orig:
print(f"[migration] html cleaned for slug {entry.get('slug', None)}") print(f"[migration] html cleaned for slug {entry.get('slug', None)}")
body_orig = body_clean body_orig = body_clean
if shout_id:
extract_footnotes(body_orig, shout_id)
body_html = str(BeautifulSoup(body_orig, features="html.parser")) body_html = str(BeautifulSoup(body_orig, features="html.parser"))
if cleanup: if cleanup:
# we do that after bs parsing because it can add dummy tags # we do that after bs parsing because it can add dummy tags

16
migration/html2text/__init__.py
View File

@ -1,13 +1,5 @@
"""html2text: Turn HTML into equivalent Markdown-structured text.""" """html2text: Turn HTML into equivalent Markdown-structured text."""
import html.entities
import html.parser
import re
import string
import urllib.parse as urlparse
from textwrap import wrap
from typing import Dict, List, Optional, Tuple, Union
from . import config from . import config
from .elements import AnchorElement, ListElement from .elements import AnchorElement, ListElement
from .typing import OutCallback from .typing import OutCallback
@ -26,6 +18,14 @@ from .utils import (
skipwrap, skipwrap,
unifiable_n, unifiable_n,
) )
from textwrap import wrap
from typing import Dict, List, Optional, Tuple, Union
import html.entities
import html.parser
import re
import string
import urllib.parse as urlparse
__version__ = (2020, 1, 16) __version__ = (2020, 1, 16)

4
migration/html2text/cli.py
View File

@ -1,8 +1,8 @@
from . import __version__, config, HTML2Text
import argparse import argparse
import sys import sys
from . import HTML2Text, __version__, config
# noinspection DuplicatedCode # noinspection DuplicatedCode
def main() -> None: def main() -> None:

4
migration/html2text/utils.py
View File

@ -1,7 +1,7 @@
import html.entities from . import config
from typing import Dict, List, Optional from typing import Dict, List, Optional
from . import config import html.entities
unifiable_n = { unifiable_n = {
html.entities.name2codepoint[k]: v for k, v in config.UNIFIABLE.items() if k != "nbsp" html.entities.name2codepoint[k]: v for k, v in config.UNIFIABLE.items() if k != "nbsp"

20
migration/tables/comments.py
View File

@ -1,8 +1,6 @@
from datetime import datetime, timezone
from dateutil.parser import parse as date_parse
from base.orm import local_session from base.orm import local_session
from datetime import datetime, timezone
from dateutil.parser import parse as date_parse
from migration.html2text import html2text from migration.html2text import html2text
from orm.reaction import Reaction, ReactionKind from orm.reaction import Reaction, ReactionKind
from orm.shout import Shout, ShoutReactionsFollower from orm.shout import Shout, ShoutReactionsFollower
@ -30,12 +28,12 @@ def auto_followers(session, topics, reaction_dict):
tf = ( tf = (
session.query(TopicFollower) session.query(TopicFollower)
.where(TopicFollower.follower == reaction_dict["createdBy"]) .where(TopicFollower.follower == reaction_dict["createdBy"])
.filter(TopicFollower.topic == t['id']) .filter(TopicFollower.topic == t["id"])
.first() .first()
) )
if not tf: if not tf:
topic_following = TopicFollower.create( topic_following = TopicFollower.create(
follower=reaction_dict["createdBy"], topic=t['id'], auto=True follower=reaction_dict["createdBy"], topic=t["id"], auto=True
) )
session.add(topic_following) session.add(topic_following)
@ -57,13 +55,13 @@ def migrate_ratings(session, entry, reaction_dict):
rr = Reaction.create(**re_reaction_dict) rr = Reaction.create(**re_reaction_dict)
following2 = ( following2 = (
session.query(ShoutReactionsFollower) session.query(ShoutReactionsFollower)
.where(ShoutReactionsFollower.follower == re_reaction_dict['createdBy']) .where(ShoutReactionsFollower.follower == re_reaction_dict["createdBy"])
.filter(ShoutReactionsFollower.shout == rr.shout) .filter(ShoutReactionsFollower.shout == rr.shout)
.first() .first()
) )
if not following2: if not following2:
following2 = ShoutReactionsFollower.create( following2 = ShoutReactionsFollower.create(
follower=re_reaction_dict['createdBy'], shout=rr.shout, auto=True follower=re_reaction_dict["createdBy"], shout=rr.shout, auto=True
) )
session.add(following2) session.add(following2)
session.add(rr) session.add(rr)
@ -160,9 +158,9 @@ async def migrate(entry, storage):
def migrate_2stage(old_comment, idmap): def migrate_2stage(old_comment, idmap):
if old_comment.get('body'): if old_comment.get("body"):
new_id = idmap.get(old_comment.get('oid')) new_id = idmap.get(old_comment.get("oid"))
new_id = idmap.get(old_comment.get('_id')) new_id = idmap.get(old_comment.get("_id"))
if new_id: if new_id:
new_replyto_id = None new_replyto_id = None
old_replyto_id = old_comment.get("replyTo") old_replyto_id = old_comment.get("replyTo")

51
migration/tables/content_items.py
View File

@ -1,18 +1,17 @@
import json
import re
from datetime import datetime, timezone
from dateutil.parser import parse as date_parse
from sqlalchemy.exc import IntegrityError
from transliterate import translit
from base.orm import local_session from base.orm import local_session
from datetime import datetime, timezone
from dateutil.parser import parse as date_parse
from migration.extract import extract_html, extract_media from migration.extract import extract_html, extract_media
from orm.reaction import Reaction, ReactionKind from orm.reaction import Reaction, ReactionKind
from orm.shout import Shout, ShoutReactionsFollower, ShoutTopic from orm.shout import Shout, ShoutReactionsFollower, ShoutTopic
from orm.topic import Topic, TopicFollower from orm.topic import Topic, TopicFollower
from orm.user import User from orm.user import User
from services.stat.viewed import ViewedStorage from services.stat.viewed import ViewedStorage
from sqlalchemy.exc import IntegrityError
from transliterate import translit
import json
import re
OLD_DATE = "2016-03-05 22:22:00.350000" OLD_DATE = "2016-03-05 22:22:00.350000"
ts = datetime.now(tz=timezone.utc) ts = datetime.now(tz=timezone.utc)
@ -35,7 +34,7 @@ def get_shout_slug(entry):
slug = friend.get("slug", "") slug = friend.get("slug", "")
if slug: if slug:
break break
slug = re.sub('[^0-9a-zA-Z]+', '-', slug) slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
return slug return slug
@ -43,27 +42,27 @@ def create_author_from_app(app):
user = None user = None
userdata = None userdata = None
# check if email is used # check if email is used
if app['email']: if app["email"]:
with local_session() as session: with local_session() as session:
user = session.query(User).where(User.email == app['email']).first() user = session.query(User).where(User.email == app["email"]).first()
if not user: if not user:
# print('[migration] app %r' % app) # print('[migration] app %r' % app)
name = app.get('name') name = app.get("name")
if name: if name:
slug = translit(name, "ru", reversed=True).lower() slug = translit(name, "ru", reversed=True).lower()
slug = re.sub('[^0-9a-zA-Z]+', '-', slug) slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
print('[migration] created slug %s' % slug) print("[migration] created slug %s" % slug)
# check if slug is used # check if slug is used
if slug: if slug:
user = session.query(User).where(User.slug == slug).first() user = session.query(User).where(User.slug == slug).first()
# get slug from email # get slug from email
if user: if user:
slug = app['email'].split('@')[0] slug = app["email"].split("@")[0]
user = session.query(User).where(User.slug == slug).first() user = session.query(User).where(User.slug == slug).first()
# one more try # one more try
if user: if user:
slug += '-author' slug += "-author"
user = session.query(User).where(User.slug == slug).first() user = session.query(User).where(User.slug == slug).first()
# create user with application data # create user with application data
@ -81,7 +80,7 @@ def create_author_from_app(app):
user = User.create(**userdata) user = User.create(**userdata)
session.add(user) session.add(user)
session.commit() session.commit()
userdata['id'] = user.id userdata["id"] = user.id
userdata = user.dict() userdata = user.dict()
return userdata return userdata
@ -119,14 +118,14 @@ async def get_user(entry, storage):
elif user_oid: elif user_oid:
userdata = storage["users"]["by_oid"].get(user_oid) userdata = storage["users"]["by_oid"].get(user_oid)
if not userdata: if not userdata:
print('no userdata by oid, anonymous') print("no userdata by oid, anonymous")
userdata = anondict userdata = anondict
print(app) print(app)
# cleanup slug # cleanup slug
if userdata: if userdata:
slug = userdata.get("slug", "") slug = userdata.get("slug", "")
if slug: if slug:
slug = re.sub('[^0-9a-zA-Z]+', '-', slug) slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
userdata["slug"] = slug userdata["slug"] = slug
else: else:
userdata = anondict userdata = anondict
@ -160,7 +159,7 @@ async def migrate(entry, storage):
} }
# main topic patch # main topic patch
r['mainTopic'] = r['topics'][0] r["mainTopic"] = r["topics"][0]
# published author auto-confirm # published author auto-confirm
if entry.get("published"): if entry.get("published"):
@ -183,7 +182,7 @@ async def migrate(entry, storage):
shout_dict["oid"] = entry.get("_id", "") shout_dict["oid"] = entry.get("_id", "")
shout = await create_shout(shout_dict) shout = await create_shout(shout_dict)
except IntegrityError as e: except IntegrityError as e:
print('[migration] create_shout integrity error', e) print("[migration] create_shout integrity error", e)
shout = await resolve_create_shout(shout_dict) shout = await resolve_create_shout(shout_dict)
except Exception as e: except Exception as e:
raise Exception(e) raise Exception(e)
@ -202,7 +201,7 @@ async def migrate(entry, storage):
# shout views # shout views
await ViewedStorage.increment( await ViewedStorage.increment(
shout_dict["slug"], amount=entry.get("views", 1), viewer='old-discours' shout_dict["slug"], amount=entry.get("views", 1), viewer="old-discours"
) )
# del shout_dict['ratings'] # del shout_dict['ratings']
@ -240,7 +239,7 @@ async def add_topics_follower(entry, storage, user):
session.add(tf) session.add(tf)
session.commit() session.commit()
except IntegrityError: except IntegrityError:
print('[migration.shout] hidden by topic ' + tpc.slug) print("[migration.shout] hidden by topic " + tpc.slug)
# main topic # main topic
maintopic = storage["replacements"].get(topics_by_oid.get(category, {}).get("slug")) maintopic = storage["replacements"].get(topics_by_oid.get(category, {}).get("slug"))
if maintopic in ttt: if maintopic in ttt:
@ -261,7 +260,7 @@ async def process_user(userdata, storage, oid):
if not user: if not user:
try: try:
slug = userdata["slug"].lower().strip() slug = userdata["slug"].lower().strip()
slug = re.sub('[^0-9a-zA-Z]+', '-', slug) slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
userdata["slug"] = slug userdata["slug"] = slug
user = User.create(**userdata) user = User.create(**userdata)
session.add(user) session.add(user)
@ -289,9 +288,9 @@ async def resolve_create_shout(shout_dict):
s = session.query(Shout).filter(Shout.slug == shout_dict["slug"]).first() s = session.query(Shout).filter(Shout.slug == shout_dict["slug"]).first()
bump = False bump = False
if s: if s:
if s.createdAt != shout_dict['createdAt']: if s.createdAt != shout_dict["createdAt"]:
# create new with different slug # create new with different slug
shout_dict["slug"] += '-' + shout_dict["layout"] shout_dict["slug"] += "-" + shout_dict["layout"]
try: try:
await create_shout(shout_dict) await create_shout(shout_dict)
except IntegrityError as e: except IntegrityError as e:

22
migration/tables/remarks.py
View File

@ -5,24 +5,24 @@ from orm.reaction import Reaction, ReactionKind
def migrate(entry, storage): def migrate(entry, storage):
post_oid = entry['contentItem'] post_oid = entry["contentItem"]
print(post_oid) print(post_oid)
shout_dict = storage['shouts']['by_oid'].get(post_oid) shout_dict = storage["shouts"]["by_oid"].get(post_oid)
if shout_dict: if shout_dict:
print(shout_dict['body']) print(shout_dict["body"])
remark = { remark = {
"shout": shout_dict['id'], "shout": shout_dict["id"],
"body": extract_md(html2text(entry['body']), shout_dict), "body": extract_md(html2text(entry["body"]), shout_dict),
"kind": ReactionKind.REMARK, "kind": ReactionKind.REMARK,
} }
if entry.get('textBefore'): if entry.get("textBefore"):
remark['range'] = ( remark["range"] = (
str(shout_dict['body'].index(entry['textBefore'] or '')) str(shout_dict["body"].index(entry["textBefore"] or ""))
+ ':' + ":"
+ str( + str(
shout_dict['body'].index(entry['textAfter'] or '') shout_dict["body"].index(entry["textAfter"] or "")
+ len(entry['textAfter'] or '') + len(entry["textAfter"] or "")
) )
) )

15
migration/tables/users.py
View File

@ -1,11 +1,10 @@
import re from base.orm import local_session
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from dateutil.parser import parse from dateutil.parser import parse
from orm.user import AuthorFollower, User, UserRating
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
from base.orm import local_session import re
from orm.user import AuthorFollower, User, UserRating
def migrate(entry): def migrate(entry):
@ -33,12 +32,12 @@ def migrate(entry):
if entry.get("profile"): if entry.get("profile"):
# slug # slug
slug = entry["profile"].get("path").lower() slug = entry["profile"].get("path").lower()
slug = re.sub('[^0-9a-zA-Z]+', '-', slug).strip() slug = re.sub("[^0-9a-zA-Z]+", "-", slug).strip()
user_dict["slug"] = slug user_dict["slug"] = slug
bio = ( bio = (
(entry.get("profile", {"bio": ""}).get("bio") or "") (entry.get("profile", {"bio": ""}).get("bio") or "")
.replace('\(', '(') .replace(r"\(", "(")
.replace('\)', ')') .replace(r"\)", ")")
) )
bio_text = BeautifulSoup(bio, features="lxml").text bio_text = BeautifulSoup(bio, features="lxml").text
@ -144,7 +143,7 @@ def migrate_2stage(entry, id_map):
} }
user_rating = UserRating.create(**user_rating_dict) user_rating = UserRating.create(**user_rating_dict)
if user_rating_dict['value'] > 0: if user_rating_dict["value"] > 0:
af = AuthorFollower.create(author=user.id, follower=rater.id, auto=True) af = AuthorFollower.create(author=user.id, follower=rater.id, auto=True)
session.add(af) session.add(af)
session.add(user_rating) session.add(user_rating)

6
orm/collection.py
View File

@ -1,8 +1,6 @@
from datetime import datetime
from sqlalchemy import Column, DateTime, ForeignKey, String
from base.orm import Base from base.orm import Base
from datetime import datetime
from sqlalchemy import Column, DateTime, ForeignKey, String
class ShoutCollection(Base): class ShoutCollection(Base):

8
orm/community.py
View File

@ -1,8 +1,6 @@
from datetime import datetime
from sqlalchemy import Column, DateTime, ForeignKey, String
from base.orm import Base, local_session from base.orm import Base, local_session
from datetime import datetime
from sqlalchemy import Column, DateTime, ForeignKey, String
class CommunityFollower(Base): class CommunityFollower(Base):
@ -33,4 +31,4 @@ class Community(Base):
session.add(d) session.add(d)
session.commit() session.commit()
Community.default_community = d Community.default_community = d
print('[orm] default community id: %s' % d.id) print("[orm] default community id: %s" % d.id)

4
orm/notification.py
View File

@ -1,11 +1,9 @@
from base.orm import Base
from datetime import datetime from datetime import datetime
from enum import Enum as Enumeration from enum import Enum as Enumeration
from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, Integer from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, Integer
from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.dialects.postgresql import JSONB
from base.orm import Base
class NotificationType(Enumeration): class NotificationType(Enumeration):
NEW_COMMENT = 1 NEW_COMMENT = 1

27
orm/rbac.py
View File

@ -1,9 +1,8 @@
import warnings from base.orm import Base, local_session, REGISTRY
from sqlalchemy import Column, ForeignKey, String, TypeDecorator, UniqueConstraint from sqlalchemy import Column, ForeignKey, String, TypeDecorator, UniqueConstraint
from sqlalchemy.orm import relationship from sqlalchemy.orm import relationship
from base.orm import REGISTRY, Base, engine, local_session import warnings
# Role Based Access Control # # Role Based Access Control #
@ -165,14 +164,14 @@ class Permission(Base):
) )
if __name__ == "__main__": # if __name__ == "__main__":
Base.metadata.create_all(engine) # Base.metadata.create_all(engine)
ops = [ # ops = [
Permission(role=1, operation=1, resource=1), # Permission(role=1, operation=1, resource=1),
Permission(role=1, operation=2, resource=1), # Permission(role=1, operation=2, resource=1),
Permission(role=1, operation=3, resource=1), # Permission(role=1, operation=3, resource=1),
Permission(role=1, operation=4, resource=1), # Permission(role=1, operation=4, resource=1),
Permission(role=2, operation=4, resource=1), # Permission(role=2, operation=4, resource=1),
] # ]
global_session.add_all(ops) # global_session.add_all(ops)
global_session.commit() # global_session.commit()

4
orm/reaction.py
View File

@ -1,10 +1,8 @@
from base.orm import Base
from datetime import datetime from datetime import datetime
from enum import Enum as Enumeration from enum import Enum as Enumeration
from sqlalchemy import Column, DateTime, Enum, ForeignKey, String from sqlalchemy import Column, DateTime, Enum, ForeignKey, String
from base.orm import Base
class ReactionKind(Enumeration): class ReactionKind(Enumeration):
AGREE = 1 # +1 AGREE = 1 # +1

17
orm/shout.py
View File

@ -1,12 +1,10 @@
from datetime import datetime
from sqlalchemy import JSON, Boolean, Column, DateTime, ForeignKey, Integer, String
from sqlalchemy.orm import column_property, relationship
from base.orm import Base, local_session from base.orm import Base, local_session
from datetime import datetime
from orm.reaction import Reaction from orm.reaction import Reaction
from orm.topic import Topic from orm.topic import Topic
from orm.user import User from orm.user import User
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, JSON, String
from sqlalchemy.orm import column_property, relationship
class ShoutTopic(Base): class ShoutTopic(Base):
@ -70,7 +68,7 @@ class Shout(Base):
# TODO: these field should be used or modified # TODO: these field should be used or modified
community = Column(ForeignKey("community.id"), default=1) community = Column(ForeignKey("community.id"), default=1)
lang = Column(String, nullable=False, default='ru', comment="Language") lang = Column(String, nullable=False, default="ru", comment="Language")
mainTopic = Column(ForeignKey("topic.slug"), nullable=True) mainTopic = Column(ForeignKey("topic.slug"), nullable=True)
visibility = Column(String, nullable=True) # owner authors community public visibility = Column(String, nullable=True) # owner authors community public
versionOf = Column(ForeignKey("shout.id"), nullable=True) versionOf = Column(ForeignKey("shout.id"), nullable=True)
@ -81,7 +79,12 @@ class Shout(Base):
with local_session() as session: with local_session() as session:
s = session.query(Shout).first() s = session.query(Shout).first()
if not s: if not s:
entry = {"slug": "genesis-block", "body": "", "title": "Ничего", "lang": "ru"} entry = {
"slug": "genesis-block",
"body": "",
"title": "Ничего",
"lang": "ru",
}
s = Shout.create(**entry) s = Shout.create(**entry)
session.add(s) session.add(s)
session.commit() session.commit()

6
orm/topic.py
View File

@ -1,8 +1,6 @@
from datetime import datetime
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, String
from base.orm import Base from base.orm import Base
from datetime import datetime
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, String
class TopicFollower(Base): class TopicFollower(Base):

11
orm/user.py
View File

@ -1,11 +1,10 @@
from datetime import datetime
from sqlalchemy import JSON as JSONType
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String
from sqlalchemy.orm import relationship
from base.orm import Base, local_session from base.orm import Base, local_session
from datetime import datetime
from orm.rbac import Role from orm.rbac import Role
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer
from sqlalchemy import JSON as JSONType
from sqlalchemy import String
from sqlalchemy.orm import relationship
class UserRating(Base): class UserRating(Base):

1
requirements-dev.txt
View File

@ -3,3 +3,4 @@ brunette
flake8 flake8
mypy mypy
pre-commit pre-commit
black

3
requirements.txt
View File

@ -18,15 +18,12 @@ transliterate~=1.10.2
requests~=2.28.1 requests~=2.28.1
bcrypt>=4.0.0 bcrypt>=4.0.0
bson~=0.5.10 bson~=0.5.10
flake8
DateTime~=4.7 DateTime~=4.7
asyncio~=3.4.3 asyncio~=3.4.3
python-dateutil~=2.8.2 python-dateutil~=2.8.2
beautifulsoup4~=4.11.1 beautifulsoup4~=4.11.1
lxml lxml
sentry-sdk>=1.14.0 sentry-sdk>=1.14.0
# sse_starlette
graphql-ws
nltk~=3.8.1 nltk~=3.8.1
pymystem3~=0.2.0 pymystem3~=0.2.0
transformers~=4.28.1 transformers~=4.28.1

35
resolvers/__init__.py
View File

@ -1,35 +0,0 @@
from resolvers.auth import (
auth_send_link,
confirm_email,
get_current_user,
is_email_used,
login,
register_by_email,
sign_out,
)
from resolvers.create.editor import create_shout, delete_shout, update_shout
from resolvers.create.migrate import markdown_body
from resolvers.inbox.chats import create_chat, delete_chat, update_chat
from resolvers.inbox.load import load_chats, load_messages_by, load_recipients
from resolvers.inbox.messages import create_message, delete_message, mark_as_read, update_message
from resolvers.inbox.search import search_recipients
from resolvers.notifications import load_notifications
from resolvers.zine.following import follow, unfollow
from resolvers.zine.load import load_shout, load_shouts_by
from resolvers.zine.profile import get_authors_all, load_authors_by, rate_user, update_profile
from resolvers.zine.reactions import (
create_reaction,
delete_reaction,
load_reactions_by,
reactions_follow,
reactions_unfollow,
update_reaction,
)
from resolvers.zine.topics import (
get_topic,
topic_follow,
topic_unfollow,
topics_all,
topics_by_author,
topics_by_community,
)

35
resolvers/auth.py
View File

@ -1,13 +1,5 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import re
from datetime import datetime, timezone
from urllib.parse import quote_plus
from graphql.type import GraphQLResolveInfo
from starlette.responses import RedirectResponse
from transliterate import translit
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from auth.email import send_auth_email from auth.email import send_auth_email
@ -23,8 +15,15 @@ from base.exceptions import (
) )
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation, query from base.resolvers import mutation, query
from datetime import datetime, timezone
from graphql.type import GraphQLResolveInfo
from orm import Role, User from orm import Role, User
from settings import FRONTEND_URL, SESSION_TOKEN_HEADER from settings import FRONTEND_URL, SESSION_TOKEN_HEADER
from starlette.responses import RedirectResponse
from transliterate import translit
from urllib.parse import quote_plus
import re
@mutation.field("getSession") @mutation.field("getSession")
@ -45,7 +44,7 @@ async def get_current_user(_, info):
async def confirm_email(_, info, token): async def confirm_email(_, info, token):
"""confirm owning email address""" """confirm owning email address"""
try: try:
print('[resolvers.auth] confirm email by token') print("[resolvers.auth] confirm email by token")
payload = JWTCodec.decode(token) payload = JWTCodec.decode(token)
user_id = payload.user_id user_id = payload.user_id
await TokenStorage.get(f"{user_id}-{payload.username}-{token}") await TokenStorage.get(f"{user_id}-{payload.username}-{token}")
@ -68,9 +67,9 @@ async def confirm_email_handler(request):
token = request.path_params["token"] # one time token = request.path_params["token"] # one time
request.session["token"] = token request.session["token"] = token
res = await confirm_email(None, {}, token) res = await confirm_email(None, {}, token)
print('[resolvers.auth] confirm_email request: %r' % request) print("[resolvers.auth] confirm_email request: %r" % request)
if "error" in res: if "error" in res:
raise BaseHttpException(res['error']) raise BaseHttpException(res["error"])
else: else:
response = RedirectResponse(url=FRONTEND_URL) response = RedirectResponse(url=FRONTEND_URL)
response.set_cookie("token", res["token"]) # session token response.set_cookie("token", res["token"]) # session token
@ -87,22 +86,22 @@ def create_user(user_dict):
def generate_unique_slug(src): def generate_unique_slug(src):
print('[resolvers.auth] generating slug from: ' + src) print("[resolvers.auth] generating slug from: " + src)
slug = translit(src, "ru", reversed=True).replace(".", "-").lower() slug = translit(src, "ru", reversed=True).replace(".", "-").lower()
slug = re.sub('[^0-9a-zA-Z]+', '-', slug) slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
if slug != src: if slug != src:
print('[resolvers.auth] translited name: ' + slug) print("[resolvers.auth] translited name: " + slug)
c = 1 c = 1
with local_session() as session: with local_session() as session:
user = session.query(User).where(User.slug == slug).first() user = session.query(User).where(User.slug == slug).first()
while user: while user:
user = session.query(User).where(User.slug == slug).first() user = session.query(User).where(User.slug == slug).first()
slug = slug + '-' + str(c) slug = slug + "-" + str(c)
c += 1 c += 1
if not user: if not user:
unique_slug = slug unique_slug = slug
print('[resolvers.auth] ' + unique_slug) print("[resolvers.auth] " + unique_slug)
return quote_plus(unique_slug.replace('\'', '')).replace('+', '-') return quote_plus(unique_slug.replace("'", "")).replace("+", "-")
@mutation.field("registerUser") @mutation.field("registerUser")
@ -117,7 +116,7 @@ async def register_by_email(_, _info, email: str, password: str = "", name: str
slug = generate_unique_slug(name) slug = generate_unique_slug(name)
user = session.query(User).where(User.slug == slug).first() user = session.query(User).where(User.slug == slug).first()
if user: if user:
slug = generate_unique_slug(email.split('@')[0]) slug = generate_unique_slug(email.split("@")[0])
user_dict = { user_dict = {
"email": email, "email": email,
"username": email, # will be used to store phone number or some messenger network id "username": email, # will be used to store phone number or some messenger network id

27
resolvers/create/editor.py
View File

@ -1,15 +1,13 @@
from datetime import datetime, timezone
from sqlalchemy import and_
from sqlalchemy.orm import joinedload
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation from base.resolvers import mutation
from datetime import datetime, timezone
from orm.shout import Shout, ShoutAuthor, ShoutTopic from orm.shout import Shout, ShoutAuthor, ShoutTopic
from orm.topic import Topic from orm.topic import Topic
from resolvers.zine.reactions import reactions_follow, reactions_unfollow from resolvers.zine.reactions import reactions_follow, reactions_unfollow
from sqlalchemy import and_
from sqlalchemy.orm import joinedload
@mutation.field("createShout") @mutation.field("createShout")
@ -18,15 +16,15 @@ async def create_shout(_, info, inp):
auth: AuthCredentials = info.context["request"].auth auth: AuthCredentials = info.context["request"].auth
with local_session() as session: with local_session() as session:
topics = session.query(Topic).filter(Topic.slug.in_(inp.get('topics', []))).all() topics = session.query(Topic).filter(Topic.slug.in_(inp.get("topics", []))).all()
new_shout = Shout.create( new_shout = Shout.create(
**{ **{
"title": inp.get("title"), "title": inp.get("title"),
"subtitle": inp.get('subtitle'), "subtitle": inp.get("subtitle"),
"lead": inp.get('lead'), "lead": inp.get("lead"),
"description": inp.get('description'), "description": inp.get("description"),
"body": inp.get("body", ''), "body": inp.get("body", ""),
"layout": inp.get("layout"), "layout": inp.get("layout"),
"authors": inp.get("authors", []), "authors": inp.get("authors", []),
"slug": inp.get("slug"), "slug": inp.get("slug"),
@ -128,7 +126,10 @@ async def update_shout(_, info, shout_id, shout_input=None, publish=False):
] ]
shout_topics_to_remove = session.query(ShoutTopic).filter( shout_topics_to_remove = session.query(ShoutTopic).filter(
and_(ShoutTopic.shout == shout.id, ShoutTopic.topic.in_(topic_to_unlink_ids)) and_(
ShoutTopic.shout == shout.id,
ShoutTopic.topic.in_(topic_to_unlink_ids),
)
) )
for shout_topic_to_remove in shout_topics_to_remove: for shout_topic_to_remove in shout_topics_to_remove:
@ -136,13 +137,13 @@ async def update_shout(_, info, shout_id, shout_input=None, publish=False):
shout_input["mainTopic"] = shout_input["mainTopic"]["slug"] shout_input["mainTopic"] = shout_input["mainTopic"]["slug"]
if shout_input["mainTopic"] == '': if shout_input["mainTopic"] == "":
del shout_input["mainTopic"] del shout_input["mainTopic"]
shout.update(shout_input) shout.update(shout_input)
updated = True updated = True
if publish and shout.visibility == 'owner': if publish and shout.visibility == "owner":
shout.visibility = "community" shout.visibility = "community"
shout.publishedAt = datetime.now(tz=timezone.utc) shout.publishedAt = datetime.now(tz=timezone.utc)
updated = True updated = True

20
resolvers/create/migrate.py
View File

@ -1,10 +1,10 @@
from base.resolvers import query # from base.resolvers import query
from migration.extract import extract_md # from migration.extract import extract_md
from resolvers.auth import login_required # from resolvers.auth import login_required
#
#
@login_required # @login_required
@query.field("markdownBody") # @query.field("markdownBody")
def markdown_body(_, info, body: str): # def markdown_body(_, info, body: str):
body = extract_md(body) # body = extract_md(body)
return body # return body

16
resolvers/inbox/chats.py
View File

@ -1,13 +1,13 @@
import json
import uuid
from datetime import datetime, timezone
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from base.redis import redis from base.redis import redis
from base.resolvers import mutation from base.resolvers import mutation
from datetime import datetime, timezone
from validations.inbox import Chat from validations.inbox import Chat
import json
import uuid
@mutation.field("updateChat") @mutation.field("updateChat")
@login_required @login_required
@ -49,7 +49,7 @@ async def update_chat(_, info, chat_new: Chat):
async def create_chat(_, info, title="", members=[]): async def create_chat(_, info, title="", members=[]):
auth: AuthCredentials = info.context["request"].auth auth: AuthCredentials = info.context["request"].auth
chat = {} chat = {}
print('create_chat members: %r' % members) print("create_chat members: %r" % members)
if auth.user_id not in members: if auth.user_id not in members:
members.append(int(auth.user_id)) members.append(int(auth.user_id))
@ -71,8 +71,8 @@ async def create_chat(_, info, title="", members=[]):
chat = await redis.execute("GET", f"chats/{c.decode('utf-8')}") chat = await redis.execute("GET", f"chats/{c.decode('utf-8')}")
if chat: if chat:
chat = json.loads(chat) chat = json.loads(chat)
if chat['title'] == "": if chat["title"] == "":
print('[inbox] createChat found old chat') print("[inbox] createChat found old chat")
print(chat) print(chat)
break break
if chat: if chat:
@ -105,7 +105,7 @@ async def delete_chat(_, info, chat_id: str):
chat = await redis.execute("GET", f"/chats/{chat_id}") chat = await redis.execute("GET", f"/chats/{chat_id}")
if chat: if chat:
chat = dict(json.loads(chat)) chat = dict(json.loads(chat))
if auth.user_id in chat['admins']: if auth.user_id in chat["admins"]:
await redis.execute("DEL", f"chats/{chat_id}") await redis.execute("DEL", f"chats/{chat_id}")
await redis.execute("SREM", "chats_by_user/" + str(auth.user_id), chat_id) await redis.execute("SREM", "chats_by_user/" + str(auth.user_id), chat_id)
await redis.execute("COMMIT") await redis.execute("COMMIT")

32
resolvers/inbox/load.py
View File

@ -1,5 +1,4 @@
import json from .unread import get_unread_counter
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from base.orm import local_session from base.orm import local_session
@ -8,13 +7,13 @@ from base.resolvers import query
from orm.user import User from orm.user import User
from resolvers.zine.profile import followed_authors from resolvers.zine.profile import followed_authors
from .unread import get_unread_counter import json
# from datetime import datetime, timedelta, timezone # from datetime import datetime, timedelta, timezone
async def load_messages(chat_id: str, limit: int = 5, offset: int = 0, ids=[]): async def load_messages(chat_id: str, limit: int = 5, offset: int = 0, ids=[]):
'''load :limit messages for :chat_id with :offset''' """load :limit messages for :chat_id with :offset"""
messages = [] messages = []
message_ids = [] message_ids = []
if ids: if ids:
@ -29,10 +28,10 @@ async def load_messages(chat_id: str, limit: int = 5, offset: int = 0, ids=[]):
if message_ids: if message_ids:
message_keys = [f"chats/{chat_id}/messages/{mid}" for mid in message_ids] message_keys = [f"chats/{chat_id}/messages/{mid}" for mid in message_ids]
messages = await redis.mget(*message_keys) messages = await redis.mget(*message_keys)
messages = [json.loads(msg.decode('utf-8')) for msg in messages] messages = [json.loads(msg.decode("utf-8")) for msg in messages]
replies = [] replies = []
for m in messages: for m in messages:
rt = m.get('replyTo') rt = m.get("replyTo")
if rt: if rt:
rt = int(rt) rt = int(rt)
if rt not in message_ids: if rt not in message_ids:
@ -52,7 +51,7 @@ async def load_chats(_, info, limit: int = 50, offset: int = 0):
if cids: if cids:
cids = list(cids)[offset : offset + limit] cids = list(cids)[offset : offset + limit]
if not cids: if not cids:
print('[inbox.load] no chats were found') print("[inbox.load] no chats were found")
cids = [] cids = []
onliners = await redis.execute("SMEMBERS", "users-online") onliners = await redis.execute("SMEMBERS", "users-online")
if not onliners: if not onliners:
@ -63,14 +62,14 @@ async def load_chats(_, info, limit: int = 50, offset: int = 0):
c = await redis.execute("GET", "chats/" + cid) c = await redis.execute("GET", "chats/" + cid)
if c: if c:
c = dict(json.loads(c)) c = dict(json.loads(c))
c['messages'] = await load_messages(cid, 5, 0) c["messages"] = await load_messages(cid, 5, 0)
c['unread'] = await get_unread_counter(cid, auth.user_id) c["unread"] = await get_unread_counter(cid, auth.user_id)
with local_session() as session: with local_session() as session:
c['members'] = [] c["members"] = []
for uid in c["users"]: for uid in c["users"]:
a = session.query(User).where(User.id == uid).first() a = session.query(User).where(User.id == uid).first()
if a: if a:
c['members'].append( c["members"].append(
{ {
"id": a.id, "id": a.id,
"slug": a.slug, "slug": a.slug,
@ -87,16 +86,16 @@ async def load_chats(_, info, limit: int = 50, offset: int = 0):
@query.field("loadMessagesBy") @query.field("loadMessagesBy")
@login_required @login_required
async def load_messages_by(_, info, by, limit: int = 10, offset: int = 0): async def load_messages_by(_, info, by, limit: int = 10, offset: int = 0):
'''load :limit messages of :chat_id with :offset''' """load :limit messages of :chat_id with :offset"""
auth: AuthCredentials = info.context["request"].auth auth: AuthCredentials = info.context["request"].auth
userchats = await redis.execute("SMEMBERS", "chats_by_user/" + str(auth.user_id)) userchats = await redis.execute("SMEMBERS", "chats_by_user/" + str(auth.user_id))
userchats = [c.decode('utf-8') for c in userchats] userchats = [c.decode("utf-8") for c in userchats]
# print('[inbox] userchats: %r' % userchats) # print('[inbox] userchats: %r' % userchats)
if userchats: if userchats:
# print('[inbox] loading messages by...') # print('[inbox] loading messages by...')
messages = [] messages = []
by_chat = by.get('chat') by_chat = by.get("chat")
if by_chat in userchats: if by_chat in userchats:
chat = await redis.execute("GET", f"chats/{by_chat}") chat = await redis.execute("GET", f"chats/{by_chat}")
# print(chat) # print(chat)
@ -104,7 +103,10 @@ async def load_messages_by(_, info, by, limit: int = 10, offset: int = 0):
return {"messages": [], "error": "chat not exist"} return {"messages": [], "error": "chat not exist"}
# everyone's messages in filtered chat # everyone's messages in filtered chat
messages = await load_messages(by_chat, limit, offset) messages = await load_messages(by_chat, limit, offset)
return {"messages": sorted(list(messages), key=lambda m: m['createdAt']), "error": None} return {
"messages": sorted(list(messages), key=lambda m: m["createdAt"]),
"error": None,
}
else: else:
return {"error": "Cannot access messages of this chat"} return {"error": "Cannot access messages of this chat"}

29
resolvers/inbox/messages.py
View File

@ -1,16 +1,11 @@
import asyncio
import json
from datetime import datetime, timezone
from typing import Any
from graphql.type import GraphQLResolveInfo
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from base.redis import redis from base.redis import redis
from base.resolvers import mutation from base.resolvers import mutation
from services.following import Following, FollowingManager, FollowingResult from datetime import datetime, timezone
from validations.inbox import Message from services.following import FollowingManager, FollowingResult
import json
@mutation.field("createMessage") @mutation.field("createMessage")
@ -27,15 +22,15 @@ async def create_message(_, info, chat: str, body: str, replyTo=None):
message_id = await redis.execute("GET", f"chats/{chat['id']}/next_message_id") message_id = await redis.execute("GET", f"chats/{chat['id']}/next_message_id")
message_id = int(message_id) message_id = int(message_id)
new_message = { new_message = {
"chatId": chat['id'], "chatId": chat["id"],
"id": message_id, "id": message_id,
"author": auth.user_id, "author": auth.user_id,
"body": body, "body": body,
"createdAt": int(datetime.now(tz=timezone.utc).timestamp()), "createdAt": int(datetime.now(tz=timezone.utc).timestamp()),
} }
if replyTo: if replyTo:
new_message['replyTo'] = replyTo new_message["replyTo"] = replyTo
chat['updatedAt'] = new_message['createdAt'] chat["updatedAt"] = new_message["createdAt"]
await redis.execute("SET", f"chats/{chat['id']}", json.dumps(chat)) await redis.execute("SET", f"chats/{chat['id']}", json.dumps(chat))
print(f"[inbox] creating message {new_message}") print(f"[inbox] creating message {new_message}")
await redis.execute( await redis.execute(
@ -48,8 +43,8 @@ async def create_message(_, info, chat: str, body: str, replyTo=None):
for user_slug in users: for user_slug in users:
await redis.execute("LPUSH", f"chats/{chat['id']}/unread/{user_slug}", str(message_id)) await redis.execute("LPUSH", f"chats/{chat['id']}/unread/{user_slug}", str(message_id))
result = FollowingResult("NEW", 'chat', new_message) result = FollowingResult("NEW", "chat", new_message)
await FollowingManager.push('chat', result) await FollowingManager.push("chat", result)
return {"message": new_message, "error": None} return {"message": new_message, "error": None}
@ -76,8 +71,8 @@ async def update_message(_, info, chat_id: str, message_id: int, body: str):
await redis.execute("SET", f"chats/{chat_id}/messages/{message_id}", json.dumps(message)) await redis.execute("SET", f"chats/{chat_id}/messages/{message_id}", json.dumps(message))
result = FollowingResult("UPDATED", 'chat', message) result = FollowingResult("UPDATED", "chat", message)
await FollowingManager.push('chat', result) await FollowingManager.push("chat", result)
return {"message": message, "error": None} return {"message": message, "error": None}
@ -106,7 +101,7 @@ async def delete_message(_, info, chat_id: str, message_id: int):
for user_id in users: for user_id in users:
await redis.execute("LREM", f"chats/{chat_id}/unread/{user_id}", 0, str(message_id)) await redis.execute("LREM", f"chats/{chat_id}/unread/{user_id}", 0, str(message_id))
result = FollowingResult("DELETED", 'chat', message) result = FollowingResult("DELETED", "chat", message)
await FollowingManager.push(result) await FollowingManager.push(result)
return {} return {}

14
resolvers/inbox/search.py
View File

@ -1,14 +1,14 @@
import json
from datetime import datetime, timedelta, timezone
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from base.orm import local_session from base.orm import local_session
from base.redis import redis from base.redis import redis
from base.resolvers import query from base.resolvers import query
from datetime import datetime, timedelta, timezone
from orm.user import AuthorFollower, User from orm.user import AuthorFollower, User
from resolvers.inbox.load import load_messages from resolvers.inbox.load import load_messages
import json
@query.field("searchRecipients") @query.field("searchRecipients")
@login_required @login_required
@ -59,22 +59,22 @@ async def search_user_chats(by, messages, user_id: int, limit, offset):
cids.union(set(await redis.execute("SMEMBERS", "chats_by_user/" + str(user_id)))) cids.union(set(await redis.execute("SMEMBERS", "chats_by_user/" + str(user_id))))
messages = [] messages = []
by_author = by.get('author') by_author = by.get("author")
if by_author: if by_author:
# all author's messages # all author's messages
cids.union(set(await redis.execute("SMEMBERS", f"chats_by_user/{by_author}"))) cids.union(set(await redis.execute("SMEMBERS", f"chats_by_user/{by_author}")))
# author's messages in filtered chat # author's messages in filtered chat
messages.union(set(filter(lambda m: m["author"] == by_author, list(messages)))) messages.union(set(filter(lambda m: m["author"] == by_author, list(messages))))
for c in cids: for c in cids:
c = c.decode('utf-8') c = c.decode("utf-8")
messages = await load_messages(c, limit, offset) messages = await load_messages(c, limit, offset)
body_like = by.get('body') body_like = by.get("body")
if body_like: if body_like:
# search in all messages in all user's chats # search in all messages in all user's chats
for c in cids: for c in cids:
# FIXME: use redis scan here # FIXME: use redis scan here
c = c.decode('utf-8') c = c.decode("utf-8")
mmm = await load_messages(c, limit, offset) mmm = await load_messages(c, limit, offset)
for m in mmm: for m in mmm:
if body_like in m["body"]: if body_like in m["body"]:

11
resolvers/notifications.py
View File

@ -1,10 +1,9 @@
from sqlalchemy import and_, desc, select, update
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation, query from base.resolvers import mutation, query
from orm import Notification from orm import Notification
from sqlalchemy import and_, desc, select, update
@query.field("loadNotifications") @query.field("loadNotifications")
@ -16,8 +15,8 @@ async def load_notifications(_, info, params=None):
auth: AuthCredentials = info.context["request"].auth auth: AuthCredentials = info.context["request"].auth
user_id = auth.user_id user_id = auth.user_id
limit = params.get('limit', 50) limit = params.get("limit", 50)
offset = params.get('offset', 0) offset = params.get("offset", 0)
q = ( q = (
select(Notification) select(Notification)
@ -33,7 +32,7 @@ async def load_notifications(_, info, params=None):
total_unread_count = ( total_unread_count = (
session.query(Notification) session.query(Notification)
.where(and_(Notification.user == user_id, Notification.seen == False)) .where(and_(Notification.user == user_id, Notification.seen == False)) # noqa: E712
.count() .count()
) )
@ -74,7 +73,7 @@ async def mark_all_notifications_as_read(_, info):
statement = ( statement = (
update(Notification) update(Notification)
.where(and_(Notification.user == user_id, Notification.seen == False)) .where(and_(Notification.user == user_id, Notification.seen == False)) # noqa: E712
.values(seen=True) .values(seen=True)
) )

32
resolvers/upload.py
View File

@ -1,33 +1,33 @@
from botocore.exceptions import BotoCoreError, ClientError
from starlette.responses import JSONResponse
import boto3
import os import os
import shutil import shutil
import tempfile import tempfile
import uuid import uuid
import boto3 STORJ_ACCESS_KEY = os.environ.get("STORJ_ACCESS_KEY")
from botocore.exceptions import BotoCoreError, ClientError STORJ_SECRET_KEY = os.environ.get("STORJ_SECRET_KEY")
from starlette.responses import JSONResponse STORJ_END_POINT = os.environ.get("STORJ_END_POINT")
STORJ_BUCKET_NAME = os.environ.get("STORJ_BUCKET_NAME")
STORJ_ACCESS_KEY = os.environ.get('STORJ_ACCESS_KEY') CDN_DOMAIN = os.environ.get("CDN_DOMAIN")
STORJ_SECRET_KEY = os.environ.get('STORJ_SECRET_KEY')
STORJ_END_POINT = os.environ.get('STORJ_END_POINT')
STORJ_BUCKET_NAME = os.environ.get('STORJ_BUCKET_NAME')
CDN_DOMAIN = os.environ.get('CDN_DOMAIN')
async def upload_handler(request): async def upload_handler(request):
form = await request.form() form = await request.form()
file = form.get('file') file = form.get("file")
if file is None: if file is None:
return JSONResponse({'error': 'No file uploaded'}, status_code=400) return JSONResponse({"error": "No file uploaded"}, status_code=400)
file_name, file_extension = os.path.splitext(file.filename) file_name, file_extension = os.path.splitext(file.filename)
key = 'files/' + str(uuid.uuid4()) + file_extension key = "files/" + str(uuid.uuid4()) + file_extension
# Create an S3 client with Storj configuration # Create an S3 client with Storj configuration
s3 = boto3.client( s3 = boto3.client(
's3', "s3",
aws_access_key_id=STORJ_ACCESS_KEY, aws_access_key_id=STORJ_ACCESS_KEY,
aws_secret_access_key=STORJ_SECRET_KEY, aws_secret_access_key=STORJ_SECRET_KEY,
endpoint_url=STORJ_END_POINT, endpoint_url=STORJ_END_POINT,
@ -45,10 +45,10 @@ async def upload_handler(request):
ExtraArgs={"ContentType": file.content_type}, ExtraArgs={"ContentType": file.content_type},
) )
url = 'https://' + CDN_DOMAIN + '/' + key url = "https://" + CDN_DOMAIN + "/" + key
return JSONResponse({'url': url, 'originalFilename': file.filename}) return JSONResponse({"url": url, "originalFilename": file.filename})
except (BotoCoreError, ClientError) as e: except (BotoCoreError, ClientError) as e:
print(e) print(e)
return JSONResponse({'error': 'Failed to upload file'}, status_code=500) return JSONResponse({"error": "Failed to upload file"}, status_code=500)

42
resolvers/zine/following.py
View File

@ -1,20 +1,12 @@
import asyncio
from graphql.type import GraphQLResolveInfo
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from base.orm import local_session
from base.resolvers import mutation from base.resolvers import mutation
from orm.shout import ShoutReactionsFollower
from orm.topic import TopicFollower
# from resolvers.community import community_follow, community_unfollow # from resolvers.community import community_follow, community_unfollow
from orm.user import AuthorFollower
from resolvers.zine.profile import author_follow, author_unfollow from resolvers.zine.profile import author_follow, author_unfollow
from resolvers.zine.reactions import reactions_follow, reactions_unfollow from resolvers.zine.reactions import reactions_follow, reactions_unfollow
from resolvers.zine.topics import topic_follow, topic_unfollow from resolvers.zine.topics import topic_follow, topic_unfollow
from services.following import Following, FollowingManager, FollowingResult from services.following import FollowingManager, FollowingResult
@mutation.field("follow") @mutation.field("follow")
@ -25,20 +17,20 @@ async def follow(_, info, what, slug):
try: try:
if what == "AUTHOR": if what == "AUTHOR":
if author_follow(auth.user_id, slug): if author_follow(auth.user_id, slug):
result = FollowingResult("NEW", 'author', slug) result = FollowingResult("NEW", "author", slug)
await FollowingManager.push('author', result) await FollowingManager.push("author", result)
elif what == "TOPIC": elif what == "TOPIC":
if topic_follow(auth.user_id, slug): if topic_follow(auth.user_id, slug):
result = FollowingResult("NEW", 'topic', slug) result = FollowingResult("NEW", "topic", slug)
await FollowingManager.push('topic', result) await FollowingManager.push("topic", result)
elif what == "COMMUNITY": elif what == "COMMUNITY":
if False: # TODO: use community_follow(auth.user_id, slug): if False: # TODO: use community_follow(auth.user_id, slug):
result = FollowingResult("NEW", 'community', slug) result = FollowingResult("NEW", "community", slug)
await FollowingManager.push('community', result) await FollowingManager.push("community", result)
elif what == "REACTIONS": elif what == "REACTIONS":
if reactions_follow(auth.user_id, slug): if reactions_follow(auth.user_id, slug):
result = FollowingResult("NEW", 'shout', slug) result = FollowingResult("NEW", "shout", slug)
await FollowingManager.push('shout', result) await FollowingManager.push("shout", result)
except Exception as e: except Exception as e:
print(Exception(e)) print(Exception(e))
return {"error": str(e)} return {"error": str(e)}
@ -54,20 +46,20 @@ async def unfollow(_, info, what, slug):
try: try:
if what == "AUTHOR": if what == "AUTHOR":
if author_unfollow(auth.user_id, slug): if author_unfollow(auth.user_id, slug):
result = FollowingResult("DELETED", 'author', slug) result = FollowingResult("DELETED", "author", slug)
await FollowingManager.push('author', result) await FollowingManager.push("author", result)
elif what == "TOPIC": elif what == "TOPIC":
if topic_unfollow(auth.user_id, slug): if topic_unfollow(auth.user_id, slug):
result = FollowingResult("DELETED", 'topic', slug) result = FollowingResult("DELETED", "topic", slug)
await FollowingManager.push('topic', result) await FollowingManager.push("topic", result)
elif what == "COMMUNITY": elif what == "COMMUNITY":
if False: # TODO: use community_unfollow(auth.user_id, slug): if False: # TODO: use community_unfollow(auth.user_id, slug):
result = FollowingResult("DELETED", 'community', slug) result = FollowingResult("DELETED", "community", slug)
await FollowingManager.push('community', result) await FollowingManager.push("community", result)
elif what == "REACTIONS": elif what == "REACTIONS":
if reactions_unfollow(auth.user_id, slug): if reactions_unfollow(auth.user_id, slug):
result = FollowingResult("DELETED", 'shout', slug) result = FollowingResult("DELETED", "shout", slug)
await FollowingManager.push('shout', result) await FollowingManager.push("shout", result)
except Exception as e: except Exception as e:
return {"error": str(e)} return {"error": str(e)}

60
resolvers/zine/load.py
View File

@ -1,26 +1,24 @@
from datetime import datetime, timedelta, timezone
from sqlalchemy.orm import aliased, joinedload
from sqlalchemy.sql.expression import and_, asc, case, desc, func, nulls_last, select, text
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from base.exceptions import ObjectNotExist, OperationNotAllowed from base.exceptions import ObjectNotExist
from base.orm import local_session from base.orm import local_session
from base.resolvers import query from base.resolvers import query
from datetime import datetime, timedelta, timezone
from orm import TopicFollower from orm import TopicFollower
from orm.reaction import Reaction, ReactionKind from orm.reaction import Reaction, ReactionKind
from orm.shout import Shout, ShoutAuthor, ShoutTopic from orm.shout import Shout, ShoutAuthor, ShoutTopic
from orm.user import AuthorFollower from orm.user import AuthorFollower
from sqlalchemy.orm import aliased, joinedload
from sqlalchemy.sql.expression import and_, asc, case, desc, func, nulls_last, select
def add_stat_columns(q): def add_stat_columns(q):
aliased_reaction = aliased(Reaction) aliased_reaction = aliased(Reaction)
q = q.outerjoin(aliased_reaction).add_columns( q = q.outerjoin(aliased_reaction).add_columns(
func.sum(aliased_reaction.id).label('reacted_stat'), func.sum(aliased_reaction.id).label("reacted_stat"),
func.sum(case((aliased_reaction.kind == ReactionKind.COMMENT, 1), else_=0)).label( func.sum(case((aliased_reaction.kind == ReactionKind.COMMENT, 1), else_=0)).label(
'commented_stat' "commented_stat"
), ),
func.sum( func.sum(
case( case(
@ -36,13 +34,13 @@ def add_stat_columns(q):
(aliased_reaction.kind == ReactionKind.DISLIKE, -1), (aliased_reaction.kind == ReactionKind.DISLIKE, -1),
else_=0, else_=0,
) )
).label('rating_stat'), ).label("rating_stat"),
func.max( func.max(
case( case(
(aliased_reaction.kind != ReactionKind.COMMENT, None), (aliased_reaction.kind != ReactionKind.COMMENT, None),
else_=aliased_reaction.createdAt, else_=aliased_reaction.createdAt,
) )
).label('last_comment'), ).label("last_comment"),
) )
return q return q
@ -60,7 +58,7 @@ def apply_filters(q, filters, user_id=None):
if filters.get("layout"): if filters.get("layout"):
q = q.filter(Shout.layout == filters.get("layout")) q = q.filter(Shout.layout == filters.get("layout"))
if filters.get('excludeLayout'): if filters.get("excludeLayout"):
q = q.filter(Shout.layout != filters.get("excludeLayout")) q = q.filter(Shout.layout != filters.get("excludeLayout"))
if filters.get("author"): if filters.get("author"):
q = q.filter(Shout.authors.any(slug=filters.get("author"))) q = q.filter(Shout.authors.any(slug=filters.get("author")))
@ -95,9 +93,13 @@ async def load_shout(_, info, slug=None, shout_id=None):
q = q.filter(Shout.deletedAt.is_(None)).group_by(Shout.id) q = q.filter(Shout.deletedAt.is_(None)).group_by(Shout.id)
try: try:
[shout, reacted_stat, commented_stat, rating_stat, last_comment] = session.execute( [
q shout,
).first() reacted_stat,
commented_stat,
rating_stat,
last_comment,
] = session.execute(q).first()
shout.stat = { shout.stat = {
"viewed": shout.views, "viewed": shout.views,
@ -154,7 +156,7 @@ async def load_shouts_by(_, info, options):
order_by = options.get("order_by", Shout.publishedAt) order_by = options.get("order_by", Shout.publishedAt)
query_order_by = desc(order_by) if options.get('order_by_desc', True) else asc(order_by) query_order_by = desc(order_by) if options.get("order_by_desc", True) else asc(order_by)
offset = options.get("offset", 0) offset = options.get("offset", 0)
limit = options.get("limit", 10) limit = options.get("limit", 10)
@ -164,9 +166,13 @@ async def load_shouts_by(_, info, options):
with local_session() as session: with local_session() as session:
shouts_map = {} shouts_map = {}
for [shout, reacted_stat, commented_stat, rating_stat, last_comment] in session.execute( for [
q shout,
).unique(): reacted_stat,
commented_stat,
rating_stat,
last_comment,
] in session.execute(q).unique():
shouts.append(shout) shouts.append(shout)
shout.stat = { shout.stat = {
"viewed": shout.views, "viewed": shout.views,
@ -225,7 +231,11 @@ async def get_my_feed(_, info, options):
joinedload(Shout.topics), joinedload(Shout.topics),
) )
.where( .where(
and_(Shout.publishedAt.is_not(None), Shout.deletedAt.is_(None), Shout.id.in_(subquery)) and_(
Shout.publishedAt.is_not(None),
Shout.deletedAt.is_(None),
Shout.id.in_(subquery),
)
) )
) )
@ -234,7 +244,7 @@ async def get_my_feed(_, info, options):
order_by = options.get("order_by", Shout.publishedAt) order_by = options.get("order_by", Shout.publishedAt)
query_order_by = desc(order_by) if options.get('order_by_desc', True) else asc(order_by) query_order_by = desc(order_by) if options.get("order_by_desc", True) else asc(order_by)
offset = options.get("offset", 0) offset = options.get("offset", 0)
limit = options.get("limit", 10) limit = options.get("limit", 10)
@ -243,9 +253,13 @@ async def get_my_feed(_, info, options):
shouts = [] shouts = []
with local_session() as session: with local_session() as session:
shouts_map = {} shouts_map = {}
for [shout, reacted_stat, commented_stat, rating_stat, last_comment] in session.execute( for [
q shout,
).unique(): reacted_stat,
commented_stat,
rating_stat,
last_comment,
] in session.execute(q).unique():
shouts.append(shout) shouts.append(shout)
shout.stat = { shout.stat = {
"viewed": shout.views, "viewed": shout.views,

35
resolvers/zine/profile.py
View File

@ -1,18 +1,16 @@
from datetime import datetime, timedelta, timezone
from typing import List
from sqlalchemy import and_, distinct, func, literal, select
from sqlalchemy.orm import aliased, joinedload
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation, query from base.resolvers import mutation, query
from datetime import datetime, timedelta, timezone
from orm.reaction import Reaction, ReactionKind from orm.reaction import Reaction, ReactionKind
from orm.shout import ShoutAuthor, ShoutTopic from orm.shout import ShoutAuthor, ShoutTopic
from orm.topic import Topic, TopicFollower from orm.topic import Topic, TopicFollower
from orm.user import AuthorFollower, Role, User, UserRating, UserRole from orm.user import AuthorFollower, Role, User, UserRating, UserRole
from resolvers.zine.topics import followed_by_user from resolvers.zine.topics import followed_by_user
from sqlalchemy import and_, distinct, func, literal, select
from sqlalchemy.orm import aliased, joinedload
from typing import List
def add_author_stat_columns(q): def add_author_stat_columns(q):
@ -22,24 +20,24 @@ def add_author_stat_columns(q):
# user_rating_aliased = aliased(UserRating) # user_rating_aliased = aliased(UserRating)
q = q.outerjoin(shout_author_aliased).add_columns( q = q.outerjoin(shout_author_aliased).add_columns(
func.count(distinct(shout_author_aliased.shout)).label('shouts_stat') func.count(distinct(shout_author_aliased.shout)).label("shouts_stat")
) )
q = q.outerjoin(author_followers, author_followers.author == User.id).add_columns( q = q.outerjoin(author_followers, author_followers.author == User.id).add_columns(
func.count(distinct(author_followers.follower)).label('followers_stat') func.count(distinct(author_followers.follower)).label("followers_stat")
) )
q = q.outerjoin(author_following, author_following.follower == User.id).add_columns( q = q.outerjoin(author_following, author_following.follower == User.id).add_columns(
func.count(distinct(author_following.author)).label('followings_stat') func.count(distinct(author_following.author)).label("followings_stat")
) )
q = q.add_columns(literal(0).label('rating_stat')) q = q.add_columns(literal(0).label("rating_stat"))
# FIXME # FIXME
# q = q.outerjoin(user_rating_aliased, user_rating_aliased.user == User.id).add_columns( # q = q.outerjoin(user_rating_aliased, user_rating_aliased.user == User.id).add_columns(
# # TODO: check # # TODO: check
# func.sum(user_rating_aliased.value).label('rating_stat') # func.sum(user_rating_aliased.value).label('rating_stat')
# ) # )
q = q.add_columns(literal(0).label('commented_stat')) q = q.add_columns(literal(0).label("commented_stat"))
# q = q.outerjoin(Reaction, and_(Reaction.createdBy == User.id, Reaction.body.is_not(None))).add_columns( # q = q.outerjoin(Reaction, and_(Reaction.createdBy == User.id, Reaction.body.is_not(None))).add_columns(
# func.count(distinct(Reaction.id)).label('commented_stat') # func.count(distinct(Reaction.id)).label('commented_stat')
# ) # )
@ -50,7 +48,13 @@ def add_author_stat_columns(q):
def add_stat(author, stat_columns): def add_stat(author, stat_columns):
[shouts_stat, followers_stat, followings_stat, rating_stat, commented_stat] = stat_columns [
shouts_stat,
followers_stat,
followings_stat,
rating_stat,
commented_stat,
] = stat_columns
author.stat = { author.stat = {
"shouts": shouts_stat, "shouts": shouts_stat,
"followers": followers_stat, "followers": followers_stat,
@ -227,7 +231,12 @@ async def get_author(_, _info, slug):
with local_session() as session: with local_session() as session:
comments_count = ( comments_count = (
session.query(Reaction) session.query(Reaction)
.where(and_(Reaction.createdBy == author.id, Reaction.kind == ReactionKind.COMMENT)) .where(
and_(
Reaction.createdBy == author.id,
Reaction.kind == ReactionKind.COMMENT,
)
)
.count() .count()
) )
author.stat["commented"] = comments_count author.stat["commented"] = comments_count

59
resolvers/zine/reactions.py
View File

@ -1,25 +1,23 @@
from datetime import datetime, timedelta, timezone
from sqlalchemy import and_, asc, case, desc, func, select, text
from sqlalchemy.orm import aliased
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from base.exceptions import OperationNotAllowed from base.exceptions import OperationNotAllowed
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation, query from base.resolvers import mutation, query
from datetime import datetime, timedelta, timezone
from orm.reaction import Reaction, ReactionKind from orm.reaction import Reaction, ReactionKind
from orm.shout import Shout, ShoutReactionsFollower from orm.shout import Shout, ShoutReactionsFollower
from orm.user import User from orm.user import User
from services.notifications.notification_service import notification_service from services.notifications.notification_service import notification_service
from sqlalchemy import and_, asc, case, desc, func, select, text
from sqlalchemy.orm import aliased
def add_reaction_stat_columns(q): def add_reaction_stat_columns(q):
aliased_reaction = aliased(Reaction) aliased_reaction = aliased(Reaction)
q = q.outerjoin(aliased_reaction, Reaction.id == aliased_reaction.replyTo).add_columns( q = q.outerjoin(aliased_reaction, Reaction.id == aliased_reaction.replyTo).add_columns(
func.sum(aliased_reaction.id).label('reacted_stat'), func.sum(aliased_reaction.id).label("reacted_stat"),
func.sum(case((aliased_reaction.body.is_not(None), 1), else_=0)).label('commented_stat'), func.sum(case((aliased_reaction.body.is_not(None), 1), else_=0)).label("commented_stat"),
func.sum( func.sum(
case( case(
(aliased_reaction.kind == ReactionKind.AGREE, 1), (aliased_reaction.kind == ReactionKind.AGREE, 1),
@ -32,7 +30,7 @@ def add_reaction_stat_columns(q):
(aliased_reaction.kind == ReactionKind.DISLIKE, -1), (aliased_reaction.kind == ReactionKind.DISLIKE, -1),
else_=0, else_=0,
) )
).label('rating_stat'), ).label("rating_stat"),
) )
return q return q
@ -91,7 +89,7 @@ def reactions_unfollow(user_id: int, shout_id: int):
def is_published_author(session, user_id): def is_published_author(session, user_id):
'''checks if user has at least one publication''' """checks if user has at least one publication"""
return ( return (
session.query(Shout) session.query(Shout)
.where(Shout.authors.contains(user_id)) .where(Shout.authors.contains(user_id))
@ -102,7 +100,7 @@ def is_published_author(session, user_id):
def check_to_publish(session, user_id, reaction): def check_to_publish(session, user_id, reaction):
'''set shout to public if publicated approvers amount > 4''' """set shout to public if publicated approvers amount > 4"""
if not reaction.replyTo and reaction.kind in [ if not reaction.replyTo and reaction.kind in [
ReactionKind.ACCEPT, ReactionKind.ACCEPT,
ReactionKind.LIKE, ReactionKind.LIKE,
@ -126,7 +124,7 @@ def check_to_publish(session, user_id, reaction):
def check_to_hide(session, user_id, reaction): def check_to_hide(session, user_id, reaction):
'''hides any shout if 20% of reactions are negative''' """hides any shout if 20% of reactions are negative"""
if not reaction.replyTo and reaction.kind in [ if not reaction.replyTo and reaction.kind in [
ReactionKind.REJECT, ReactionKind.REJECT,
ReactionKind.DISLIKE, ReactionKind.DISLIKE,
@ -136,7 +134,11 @@ def check_to_hide(session, user_id, reaction):
approvers_reactions = session.query(Reaction).where(Reaction.shout == reaction.shout).all() approvers_reactions = session.query(Reaction).where(Reaction.shout == reaction.shout).all()
rejects = 0 rejects = 0
for r in approvers_reactions: for r in approvers_reactions:
if r.kind in [ReactionKind.REJECT, ReactionKind.DISLIKE, ReactionKind.DISPROOF]: if r.kind in [
ReactionKind.REJECT,
ReactionKind.DISLIKE,
ReactionKind.DISPROOF,
]:
rejects += 1 rejects += 1
if len(approvers_reactions) / rejects < 5: if len(approvers_reactions) / rejects < 5:
return True return True
@ -146,14 +148,14 @@ def check_to_hide(session, user_id, reaction):
def set_published(session, shout_id): def set_published(session, shout_id):
s = session.query(Shout).where(Shout.id == shout_id).first() s = session.query(Shout).where(Shout.id == shout_id).first()
s.publishedAt = datetime.now(tz=timezone.utc) s.publishedAt = datetime.now(tz=timezone.utc)
s.visibility = text('public') s.visibility = text("public")
session.add(s) session.add(s)
session.commit() session.commit()
def set_hidden(session, shout_id): def set_hidden(session, shout_id):
s = session.query(Shout).where(Shout.id == shout_id).first() s = session.query(Shout).where(Shout.id == shout_id).first()
s.visibility = text('community') s.visibility = text("community")
session.add(s) session.add(s)
session.commit() session.commit()
@ -162,7 +164,7 @@ def set_hidden(session, shout_id):
@login_required @login_required
async def create_reaction(_, info, reaction): async def create_reaction(_, info, reaction):
auth: AuthCredentials = info.context["request"].auth auth: AuthCredentials = info.context["request"].auth
reaction['createdBy'] = auth.user_id reaction["createdBy"] = auth.user_id
rdict = {} rdict = {}
with local_session() as session: with local_session() as session:
shout = session.query(Shout).where(Shout.id == reaction["shout"]).one() shout = session.query(Shout).where(Shout.id == reaction["shout"]).one()
@ -230,8 +232,8 @@ async def create_reaction(_, info, reaction):
await notification_service.handle_new_reaction(r.id) await notification_service.handle_new_reaction(r.id)
rdict = r.dict() rdict = r.dict()
rdict['shout'] = shout.dict() rdict["shout"] = shout.dict()
rdict['createdBy'] = author.dict() rdict["createdBy"] = author.dict()
# self-regulation mechanics # self-regulation mechanics
if check_to_hide(session, auth.user_id, r): if check_to_hide(session, auth.user_id, r):
@ -244,7 +246,7 @@ async def create_reaction(_, info, reaction):
except Exception as e: except Exception as e:
print(f"[resolvers.reactions] error on reactions autofollowing: {e}") print(f"[resolvers.reactions] error on reactions autofollowing: {e}")
rdict['stat'] = {"commented": 0, "reacted": 0, "rating": 0} rdict["stat"] = {"commented": 0, "reacted": 0, "rating": 0}
return {"reaction": rdict} return {"reaction": rdict}
@ -274,7 +276,11 @@ async def update_reaction(_, info, id, reaction={}):
if reaction.get("range"): if reaction.get("range"):
r.range = reaction.get("range") r.range = reaction.get("range")
session.commit() session.commit()
r.stat = {"commented": commented_stat, "reacted": reacted_stat, "rating": rating_stat} r.stat = {
"commented": commented_stat,
"reacted": reacted_stat,
"rating": rating_stat,
}
return {"reaction": r} return {"reaction": r}
@ -338,7 +344,7 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0):
if by.get("comment"): if by.get("comment"):
q = q.filter(func.length(Reaction.body) > 0) q = q.filter(func.length(Reaction.body) > 0)
if len(by.get('search', '')) > 2: if len(by.get("search", "")) > 2:
q = q.filter(Reaction.body.ilike(f'%{by["body"]}%')) q = q.filter(Reaction.body.ilike(f'%{by["body"]}%'))
if by.get("days"): if by.get("days"):
@ -346,7 +352,7 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0):
q = q.filter(Reaction.createdAt > after) q = q.filter(Reaction.createdAt > after)
order_way = asc if by.get("sort", "").startswith("-") else desc order_way = asc if by.get("sort", "").startswith("-") else desc
order_field = by.get("sort", "").replace('-', '') or Reaction.createdAt order_field = by.get("sort", "").replace("-", "") or Reaction.createdAt
q = q.group_by(Reaction.id, User.id, Shout.id).order_by(order_way(order_field)) q = q.group_by(Reaction.id, User.id, Shout.id).order_by(order_way(order_field))
@ -357,9 +363,14 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0):
reactions = [] reactions = []
with local_session() as session: with local_session() as session:
for [reaction, user, shout, reacted_stat, commented_stat, rating_stat] in session.execute( for [
q reaction,
): user,
shout,
reacted_stat,
commented_stat,
rating_stat,
] in session.execute(q):
reaction.createdBy = user reaction.createdBy = user
reaction.shout = shout reaction.shout = shout
reaction.stat = { reaction.stat = {

17
resolvers/zine/topics.py
View File

@ -1,12 +1,11 @@
from sqlalchemy import and_, distinct, func, select
from sqlalchemy.orm import aliased
from auth.authenticate import login_required from auth.authenticate import login_required
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation, query from base.resolvers import mutation, query
from orm import User from orm import User
from orm.shout import ShoutAuthor, ShoutTopic from orm.shout import ShoutAuthor, ShoutTopic
from orm.topic import Topic, TopicFollower from orm.topic import Topic, TopicFollower
from sqlalchemy import and_, distinct, func, select
from sqlalchemy.orm import aliased
def add_topic_stat_columns(q): def add_topic_stat_columns(q):
@ -15,11 +14,11 @@ def add_topic_stat_columns(q):
q = ( q = (
q.outerjoin(ShoutTopic, Topic.id == ShoutTopic.topic) q.outerjoin(ShoutTopic, Topic.id == ShoutTopic.topic)
.add_columns(func.count(distinct(ShoutTopic.shout)).label('shouts_stat')) .add_columns(func.count(distinct(ShoutTopic.shout)).label("shouts_stat"))
.outerjoin(aliased_shout_author, ShoutTopic.shout == aliased_shout_author.shout) .outerjoin(aliased_shout_author, ShoutTopic.shout == aliased_shout_author.shout)
.add_columns(func.count(distinct(aliased_shout_author.user)).label('authors_stat')) .add_columns(func.count(distinct(aliased_shout_author.user)).label("authors_stat"))
.outerjoin(aliased_topic_follower) .outerjoin(aliased_topic_follower)
.add_columns(func.count(distinct(aliased_topic_follower.follower)).label('followers_stat')) .add_columns(func.count(distinct(aliased_topic_follower.follower)).label("followers_stat"))
) )
q = q.group_by(Topic.id) q = q.group_by(Topic.id)
@ -29,7 +28,11 @@ def add_topic_stat_columns(q):
def add_stat(topic, stat_columns): def add_stat(topic, stat_columns):
[shouts_stat, authors_stat, followers_stat] = stat_columns [shouts_stat, authors_stat, followers_stat] = stat_columns
topic.stat = {"shouts": shouts_stat, "authors": authors_stat, "followers": followers_stat} topic.stat = {
"shouts": shouts_stat,
"authors": authors_stat,
"followers": followers_stat,
}
return topic return topic

62
server.py
View File

@ -1,45 +1,44 @@
from settings import DEV_SERVER_PID_FILE_NAME, PORT
import os import os
import sys import sys
import uvicorn import uvicorn
from settings import DEV_SERVER_PID_FILE_NAME, PORT
def exception_handler(exception_type, exception, traceback, debug_hook=sys.excepthook): def exception_handler(exception_type, exception, traceback, debug_hook=sys.excepthook):
print("%s: %s" % (exception_type.__name__, exception)) print("%s: %s" % (exception_type.__name__, exception))
log_settings = { log_settings = {
'version': 1, "version": 1,
'disable_existing_loggers': True, "disable_existing_loggers": True,
'formatters': { "formatters": {
'default': { "default": {
'()': 'uvicorn.logging.DefaultFormatter', "()": "uvicorn.logging.DefaultFormatter",
'fmt': '%(levelprefix)s %(message)s', "fmt": "%(levelprefix)s %(message)s",
'use_colors': None, "use_colors": None,
}, },
'access': { "access": {
'()': 'uvicorn.logging.AccessFormatter', "()": "uvicorn.logging.AccessFormatter",
'fmt': '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s', "fmt": '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s',
}, },
}, },
'handlers': { "handlers": {
'default': { "default": {
'formatter': 'default', "formatter": "default",
'class': 'logging.StreamHandler', "class": "logging.StreamHandler",
'stream': 'ext://sys.stderr', "stream": "ext://sys.stderr",
}, },
'access': { "access": {
'formatter': 'access', "formatter": "access",
'class': 'logging.StreamHandler', "class": "logging.StreamHandler",
'stream': 'ext://sys.stdout', "stream": "ext://sys.stdout",
}, },
}, },
'loggers': { "loggers": {
'uvicorn': {'handlers': ['default'], 'level': 'INFO'}, "uvicorn": {"handlers": ["default"], "level": "INFO"},
'uvicorn.error': {'level': 'INFO', 'handlers': ['default'], 'propagate': True}, "uvicorn.error": {"level": "INFO", "handlers": ["default"], "propagate": True},
'uvicorn.access': {'handlers': ['access'], 'level': 'INFO', 'propagate': False}, "uvicorn.access": {"handlers": ["access"], "level": "INFO", "propagate": False},
}, },
} }
@ -48,7 +47,8 @@ local_headers = [
("Access-Control-Allow-Origin", "https://localhost:3000"), ("Access-Control-Allow-Origin", "https://localhost:3000"),
( (
"Access-Control-Allow-Headers", "Access-Control-Allow-Headers",
"DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization", "DNT,User-Agent,X-Requested-With,If-Modified-Since,"
+ "Cache-Control,Content-Type,Range,Authorization",
), ),
("Access-Control-Expose-Headers", "Content-Length,Content-Range"), ("Access-Control-Expose-Headers", "Content-Length,Content-Range"),
("Access-Control-Allow-Credentials", "true"), ("Access-Control-Allow-Credentials", "true"),
@ -92,4 +92,10 @@ if __name__ == "__main__":
json_tables() json_tables()
else: else:
sys.excepthook = exception_handler sys.excepthook = exception_handler
uvicorn.run("main:app", host="0.0.0.0", port=PORT, proxy_headers=True, server_header=True) uvicorn.run(
"main:app",
host="0.0.0.0",
port=PORT,
proxy_headers=True,
server_header=True,
)

8
services/following.py
View File

@ -18,7 +18,7 @@ class Following:
class FollowingManager: class FollowingManager:
lock = asyncio.Lock() lock = asyncio.Lock()
data = {'author': [], 'topic': [], 'shout': [], 'chat': []} data = {"author": [], "topic": [], "shout": [], "chat": []}
@staticmethod @staticmethod
async def register(kind, uid): async def register(kind, uid):
@ -34,13 +34,13 @@ class FollowingManager:
async def push(kind, payload): async def push(kind, payload):
try: try:
async with FollowingManager.lock: async with FollowingManager.lock:
if kind == 'chat': if kind == "chat":
for chat in FollowingManager['chat']: for chat in FollowingManager["chat"]:
if payload.message["chatId"] == chat.uid: if payload.message["chatId"] == chat.uid:
chat.queue.put_nowait(payload) chat.queue.put_nowait(payload)
else: else:
for entity in FollowingManager[kind]: for entity in FollowingManager[kind]:
if payload.shout['createdBy'] == entity.uid: if payload.shout["createdBy"] == entity.uid:
entity.queue.put_nowait(payload) entity.queue.put_nowait(payload)
except Exception as e: except Exception as e:
print(Exception(e)) print(Exception(e))

8
services/main.py
View File

@ -5,9 +5,9 @@ from services.stat.viewed import ViewedStorage
async def storages_init(): async def storages_init():
with local_session() as session: with local_session() as session:
print('[main] initialize SearchService') print("[main] initialize SearchService")
await SearchService.init(session) await SearchService.init(session)
print('[main] SearchService initialized') print("[main] SearchService initialized")
print('[main] initialize storages') print("[main] initialize storages")
await ViewedStorage.init() await ViewedStorage.init()
print('[main] storages initialized') print("[main] storages initialized")

26
services/notifications/notification_service.py
View File

@ -1,14 +1,13 @@
import asyncio
import json
from datetime import datetime, timezone
from sqlalchemy import and_
from base.orm import local_session from base.orm import local_session
from datetime import datetime, timezone
from orm import Notification, Reaction, Shout, User from orm import Notification, Reaction, Shout, User
from orm.notification import NotificationType from orm.notification import NotificationType
from orm.reaction import ReactionKind from orm.reaction import ReactionKind
from services.notifications.sse import connection_manager from services.notifications.sse import connection_manager
from sqlalchemy import and_
import asyncio
import json
def shout_to_shout_data(shout): def shout_to_shout_data(shout):
@ -16,13 +15,18 @@ def shout_to_shout_data(shout):
def user_to_user_data(user): def user_to_user_data(user):
return {"id": user.id, "name": user.name, "slug": user.slug, "userpic": user.userpic} return {
"id": user.id,
"name": user.name,
"slug": user.slug,
"userpic": user.userpic,
}
def update_prev_notification(notification, user, reaction): def update_prev_notification(notification, user, reaction):
notification_data = json.loads(notification.data) notification_data = json.loads(notification.data)
notification_data["users"] = [u for u in notification_data["users"] if u['id'] != user.id] notification_data["users"] = [u for u in notification_data["users"] if u["id"] != user.id]
notification_data["users"].append(user_to_user_data(user)) notification_data["users"].append(user_to_user_data(user))
if notification_data["reactionIds"] is None: if notification_data["reactionIds"] is None:
@ -61,7 +65,7 @@ class NewReactionNotificator:
Notification.type == NotificationType.NEW_REPLY, Notification.type == NotificationType.NEW_REPLY,
Notification.shout == shout.id, Notification.shout == shout.id,
Notification.reaction == parent_reaction.id, Notification.reaction == parent_reaction.id,
Notification.seen == False, Notification.seen == False, # noqa: E712
) )
) )
.first() .first()
@ -103,7 +107,7 @@ class NewReactionNotificator:
Notification.user == shout.createdBy, Notification.user == shout.createdBy,
Notification.type == NotificationType.NEW_COMMENT, Notification.type == NotificationType.NEW_COMMENT,
Notification.shout == shout.id, Notification.shout == shout.id,
Notification.seen == False, Notification.seen == False, # noqa: E712
) )
) )
.first() .first()
@ -154,7 +158,7 @@ class NotificationService:
try: try:
await notificator.run() await notificator.run()
except Exception as e: except Exception as e:
print(f'[NotificationService.worker] error: {str(e)}') print(f"[NotificationService.worker] error: {str(e)}")
notification_service = NotificationService() notification_service = NotificationService()

6
services/notifications/sse.py
View File

@ -1,9 +1,9 @@
import asyncio
import json
from sse_starlette.sse import EventSourceResponse from sse_starlette.sse import EventSourceResponse
from starlette.requests import Request from starlette.requests import Request
import asyncio
import json
class ConnectionManager: class ConnectionManager:
def __init__(self): def __init__(self):

15
services/search.py
View File

@ -1,10 +1,10 @@
import asyncio
import json
from base.redis import redis from base.redis import redis
from orm.shout import Shout from orm.shout import Shout
from resolvers.zine.load import load_shouts_by from resolvers.zine.load import load_shouts_by
import asyncio
import json
class SearchService: class SearchService:
lock = asyncio.Lock() lock = asyncio.Lock()
@ -13,7 +13,7 @@ class SearchService:
@staticmethod @staticmethod
async def init(session): async def init(session):
async with SearchService.lock: async with SearchService.lock:
print('[search.service] did nothing') print("[search.service] did nothing")
SearchService.cache = {} SearchService.cache = {}
@staticmethod @staticmethod
@ -21,7 +21,12 @@ class SearchService:
cached = await redis.execute("GET", text) cached = await redis.execute("GET", text)
if not cached: if not cached:
async with SearchService.lock: async with SearchService.lock:
options = {"title": text, "body": text, "limit": limit, "offset": offset} options = {
"title": text,
"body": text,
"limit": limit,
"offset": offset,
}
payload = await load_shouts_by(None, None, options) payload = await load_shouts_by(None, None, options)
await redis.execute("SET", text, json.dumps(payload)) await redis.execute("SET", text, json.dumps(payload))
return payload return payload

28
services/stat/viewed.py
View File

@ -1,16 +1,14 @@
import asyncio from base.orm import local_session
import time
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from gql import Client, gql
from gql.transport.aiohttp import AIOHTTPTransport
from orm import Topic
from orm.shout import Shout, ShoutTopic
from os import environ, path from os import environ, path
from ssl import create_default_context from ssl import create_default_context
from gql import Client, gql import asyncio
from gql.transport.aiohttp import AIOHTTPTransport import time
from sqlalchemy import func
from base.orm import local_session
from orm import Topic, User
from orm.shout import Shout, ShoutTopic
load_facts = gql( load_facts = gql(
""" """
@ -46,7 +44,7 @@ query getDomains {
} }
""" """
) )
schema_str = open(path.dirname(__file__) + '/ackee.graphql').read() schema_str = open(path.dirname(__file__) + "/ackee.graphql").read()
token = environ.get("ACKEE_TOKEN", "") token = environ.get("ACKEE_TOKEN", "")
@ -54,7 +52,9 @@ def create_client(headers=None, schema=None):
return Client( return Client(
schema=schema, schema=schema,
transport=AIOHTTPTransport( transport=AIOHTTPTransport(
url="https://ackee.discours.io/api", ssl=create_default_context(), headers=headers url="https://ackee.discours.io/api",
ssl=create_default_context(),
headers=headers,
), ),
) )
@ -98,7 +98,7 @@ class ViewedStorage:
try: try:
for page in self.pages: for page in self.pages:
p = page["value"].split("?")[0] p = page["value"].split("?")[0]
slug = p.split('discours.io/')[-1] slug = p.split("discours.io/")[-1]
shouts[slug] = page["count"] shouts[slug] = page["count"]
for slug in shouts.keys(): for slug in shouts.keys():
await ViewedStorage.increment(slug, shouts[slug]) await ViewedStorage.increment(slug, shouts[slug])
@ -162,14 +162,14 @@ class ViewedStorage:
self.by_topics[topic.slug][shout_slug] = self.by_shouts[shout_slug] self.by_topics[topic.slug][shout_slug] = self.by_shouts[shout_slug]
@staticmethod @staticmethod
async def increment(shout_slug, amount=1, viewer='ackee'): async def increment(shout_slug, amount=1, viewer="ackee"):
"""the only way to change views counter""" """the only way to change views counter"""
self = ViewedStorage self = ViewedStorage
async with self.lock: async with self.lock:
# TODO optimize, currenty we execute 1 DB transaction per shout # TODO optimize, currenty we execute 1 DB transaction per shout
with local_session() as session: with local_session() as session:
shout = session.query(Shout).where(Shout.slug == shout_slug).one() shout = session.query(Shout).where(Shout.slug == shout_slug).one()
if viewer == 'old-discours': if viewer == "old-discours":
# this is needed for old db migration # this is needed for old db migration
if shout.viewsOld == amount: if shout.viewsOld == amount:
print(f"viewsOld amount: {amount}") print(f"viewsOld amount: {amount}")

6
services/zine/gittask.py
View File

@ -1,8 +1,8 @@
from pathlib import Path
from settings import SHOUTS_REPO
import asyncio import asyncio
import subprocess import subprocess
from pathlib import Path
from settings import SHOUTS_REPO
class GitTask: class GitTask:

2
settings.py
View File

@ -31,4 +31,4 @@ SENTRY_DSN = environ.get("SENTRY_DSN")
SESSION_SECRET_KEY = environ.get("SESSION_SECRET_KEY") or "!secret" SESSION_SECRET_KEY = environ.get("SESSION_SECRET_KEY") or "!secret"
# for local development # for local development
DEV_SERVER_PID_FILE_NAME = 'dev-server.pid' DEV_SERVER_PID_FILE_NAME = "dev-server.pid"

7
setup.cfg Executable file → Normal file
View File

@ -9,15 +9,16 @@ force_alphabetical_sort = false
[tool:brunette] [tool:brunette]
# https://github.com/odwyersoftware/brunette # https://github.com/odwyersoftware/brunette
line-length = 120 line-length = 100
single-quotes = false single-quotes = false
[flake8] [flake8]
# https://github.com/PyCQA/flake8 # https://github.com/PyCQA/flake8
exclude = .git,__pycache__,.mypy_cache,.vercel exclude = .git,__pycache__,.mypy_cache,.vercel
max-line-length = 120 max-line-length = 100
max-complexity = 15 max-complexity = 10
select = B,C,E,F,W,T4,B9 select = B,C,E,F,W,T4,B9
# FIXME
# E203: Whitespace before ':' # E203: Whitespace before ':'
# E266: Too many leading '#' for block comment # E266: Too many leading '#' for block comment
# E501: Line too long (82 > 79 characters) # E501: Line too long (82 > 79 characters)

39
setup.cfg.bak Normal file
View File

@ -0,0 +1,39 @@
[isort]
# https://github.com/PyCQA/isort
line_length = 120
multi_line_output = 3
include_trailing_comma = true
force_grid_wrap = 0
use_parentheses = true
force_alphabetical_sort = false
[tool:brunette]
# https://github.com/odwyersoftware/brunette
line-length = 120
single-quotes = false
[flake8]
# https://github.com/PyCQA/flake8
exclude = .git,__pycache__,.mypy_cache,.vercel
max-line-length = 120
max-complexity = 15
select = B,C,E,F,W,T4,B9
# E203: Whitespace before ':'
# E266: Too many leading '#' for block comment
# E501: Line too long (82 > 79 characters)
# E722: Do not use bare except, specify exception instead
# W503: Line break occurred before a binary operator
# F403: 'from module import *' used; unable to detect undefined names
# C901: Function is too complex
ignore = E203,E266,E501,E722,W503,F403,C901
[mypy]
# https://github.com/python/mypy
ignore_missing_imports = true
warn_return_any = false
warn_unused_configs = true
disallow_untyped_calls = true
disallow_untyped_defs = true
disallow_incomplete_defs = true
[mypy-api.*]
ignore_errors = true

3
validations/auth.py
View File

@ -1,6 +1,5 @@
from typing import Optional, Text
from pydantic import BaseModel from pydantic import BaseModel
from typing import Optional, Text
class AuthInput(BaseModel): class AuthInput(BaseModel):

3
validations/inbox.py
View File

@ -1,6 +1,5 @@
from typing import List, Optional, Text
from pydantic import BaseModel from pydantic import BaseModel
from typing import List, Optional, Text
class Message(BaseModel): class Message(BaseModel):