diff --git a/.editorconfig b/.editorconfig index 61ba0b13..58b88ea4 100644 --- a/.editorconfig +++ b/.editorconfig @@ -2,7 +2,7 @@ root = true [*] indent_style = tabs -indent_size = 1 +indent_size = 2 end_of_line = lf charset = utf-8 trim_trailing_whitespace=true diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..a872cf27 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +ignore = D203 +exclude = .git,__pycache__ +max-complexity = 10 +max-line-length = 108 diff --git a/auth/authenticate.py b/auth/authenticate.py index 4d4e651b..db58768d 100644 --- a/auth/authenticate.py +++ b/auth/authenticate.py @@ -16,119 +16,126 @@ from settings import JWT_AUTH_HEADER, EMAIL_TOKEN_LIFE_SPAN class _Authenticate: - @classmethod - async def verify(cls, token: str): - """ - Rules for a token to be valid. - 1. token format is legal && - token exists in redis database && - token is not expired - 2. token format is legal && - token exists in redis database && - token is expired && - token is of specified type - """ - try: - payload = JWTCodec.decode(token) - except ExpiredSignatureError: - payload = JWTCodec.decode(token, verify_exp=False) - if not await cls.exists(payload.user_id, token): - raise InvalidToken("Login expired, please login again") - if payload.device == "mobile": # noqa - "we cat set mobile token to be valid forever" - return payload - except DecodeError as e: - raise InvalidToken("token format error") from e - else: - if not await cls.exists(payload.user_id, token): - raise InvalidToken("Login expired, please login again") - return payload + @classmethod + async def verify(cls, token: str): + """ + Rules for a token to be valid. + 1. token format is legal && + token exists in redis database && + token is not expired + 2. token format is legal && + token exists in redis database && + token is expired && + token is of specified type + """ + try: + payload = JWTCodec.decode(token) + except ExpiredSignatureError: + payload = JWTCodec.decode(token, verify_exp=False) + if not await cls.exists(payload.user_id, token): + raise InvalidToken("Login expired, please login again") + if payload.device == "mobile": # noqa + "we cat set mobile token to be valid forever" + return payload + except DecodeError as e: + raise InvalidToken("token format error") from e + else: + if not await cls.exists(payload.user_id, token): + raise InvalidToken("Login expired, please login again") + return payload - @classmethod - async def exists(cls, user_id, token): - return await TokenStorage.exist(f"{user_id}-{token}") + @classmethod + async def exists(cls, user_id, token): + return await TokenStorage.exist(f"{user_id}-{token}") class JWTAuthenticate(AuthenticationBackend): - async def authenticate( - self, request: HTTPConnection - ) -> Optional[Tuple[AuthCredentials, AuthUser]]: - if JWT_AUTH_HEADER not in request.headers: - return AuthCredentials(scopes=[]), AuthUser(user_id=None) + async def authenticate( + self, request: HTTPConnection + ) -> Optional[Tuple[AuthCredentials, AuthUser]]: + if JWT_AUTH_HEADER not in request.headers: + return AuthCredentials(scopes=[]), AuthUser(user_id=None) - token = request.headers[JWT_AUTH_HEADER] - try: - payload = await _Authenticate.verify(token) - except Exception as exc: - return AuthCredentials(scopes=[], error_message=str(exc)), AuthUser(user_id=None) - - if payload is None: - return AuthCredentials(scopes=[]), AuthUser(user_id=None) + token = request.headers[JWT_AUTH_HEADER] + try: + payload = await _Authenticate.verify(token) + except Exception as exc: + return AuthCredentials(scopes=[], error_message=str(exc)), AuthUser( + user_id=None + ) - if not payload.device in ("pc", "mobile"): - return AuthCredentials(scopes=[]), AuthUser(user_id=None) + if payload is None: + return AuthCredentials(scopes=[]), AuthUser(user_id=None) - user = await UserStorage.get_user(payload.user_id) - if not user: - return AuthCredentials(scopes=[]), AuthUser(user_id=None) + if not payload.device in ("pc", "mobile"): + return AuthCredentials(scopes=[]), AuthUser(user_id=None) + + user = await UserStorage.get_user(payload.user_id) + if not user: + return AuthCredentials(scopes=[]), AuthUser(user_id=None) + + scopes = await user.get_permission() + return ( + AuthCredentials(user_id=payload.user_id, scopes=scopes, logged_in=True), + user, + ) - scopes = await user.get_permission() - return AuthCredentials(user_id=payload.user_id, scopes=scopes, logged_in=True), user class EmailAuthenticate: - @staticmethod - async def get_email_token(user): - token = await Authorize.authorize( - user, - device="email", - life_span=EMAIL_TOKEN_LIFE_SPAN - ) - return token + @staticmethod + async def get_email_token(user): + token = await Authorize.authorize( + user, device="email", life_span=EMAIL_TOKEN_LIFE_SPAN + ) + return token + + @staticmethod + async def authenticate(token): + payload = await _Authenticate.verify(token) + if payload is None: + raise InvalidToken("invalid token") + if payload.device != "email": + raise InvalidToken("invalid token") + with local_session() as session: + user = session.query(User).filter_by(id=payload.user_id).first() + if not user: + raise Exception("user not exist") + if not user.emailConfirmed: + user.emailConfirmed = True + session.commit() + auth_token = await Authorize.authorize(user) + return (auth_token, user) - @staticmethod - async def authenticate(token): - payload = await _Authenticate.verify(token) - if payload is None: - raise InvalidToken("invalid token") - if payload.device != "email": - raise InvalidToken("invalid token") - with local_session() as session: - user = session.query(User).filter_by(id=payload.user_id).first() - if not user: - raise Exception("user not exist") - if not user.emailConfirmed: - user.emailConfirmed = True - session.commit() - auth_token = await Authorize.authorize(user) - return (auth_token, user) class ResetPassword: - @staticmethod - async def get_reset_token(user): - exp = datetime.utcnow() + timedelta(seconds=EMAIL_TOKEN_LIFE_SPAN) - token = JWTCodec.encode(user, exp=exp, device="pc") - await TokenStorage.save(f"{user.id}-reset-{token}", EMAIL_TOKEN_LIFE_SPAN, True) - return token + @staticmethod + async def get_reset_token(user): + exp = datetime.utcnow() + timedelta(seconds=EMAIL_TOKEN_LIFE_SPAN) + token = JWTCodec.encode(user, exp=exp, device="pc") + await TokenStorage.save(f"{user.id}-reset-{token}", EMAIL_TOKEN_LIFE_SPAN, True) + return token - @staticmethod - async def verify(token): - try: - payload = JWTCodec.decode(token) - except ExpiredSignatureError: - raise InvalidToken("Login expired, please login again") - except DecodeError as e: - raise InvalidToken("token format error") from e - else: - if not await TokenStorage.exist(f"{payload.user_id}-reset-{token}"): - raise InvalidToken("Login expired, please login again") + @staticmethod + async def verify(token): + try: + payload = JWTCodec.decode(token) + except ExpiredSignatureError: + raise InvalidToken("Login expired, please login again") + except DecodeError as e: + raise InvalidToken("token format error") from e + else: + if not await TokenStorage.exist(f"{payload.user_id}-reset-{token}"): + raise InvalidToken("Login expired, please login again") + + return payload.user_id - return payload.user_id def login_required(func): - @wraps(func) - async def wrap(parent, info: GraphQLResolveInfo, *args, **kwargs): - auth: AuthCredentials = info.context["request"].auth - if not auth.logged_in: - return {"error" : auth.error_message or "Please login"} - return await func(parent, info, *args, **kwargs) - return wrap + @wraps(func) + async def wrap(parent, info: GraphQLResolveInfo, *args, **kwargs): + auth: AuthCredentials = info.context["request"].auth + if not auth.logged_in: + return {"error": auth.error_message or "Please login"} + return await func(parent, info, *args, **kwargs) + + return wrap diff --git a/auth/authorize.py b/auth/authorize.py index b8fb2c05..f9a538be 100644 --- a/auth/authorize.py +++ b/auth/authorize.py @@ -5,38 +5,41 @@ from base.redis import redis from settings import JWT_LIFE_SPAN from auth.validations import User -class TokenStorage: - @staticmethod - async def save(token_key, life_span, auto_delete=True): - await redis.execute("SET", token_key, "True") - if auto_delete: - expire_at = (datetime.now() + timedelta(seconds=life_span)).timestamp() - await redis.execute("EXPIREAT", token_key, int(expire_at)) - @staticmethod - async def exist(token_key): - return await redis.execute("GET", token_key) +class TokenStorage: + @staticmethod + async def save(token_key, life_span, auto_delete=True): + await redis.execute("SET", token_key, "True") + if auto_delete: + expire_at = (datetime.now() + timedelta(seconds=life_span)).timestamp() + await redis.execute("EXPIREAT", token_key, int(expire_at)) + + @staticmethod + async def exist(token_key): + return await redis.execute("GET", token_key) class Authorize: - @staticmethod - async def authorize(user: User, device: str = "pc", life_span = JWT_LIFE_SPAN, auto_delete=True) -> str: - exp = datetime.utcnow() + timedelta(seconds=life_span) - token = JWTCodec.encode(user, exp=exp, device=device) - await TokenStorage.save(f"{user.id}-{token}", life_span, auto_delete) - return token + @staticmethod + async def authorize( + user: User, device: str = "pc", life_span=JWT_LIFE_SPAN, auto_delete=True + ) -> str: + exp = datetime.utcnow() + timedelta(seconds=life_span) + token = JWTCodec.encode(user, exp=exp, device=device) + await TokenStorage.save(f"{user.id}-{token}", life_span, auto_delete) + return token - @staticmethod - async def revoke(token: str) -> bool: - try: - payload = JWTCodec.decode(token) - except: # noqa - pass - else: - await redis.execute("DEL", f"{payload.user_id}-{token}") - return True + @staticmethod + async def revoke(token: str) -> bool: + try: + payload = JWTCodec.decode(token) + except: # noqa + pass + else: + await redis.execute("DEL", f"{payload.user_id}-{token}") + return True - @staticmethod - async def revoke_all(user: User): - tokens = await redis.execute("KEYS", f"{user.id}-*") - await redis.execute("DEL", *tokens) + @staticmethod + async def revoke_all(user: User): + tokens = await redis.execute("KEYS", f"{user.id}-*") + await redis.execute("DEL", *tokens) diff --git a/auth/email.py b/auth/email.py index dcc815af..9e5c002f 100644 --- a/auth/email.py +++ b/auth/email.py @@ -2,71 +2,83 @@ import requests from starlette.responses import RedirectResponse from auth.authenticate import EmailAuthenticate, ResetPassword from base.orm import local_session -from settings import BACKEND_URL, MAILGUN_API_KEY, MAILGUN_DOMAIN, RESET_PWD_URL, \ - CONFIRM_EMAIL_URL, ERROR_URL_ON_FRONTEND +from settings import ( + BACKEND_URL, + MAILGUN_API_KEY, + MAILGUN_DOMAIN, + RESET_PWD_URL, + CONFIRM_EMAIL_URL, + ERROR_URL_ON_FRONTEND, +) MAILGUN_API_URL = "https://api.mailgun.net/v3/%s/messages" % (MAILGUN_DOMAIN) MAILGUN_FROM = "discours.io " % (MAILGUN_DOMAIN) AUTH_URL = "%s/email_authorize" % (BACKEND_URL) -email_templates = {"confirm_email" : "", "auth_email" : "", "reset_password_email" : ""} +email_templates = {"confirm_email": "", "auth_email": "", "reset_password_email": ""} + def load_email_templates(): - for name in email_templates: - filename = "auth/templates/%s.tmpl" % name - with open(filename) as f: - email_templates[name] = f.read() - print("[auth.email] templates loaded") + for name in email_templates: + filename = "auth/templates/%s.tmpl" % name + with open(filename) as f: + email_templates[name] = f.read() + print("[auth.email] templates loaded") + async def send_confirm_email(user): - text = email_templates["confirm_email"] - token = await EmailAuthenticate.get_email_token(user) - await send_email(user, AUTH_URL, text, token) + text = email_templates["confirm_email"] + token = await EmailAuthenticate.get_email_token(user) + await send_email(user, AUTH_URL, text, token) + async def send_auth_email(user): - text = email_templates["auth_email"] - token = await EmailAuthenticate.get_email_token(user) - await send_email(user, AUTH_URL, text, token) + text = email_templates["auth_email"] + token = await EmailAuthenticate.get_email_token(user) + await send_email(user, AUTH_URL, text, token) + async def send_reset_password_email(user): - text = email_templates["reset_password_email"] - token = await ResetPassword.get_reset_token(user) - await send_email(user, RESET_PWD_URL, text, token) + text = email_templates["reset_password_email"] + token = await ResetPassword.get_reset_token(user) + await send_email(user, RESET_PWD_URL, text, token) + async def send_email(user, url, text, token): - to = "%s <%s>" % (user.username, user.email) - url_with_token = "%s?token=%s" % (url, token) - text = text % (url_with_token) - response = requests.post( - MAILGUN_API_URL, - auth = ("api", MAILGUN_API_KEY), - data = { - "from": MAILGUN_FROM, - "to": to, - "subject": "authorize log in", - "html": text - } - ) - response.raise_for_status() + to = "%s <%s>" % (user.username, user.email) + url_with_token = "%s?token=%s" % (url, token) + text = text % (url_with_token) + response = requests.post( + MAILGUN_API_URL, + auth=("api", MAILGUN_API_KEY), + data={ + "from": MAILGUN_FROM, + "to": to, + "subject": "authorize log in", + "html": text, + }, + ) + response.raise_for_status() + async def email_authorize(request): - token = request.query_params.get('token') - if not token: - url_with_error = "%s?error=%s" % (ERROR_URL_ON_FRONTEND, "INVALID_TOKEN") - return RedirectResponse(url = url_with_error) + token = request.query_params.get("token") + if not token: + url_with_error = "%s?error=%s" % (ERROR_URL_ON_FRONTEND, "INVALID_TOKEN") + return RedirectResponse(url=url_with_error) - try: - auth_token, user = await EmailAuthenticate.authenticate(token) - except: - url_with_error = "%s?error=%s" % (ERROR_URL_ON_FRONTEND, "INVALID_TOKEN") - return RedirectResponse(url = url_with_error) - - if not user.emailConfirmed: - with local_session() as session: - user.emailConfirmed = True - session.commit() + try: + auth_token, user = await EmailAuthenticate.authenticate(token) + except: + url_with_error = "%s?error=%s" % (ERROR_URL_ON_FRONTEND, "INVALID_TOKEN") + return RedirectResponse(url=url_with_error) - response = RedirectResponse(url = CONFIRM_EMAIL_URL) - response.set_cookie("token", auth_token) - return response + if not user.emailConfirmed: + with local_session() as session: + user.emailConfirmed = True + session.commit() + + response = RedirectResponse(url=CONFIRM_EMAIL_URL) + response.set_cookie("token", auth_token) + return response diff --git a/auth/identity.py b/auth/identity.py index 5e58755c..44f6a8c9 100644 --- a/auth/identity.py +++ b/auth/identity.py @@ -8,26 +8,32 @@ from sqlalchemy import or_ class Identity: - @staticmethod - def identity(orm_user: OrmUser, password: str) -> User: - user = User(**orm_user.dict()) - if user.password is None: - raise InvalidPassword("Wrong user password") - if not Password.verify(password, user.password): - raise InvalidPassword("Wrong user password") - return user - - @staticmethod - def identity_oauth(input) -> User: - with local_session() as session: - user = session.query(OrmUser).filter( - or_(OrmUser.oauth == input["oauth"], OrmUser.email == input["email"]) - ).first() - if not user: - user = OrmUser.create(**input) - if not user.oauth: - user.oauth = input["oauth"] - session.commit() + @staticmethod + def identity(orm_user: OrmUser, password: str) -> User: + user = User(**orm_user.dict()) + if user.password is None: + raise InvalidPassword("Wrong user password") + if not Password.verify(password, user.password): + raise InvalidPassword("Wrong user password") + return user - user = User(**user.dict()) - return user + @staticmethod + def identity_oauth(input) -> User: + with local_session() as session: + user = ( + session.query(OrmUser) + .filter( + or_( + OrmUser.oauth == input["oauth"], OrmUser.email == input["email"] + ) + ) + .first() + ) + if not user: + user = OrmUser.create(**input) + if not user.oauth: + user.oauth = input["oauth"] + session.commit() + + user = User(**user.dict()) + return user diff --git a/auth/jwtcodec.py b/auth/jwtcodec.py index 30b86dfe..cc1bf9dd 100644 --- a/auth/jwtcodec.py +++ b/auth/jwtcodec.py @@ -5,17 +5,22 @@ from auth.validations import PayLoad, User class JWTCodec: - @staticmethod - def encode(user: User, exp: datetime, device: str = "pc") -> str: - payload = {"user_id": user.id, "device": device, "exp": exp, "iat": datetime.utcnow()} - return jwt.encode(payload, JWT_SECRET_KEY, JWT_ALGORITHM) + @staticmethod + def encode(user: User, exp: datetime, device: str = "pc") -> str: + payload = { + "user_id": user.id, + "device": device, + "exp": exp, + "iat": datetime.utcnow(), + } + return jwt.encode(payload, JWT_SECRET_KEY, JWT_ALGORITHM) - @staticmethod - def decode(token: str, verify_exp: bool = True) -> PayLoad: - payload = jwt.decode( - token, - key=JWT_SECRET_KEY, - options={"verify_exp": verify_exp}, - algorithms=[JWT_ALGORITHM], - ) - return PayLoad(**payload) + @staticmethod + def decode(token: str, verify_exp: bool = True) -> PayLoad: + payload = jwt.decode( + token, + key=JWT_SECRET_KEY, + options={"verify_exp": verify_exp}, + algorithms=[JWT_ALGORITHM], + ) + return PayLoad(**payload) diff --git a/auth/oauth.py b/auth/oauth.py index 868187d2..09600eea 100644 --- a/auth/oauth.py +++ b/auth/oauth.py @@ -8,79 +8,84 @@ from settings import OAUTH_CLIENTS, BACKEND_URL, OAUTH_CALLBACK_URL oauth = OAuth() oauth.register( - name='facebook', - client_id=OAUTH_CLIENTS["FACEBOOK"]["id"], - client_secret=OAUTH_CLIENTS["FACEBOOK"]["key"], - access_token_url='https://graph.facebook.com/v11.0/oauth/access_token', - access_token_params=None, - authorize_url='https://www.facebook.com/v11.0/dialog/oauth', - authorize_params=None, - api_base_url='https://graph.facebook.com/', - client_kwargs={'scope': 'public_profile email'}, + name="facebook", + client_id=OAUTH_CLIENTS["FACEBOOK"]["id"], + client_secret=OAUTH_CLIENTS["FACEBOOK"]["key"], + access_token_url="https://graph.facebook.com/v11.0/oauth/access_token", + access_token_params=None, + authorize_url="https://www.facebook.com/v11.0/dialog/oauth", + authorize_params=None, + api_base_url="https://graph.facebook.com/", + client_kwargs={"scope": "public_profile email"}, ) oauth.register( - name='github', - client_id=OAUTH_CLIENTS["GITHUB"]["id"], - client_secret=OAUTH_CLIENTS["GITHUB"]["key"], - access_token_url='https://github.com/login/oauth/access_token', - access_token_params=None, - authorize_url='https://github.com/login/oauth/authorize', - authorize_params=None, - api_base_url='https://api.github.com/', - client_kwargs={'scope': 'user:email'}, + name="github", + client_id=OAUTH_CLIENTS["GITHUB"]["id"], + client_secret=OAUTH_CLIENTS["GITHUB"]["key"], + access_token_url="https://github.com/login/oauth/access_token", + access_token_params=None, + authorize_url="https://github.com/login/oauth/authorize", + authorize_params=None, + api_base_url="https://api.github.com/", + client_kwargs={"scope": "user:email"}, ) oauth.register( - name='google', - client_id=OAUTH_CLIENTS["GOOGLE"]["id"], - client_secret=OAUTH_CLIENTS["GOOGLE"]["key"], - server_metadata_url="https://accounts.google.com/.well-known/openid-configuration", - client_kwargs={'scope': 'openid email profile'} + name="google", + client_id=OAUTH_CLIENTS["GOOGLE"]["id"], + client_secret=OAUTH_CLIENTS["GOOGLE"]["key"], + server_metadata_url="https://accounts.google.com/.well-known/openid-configuration", + client_kwargs={"scope": "openid email profile"}, ) + async def google_profile(client, request, token): - profile = await client.parse_id_token(request, token) - profile["id"] = profile["sub"] - return profile + profile = await client.parse_id_token(request, token) + profile["id"] = profile["sub"] + return profile + async def facebook_profile(client, request, token): - profile = await client.get('me?fields=name,id,email', token=token) - return profile.json() + profile = await client.get("me?fields=name,id,email", token=token) + return profile.json() + async def github_profile(client, request, token): - profile = await client.get('user', token=token) - return profile.json() + profile = await client.get("user", token=token) + return profile.json() + profile_callbacks = { - "google" : google_profile, - "facebook" : facebook_profile, - "github" : github_profile + "google": google_profile, + "facebook": facebook_profile, + "github": github_profile, } async def oauth_login(request): - provider = request.path_params['provider'] - request.session['provider'] = provider - client = oauth.create_client(provider) - redirect_uri = "%s/%s" % (BACKEND_URL, 'oauth_authorize') - return await client.authorize_redirect(request, redirect_uri) + provider = request.path_params["provider"] + request.session["provider"] = provider + client = oauth.create_client(provider) + redirect_uri = "%s/%s" % (BACKEND_URL, "oauth_authorize") + return await client.authorize_redirect(request, redirect_uri) + async def oauth_authorize(request): - provider = request.session['provider'] - client = oauth.create_client(provider) - token = await client.authorize_access_token(request) - get_profile = profile_callbacks[provider] - profile = await get_profile(client, request, token) - user_oauth_info = "%s:%s" % (provider, profile["id"]) - user_input = { - "oauth" : user_oauth_info, - "email" : profile["email"], - "username" : profile["name"] - } - user = Identity.identity_oauth(user_input) - token = await Authorize.authorize(user, device="pc") + provider = request.session["provider"] + client = oauth.create_client(provider) + token = await client.authorize_access_token(request) + get_profile = profile_callbacks[provider] + profile = await get_profile(client, request, token) + user_oauth_info = "%s:%s" % (provider, profile["id"]) + user_input = { + "oauth": user_oauth_info, + "email": profile["email"], + "username": profile["name"], + } + user = Identity.identity_oauth(user_input) + token = await Authorize.authorize(user, device="pc") - response = RedirectResponse(url = OAUTH_CALLBACK_URL) - response.set_cookie("token", token) - return response + response = RedirectResponse(url=OAUTH_CALLBACK_URL) + response.set_cookie("token", token) + return response diff --git a/base/orm.py b/base/orm.py index 22c135db..d4d1c5ba 100644 --- a/base/orm.py +++ b/base/orm.py @@ -5,50 +5,52 @@ from sqlalchemy.orm import Session from sqlalchemy.sql.schema import Table from settings import DB_URL -if DB_URL.startswith('sqlite'): - engine = create_engine(DB_URL) +if DB_URL.startswith("sqlite"): + engine = create_engine(DB_URL) else: - engine = create_engine(DB_URL, convert_unicode=True, echo=False, \ - pool_size=10, max_overflow=20) + engine = create_engine( + DB_URL, convert_unicode=True, echo=False, pool_size=10, max_overflow=20 + ) T = TypeVar("T") REGISTRY: Dict[str, type] = {} + def local_session(): - return Session(bind=engine, expire_on_commit=False) + return Session(bind=engine, expire_on_commit=False) class Base(declarative_base()): - __table__: Table - __tablename__: str - __new__: Callable - __init__: Callable + __table__: Table + __tablename__: str + __new__: Callable + __init__: Callable - __abstract__: bool = True - __table_args__ = {"extend_existing": True} - id: int = Column(Integer, primary_key=True) + __abstract__: bool = True + __table_args__ = {"extend_existing": True} + id: int = Column(Integer, primary_key=True) - def __init_subclass__(cls, **kwargs): - REGISTRY[cls.__name__] = cls + def __init_subclass__(cls, **kwargs): + REGISTRY[cls.__name__] = cls - @classmethod - def create(cls: Generic[T], **kwargs) -> Generic[T]: - instance = cls(**kwargs) - return instance.save() + @classmethod + def create(cls: Generic[T], **kwargs) -> Generic[T]: + instance = cls(**kwargs) + return instance.save() - def save(self) -> Generic[T]: - with local_session() as session: - session.add(self) - session.commit() - return self + def save(self) -> Generic[T]: + with local_session() as session: + session.add(self) + session.commit() + return self - def update(self, input): - column_names = self.__table__.columns.keys() - for (name, value) in input.items(): - if name in column_names: - setattr(self, name, value) + def update(self, input): + column_names = self.__table__.columns.keys() + for (name, value) in input.items(): + if name in column_names: + setattr(self, name, value) - def dict(self) -> Dict[str, Any]: - column_names = self.__table__.columns.keys() - return {c: getattr(self, c) for c in column_names} + def dict(self) -> Dict[str, Any]: + column_names = self.__table__.columns.keys() + return {c: getattr(self, c) for c in column_names} diff --git a/base/redis.py b/base/redis.py index 33dcc3ca..410a2713 100644 --- a/base/redis.py +++ b/base/redis.py @@ -1,34 +1,34 @@ import aioredis from settings import REDIS_URL + class Redis: - def __init__(self, uri=REDIS_URL): - self._uri: str = uri - self._instance = None + def __init__(self, uri=REDIS_URL): + self._uri: str = uri + self._instance = None - async def connect(self): - if self._instance is not None: - return - self._instance = aioredis.from_url(self._uri, encoding="utf-8") + async def connect(self): + if self._instance is not None: + return + self._instance = aioredis.from_url(self._uri, encoding="utf-8") - async def disconnect(self): - if self._instance is None: - return - self._instance.close() - await self._instance.wait_closed() - self._instance = None + async def disconnect(self): + if self._instance is None: + return + self._instance.close() + await self._instance.wait_closed() + self._instance = None - async def execute(self, command, *args, **kwargs): - return await self._instance.execute_command(command, *args, **kwargs) + async def execute(self, command, *args, **kwargs): + return await self._instance.execute_command(command, *args, **kwargs) - async def lrange(self, key, start, stop): - return await self._instance.lrange(key, start, stop) + async def lrange(self, key, start, stop): + return await self._instance.lrange(key, start, stop) - async def mget(self, key, *keys): - return await self._instance.mget(key, *keys) + async def mget(self, key, *keys): + return await self._instance.mget(key, *keys) redis = Redis() -__all__ = ['redis'] - +__all__ = ["redis"] diff --git a/base/resolvers.py b/base/resolvers.py index e6296549..4c771976 100644 --- a/base/resolvers.py +++ b/base/resolvers.py @@ -3,9 +3,11 @@ from ariadne import MutationType, QueryType, SubscriptionType, ScalarType datetime_scalar = ScalarType("DateTime") + @datetime_scalar.serializer def serialize_datetime(value): - return value.isoformat() + return value.isoformat() + query = QueryType() mutation = MutationType() diff --git a/main.py b/main.py index f559c9ac..b5085856 100644 --- a/main.py +++ b/main.py @@ -1,49 +1,58 @@ -from importlib import import_module -from ariadne import load_schema_from_path, make_executable_schema -from ariadne.asgi import GraphQL -from starlette.applications import Starlette -from starlette.middleware import Middleware -from starlette.middleware.authentication import AuthenticationMiddleware -from starlette.middleware.sessions import SessionMiddleware -from starlette.routing import Route -from auth.authenticate import JWTAuthenticate -from auth.oauth import oauth_login, oauth_authorize -from auth.email import email_authorize -from base.redis import redis -from base.resolvers import resolvers -from resolvers.zine import ShoutsCache -from services.stat.reacted import ReactedStorage -from services.stat.viewed import ViewedStorage -from services.zine.gittask import GitTask -from services.stat.topicstat import TopicStat -from services.zine.shoutauthor import ShoutAuthorStorage -import asyncio - -import_module('resolvers') -schema = make_executable_schema(load_schema_from_path("schema.graphql"), resolvers) - -middleware = [ - Middleware(AuthenticationMiddleware, backend=JWTAuthenticate()), - Middleware(SessionMiddleware, secret_key="!secret") -] - -async def start_up(): - await redis.connect() - viewed_storage_task = asyncio.create_task(ViewedStorage.worker()) - # reacted_storage_task = asyncio.create_task(ReactedStorage.worker()) - shouts_cache_task = asyncio.create_task(ShoutsCache.worker()) - shout_author_task = asyncio.create_task(ShoutAuthorStorage.worker()) - topic_stat_task = asyncio.create_task(TopicStat.worker()) - git_task = asyncio.create_task(GitTask.git_task_worker()) - -async def shutdown(): - await redis.disconnect() - -routes = [ - Route("/oauth/{provider}", endpoint=oauth_login), - Route("/oauth_authorize", endpoint=oauth_authorize), - Route("/email_authorize", endpoint=email_authorize) -] - -app = Starlette(debug=True, on_startup=[start_up], on_shutdown=[shutdown], middleware=middleware, routes=routes) -app.mount("/", GraphQL(schema, debug=True)) +from importlib import import_module +from ariadne import load_schema_from_path, make_executable_schema +from ariadne.asgi import GraphQL +from starlette.applications import Starlette +from starlette.middleware import Middleware +from starlette.middleware.authentication import AuthenticationMiddleware +from starlette.middleware.sessions import SessionMiddleware +from starlette.routing import Route +from auth.authenticate import JWTAuthenticate +from auth.oauth import oauth_login, oauth_authorize +from auth.email import email_authorize +from base.redis import redis +from base.resolvers import resolvers +from resolvers.zine import ShoutsCache +from services.stat.reacted import ReactedStorage +from services.stat.viewed import ViewedStorage +from services.zine.gittask import GitTask +from services.stat.topicstat import TopicStat +from services.zine.shoutauthor import ShoutAuthorStorage +import asyncio + +import_module("resolvers") +schema = make_executable_schema(load_schema_from_path("schema.graphql"), resolvers) # type: ignore + +middleware = [ + Middleware(AuthenticationMiddleware, backend=JWTAuthenticate()), + Middleware(SessionMiddleware, secret_key="!secret"), +] + + +async def start_up(): + await redis.connect() + viewed_storage_task = asyncio.create_task(ViewedStorage.worker()) + # reacted_storage_task = asyncio.create_task(ReactedStorage.worker()) + shouts_cache_task = asyncio.create_task(ShoutsCache.worker()) + shout_author_task = asyncio.create_task(ShoutAuthorStorage.worker()) + topic_stat_task = asyncio.create_task(TopicStat.worker()) + git_task = asyncio.create_task(GitTask.git_task_worker()) + + +async def shutdown(): + await redis.disconnect() + + +routes = [ + Route("/oauth/{provider}", endpoint=oauth_login), + Route("/oauth_authorize", endpoint=oauth_authorize), + Route("/email_authorize", endpoint=email_authorize), +] + +app = Starlette( + debug=True, + on_startup=[start_up], + on_shutdown=[shutdown], + middleware=middleware, + routes=routes, +) +app.mount("/", GraphQL(schema, debug=True)) diff --git a/migration/__init__.py b/migration/__init__.py index 0ee49822..6f8e0430 100644 --- a/migration/__init__.py +++ b/migration/__init__.py @@ -1,4 +1,4 @@ -''' cmd managed migration ''' +""" cmd managed migration """ import csv import asyncio from datetime import datetime @@ -8,6 +8,7 @@ import sys import os import bs4 import numpy as np + # from export import export_email_subscriptions from .export import export_mdx, export_slug from orm.reaction import Reaction @@ -21,293 +22,308 @@ from .tables.comments import migrate_2stage as migrateComment_2stage from settings import DB_URL -TODAY = datetime.strftime(datetime.now(), '%Y%m%d') +TODAY = datetime.strftime(datetime.now(), "%Y%m%d") -OLD_DATE = '2016-03-05 22:22:00.350000' +OLD_DATE = "2016-03-05 22:22:00.350000" def users_handle(storage): - ''' migrating users first ''' - counter = 0 - id_map = {} - print('[migration] migrating %d users' % (len(storage['users']['data']))) - for entry in storage['users']['data']: - oid = entry['_id'] - user = migrateUser(entry) - storage['users']['by_oid'][oid] = user # full - del user['password'] - del user['notifications'] - del user['emailConfirmed'] - del user['username'] - del user['email'] - storage['users']['by_slug'][user['slug']] = user # public - id_map[user['oid']] = user['slug'] - counter += 1 - ce = 0 - for entry in storage['users']['data']: - ce += migrateUser_2stage(entry, id_map) - return storage + """migrating users first""" + counter = 0 + id_map = {} + print("[migration] migrating %d users" % (len(storage["users"]["data"]))) + for entry in storage["users"]["data"]: + oid = entry["_id"] + user = migrateUser(entry) + storage["users"]["by_oid"][oid] = user # full + del user["password"] + del user["notifications"] + del user["emailConfirmed"] + del user["username"] + del user["email"] + storage["users"]["by_slug"][user["slug"]] = user # public + id_map[user["oid"]] = user["slug"] + counter += 1 + ce = 0 + for entry in storage["users"]["data"]: + ce += migrateUser_2stage(entry, id_map) + return storage def topics_handle(storage): - ''' topics from categories and tags ''' - counter = 0 - for t in (storage['topics']['tags'] + storage['topics']['cats']): - if t['slug'] in storage['replacements']: - t['slug'] = storage['replacements'][t['slug']] - topic = migrateTopic(t) - storage['topics']['by_oid'][t['_id']] = topic - storage['topics']['by_slug'][t['slug']] = topic - counter += 1 - else: - print('[migration] topic ' + t['slug'] + ' ignored') - for oldslug, newslug in storage['replacements'].items(): - if oldslug != newslug and oldslug in storage['topics']['by_slug']: - oid = storage['topics']['by_slug'][oldslug]['_id'] - del storage['topics']['by_slug'][oldslug] - storage['topics']['by_oid'][oid] = storage['topics']['by_slug'][newslug] - print('[migration] ' + str(counter) + ' topics migrated') - print('[migration] ' + str(len(storage['topics'] - ['by_oid'].values())) + ' topics by oid') - print('[migration] ' + str(len(storage['topics'] - ['by_slug'].values())) + ' topics by slug') - # raise Exception - return storage + """topics from categories and tags""" + counter = 0 + for t in storage["topics"]["tags"] + storage["topics"]["cats"]: + if t["slug"] in storage["replacements"]: + t["slug"] = storage["replacements"][t["slug"]] + topic = migrateTopic(t) + storage["topics"]["by_oid"][t["_id"]] = topic + storage["topics"]["by_slug"][t["slug"]] = topic + counter += 1 + else: + print("[migration] topic " + t["slug"] + " ignored") + for oldslug, newslug in storage["replacements"].items(): + if oldslug != newslug and oldslug in storage["topics"]["by_slug"]: + oid = storage["topics"]["by_slug"][oldslug]["_id"] + del storage["topics"]["by_slug"][oldslug] + storage["topics"]["by_oid"][oid] = storage["topics"]["by_slug"][newslug] + print("[migration] " + str(counter) + " topics migrated") + print( + "[migration] " + + str(len(storage["topics"]["by_oid"].values())) + + " topics by oid" + ) + print( + "[migration] " + + str(len(storage["topics"]["by_slug"].values())) + + " topics by slug" + ) + # raise Exception + return storage async def shouts_handle(storage, args): - ''' migrating content items one by one ''' - counter = 0 - discours_author = 0 - pub_counter = 0 - topics_dataset_bodies = [] - topics_dataset_tlist = [] - for entry in storage['shouts']['data']: - # slug - slug = get_shout_slug(entry) + """migrating content items one by one""" + counter = 0 + discours_author = 0 + pub_counter = 0 + topics_dataset_bodies = [] + topics_dataset_tlist = [] + for entry in storage["shouts"]["data"]: + # slug + slug = get_shout_slug(entry) - # single slug mode - if '-' in args and slug not in args: continue + # single slug mode + if "-" in args and slug not in args: + continue - # migrate - shout = await migrateShout(entry, storage) - storage['shouts']['by_oid'][entry['_id']] = shout - storage['shouts']['by_slug'][shout['slug']] = shout - # shouts.topics - if not shout['topics']: print('[migration] no topics!') + # migrate + shout = await migrateShout(entry, storage) + storage["shouts"]["by_oid"][entry["_id"]] = shout + storage["shouts"]["by_slug"][shout["slug"]] = shout + # shouts.topics + if not shout["topics"]: + print("[migration] no topics!") - # wuth author - author = shout['authors'][0].slug - if author == 'discours': discours_author += 1 - # print('[migration] ' + shout['slug'] + ' with author ' + author) + # wuth author + author = shout["authors"][0].slug + if author == "discours": + discours_author += 1 + # print('[migration] ' + shout['slug'] + ' with author ' + author) - if entry.get('published'): - if 'mdx' in args: export_mdx(shout) - pub_counter += 1 + if entry.get("published"): + if "mdx" in args: + export_mdx(shout) + pub_counter += 1 - # print main counter - counter += 1 - line = str(counter+1) + ': ' + shout['slug'] + " @" + author - print(line) - b = bs4.BeautifulSoup(shout['body'], 'html.parser') - texts = [] - texts.append(shout['title'].lower().replace(r'[^а-яА-Яa-zA-Z]', '')) - texts = b.findAll(text=True) - topics_dataset_bodies.append(u" ".join([x.strip().lower() for x in texts])) - topics_dataset_tlist.append(shout['topics']) - - # np.savetxt('topics_dataset.csv', (topics_dataset_bodies, topics_dataset_tlist), delimiter=',', fmt='%s') + # print main counter + counter += 1 + line = str(counter + 1) + ": " + shout["slug"] + " @" + author + print(line) + b = bs4.BeautifulSoup(shout["body"], "html.parser") + texts = [] + texts.append(shout["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")) + texts = b.findAll(text=True) + topics_dataset_bodies.append(" ".join([x.strip().lower() for x in texts])) + topics_dataset_tlist.append(shout["topics"]) - print('[migration] ' + str(counter) + ' content items were migrated') - print('[migration] ' + str(pub_counter) + ' have been published') - print('[migration] ' + str(discours_author) + ' authored by @discours') - return storage + # np.savetxt('topics_dataset.csv', (topics_dataset_bodies, topics_dataset_tlist), delimiter=',', fmt='%s') + + print("[migration] " + str(counter) + " content items were migrated") + print("[migration] " + str(pub_counter) + " have been published") + print("[migration] " + str(discours_author) + " authored by @discours") + return storage async def comments_handle(storage): - id_map = {} - ignored_counter = 0 - missed_shouts = {} - for oldcomment in storage['reactions']['data']: - if not oldcomment.get('deleted'): - reaction = await migrateComment(oldcomment, storage) - if type(reaction) == str: - missed_shouts[reaction] = oldcomment - elif type(reaction) == Reaction: - reaction = reaction.dict() - id = reaction['id'] - oid = reaction['oid'] - id_map[oid] = id - else: - ignored_counter += 1 + id_map = {} + ignored_counter = 0 + missed_shouts = {} + for oldcomment in storage["reactions"]["data"]: + if not oldcomment.get("deleted"): + reaction = await migrateComment(oldcomment, storage) + if type(reaction) == str: + missed_shouts[reaction] = oldcomment + elif type(reaction) == Reaction: + reaction = reaction.dict() + id = reaction["id"] + oid = reaction["oid"] + id_map[oid] = id + else: + ignored_counter += 1 - for reaction in storage['reactions']['data']: migrateComment_2stage( - reaction, id_map) - print('[migration] ' + str(len(id_map)) + ' comments migrated') - print('[migration] ' + str(ignored_counter) + ' comments ignored') - print('[migration] ' + str(len(missed_shouts.keys())) + - ' commented shouts missed') - missed_counter = 0 - for missed in missed_shouts.values(): - missed_counter += len(missed) - print('[migration] ' + str(missed_counter) + ' comments dropped') - return storage + for reaction in storage["reactions"]["data"]: + migrateComment_2stage(reaction, id_map) + print("[migration] " + str(len(id_map)) + " comments migrated") + print("[migration] " + str(ignored_counter) + " comments ignored") + print("[migration] " + str(len(missed_shouts.keys())) + " commented shouts missed") + missed_counter = 0 + for missed in missed_shouts.values(): + missed_counter += len(missed) + print("[migration] " + str(missed_counter) + " comments dropped") + return storage def bson_handle(): - # decode bson # preparing data - from migration import bson2json - bson2json.json_tables() + # decode bson # preparing data + from migration import bson2json + + bson2json.json_tables() -def export_one(slug, storage, args = None): - topics_handle(storage) - users_handle(storage) - shouts_handle(storage, args) - export_slug(slug, storage) +def export_one(slug, storage, args=None): + topics_handle(storage) + users_handle(storage) + shouts_handle(storage, args) + export_slug(slug, storage) async def all_handle(storage, args): - print('[migration] handle everything') - users_handle(storage) - topics_handle(storage) - await shouts_handle(storage, args) - await comments_handle(storage) - # export_email_subscriptions() - print('[migration] done!') + print("[migration] handle everything") + users_handle(storage) + topics_handle(storage) + await shouts_handle(storage, args) + await comments_handle(storage) + # export_email_subscriptions() + print("[migration] done!") def data_load(): - storage = { - 'content_items': { - 'by_oid': {}, - 'by_slug': {}, - }, - 'shouts': { - 'by_oid': {}, - 'by_slug': {}, - 'data': [] - }, - 'reactions': { - 'by_oid': {}, - 'by_slug': {}, - 'by_content': {}, - 'data': [] - }, - 'topics': { - 'by_oid': {}, - 'by_slug': {}, - 'cats': [], - 'tags': [], - }, - 'users': { - 'by_oid': {}, - 'by_slug': {}, - 'data': [] - }, - 'replacements': json.loads(open('migration/tables/replacements.json').read()) - } - users_data = [] - tags_data = [] - cats_data = [] - comments_data = [] - content_data = [] - try: - users_data = json.loads(open('migration/data/users.json').read()) - print('[migration.load] ' + str(len(users_data)) + ' users ') - tags_data = json.loads(open('migration/data/tags.json').read()) - storage['topics']['tags'] = tags_data - print('[migration.load] ' + str(len(tags_data)) + ' tags ') - cats_data = json.loads( - open('migration/data/content_item_categories.json').read()) - storage['topics']['cats'] = cats_data - print('[migration.load] ' + str(len(cats_data)) + ' cats ') - comments_data = json.loads(open('migration/data/comments.json').read()) - storage['reactions']['data'] = comments_data - print('[migration.load] ' + str(len(comments_data)) + ' comments ') - content_data = json.loads(open('migration/data/content_items.json').read()) - storage['shouts']['data'] = content_data - print('[migration.load] ' + str(len(content_data)) + ' content items ') - # fill out storage - for x in users_data: - storage['users']['by_oid'][x['_id']] = x - # storage['users']['by_slug'][x['slug']] = x - # no user.slug yet - print('[migration.load] ' + str(len(storage['users'] - ['by_oid'].keys())) + ' users by oid') - for x in tags_data: - storage['topics']['by_oid'][x['_id']] = x - storage['topics']['by_slug'][x['slug']] = x - for x in cats_data: - storage['topics']['by_oid'][x['_id']] = x - storage['topics']['by_slug'][x['slug']] = x - print('[migration.load] ' + str(len(storage['topics'] - ['by_slug'].keys())) + ' topics by slug') - for item in content_data: - slug = get_shout_slug(item) - storage['content_items']['by_slug'][slug] = item - storage['content_items']['by_oid'][item['_id']] = item - print('[migration.load] ' + str(len(content_data)) + ' content items') - for x in comments_data: - storage['reactions']['by_oid'][x['_id']] = x - cid = x['contentItem'] - storage['reactions']['by_content'][cid] = x - ci = storage['content_items']['by_oid'].get(cid, {}) - if 'slug' in ci: storage['reactions']['by_slug'][ci['slug']] = x - print('[migration.load] ' + str(len(storage['reactions'] - ['by_content'].keys())) + ' with comments') - except Exception as e: raise e - storage['users']['data'] = users_data - storage['topics']['tags'] = tags_data - storage['topics']['cats'] = cats_data - storage['shouts']['data'] = content_data - storage['reactions']['data'] = comments_data - return storage + storage = { + "content_items": { + "by_oid": {}, + "by_slug": {}, + }, + "shouts": {"by_oid": {}, "by_slug": {}, "data": []}, + "reactions": {"by_oid": {}, "by_slug": {}, "by_content": {}, "data": []}, + "topics": { + "by_oid": {}, + "by_slug": {}, + "cats": [], + "tags": [], + }, + "users": {"by_oid": {}, "by_slug": {}, "data": []}, + "replacements": json.loads(open("migration/tables/replacements.json").read()), + } + users_data = [] + tags_data = [] + cats_data = [] + comments_data = [] + content_data = [] + try: + users_data = json.loads(open("migration/data/users.json").read()) + print("[migration.load] " + str(len(users_data)) + " users ") + tags_data = json.loads(open("migration/data/tags.json").read()) + storage["topics"]["tags"] = tags_data + print("[migration.load] " + str(len(tags_data)) + " tags ") + cats_data = json.loads( + open("migration/data/content_item_categories.json").read() + ) + storage["topics"]["cats"] = cats_data + print("[migration.load] " + str(len(cats_data)) + " cats ") + comments_data = json.loads(open("migration/data/comments.json").read()) + storage["reactions"]["data"] = comments_data + print("[migration.load] " + str(len(comments_data)) + " comments ") + content_data = json.loads(open("migration/data/content_items.json").read()) + storage["shouts"]["data"] = content_data + print("[migration.load] " + str(len(content_data)) + " content items ") + # fill out storage + for x in users_data: + storage["users"]["by_oid"][x["_id"]] = x + # storage['users']['by_slug'][x['slug']] = x + # no user.slug yet + print( + "[migration.load] " + + str(len(storage["users"]["by_oid"].keys())) + + " users by oid" + ) + for x in tags_data: + storage["topics"]["by_oid"][x["_id"]] = x + storage["topics"]["by_slug"][x["slug"]] = x + for x in cats_data: + storage["topics"]["by_oid"][x["_id"]] = x + storage["topics"]["by_slug"][x["slug"]] = x + print( + "[migration.load] " + + str(len(storage["topics"]["by_slug"].keys())) + + " topics by slug" + ) + for item in content_data: + slug = get_shout_slug(item) + storage["content_items"]["by_slug"][slug] = item + storage["content_items"]["by_oid"][item["_id"]] = item + print("[migration.load] " + str(len(content_data)) + " content items") + for x in comments_data: + storage["reactions"]["by_oid"][x["_id"]] = x + cid = x["contentItem"] + storage["reactions"]["by_content"][cid] = x + ci = storage["content_items"]["by_oid"].get(cid, {}) + if "slug" in ci: + storage["reactions"]["by_slug"][ci["slug"]] = x + print( + "[migration.load] " + + str(len(storage["reactions"]["by_content"].keys())) + + " with comments" + ) + except Exception as e: + raise e + storage["users"]["data"] = users_data + storage["topics"]["tags"] = tags_data + storage["topics"]["cats"] = cats_data + storage["shouts"]["data"] = content_data + storage["reactions"]["data"] = comments_data + return storage def mongo_download(url): - if not url: raise Exception('\n\nYou should set MONGODB_URL enviroment variable\n') - print('[migration] mongodump ' + url) - subprocess.call([ - 'mongodump', - '--uri', url + '/?authSource=admin', - '--forceTableScan', - ], stderr = subprocess.STDOUT) + if not url: + raise Exception("\n\nYou should set MONGODB_URL enviroment variable\n") + print("[migration] mongodump " + url) + subprocess.call( + [ + "mongodump", + "--uri", + url + "/?authSource=admin", + "--forceTableScan", + ], + stderr=subprocess.STDOUT, + ) def create_pgdump(): - pgurl = DB_URL - if not pgurl: raise Exception('\n\nYou should set DATABASE_URL enviroment variable\n') - subprocess.call( - [ 'pg_dump', pgurl, '-f', TODAY + '-pgdump.sql'], - stderr = subprocess.STDOUT - ) - subprocess.call([ - 'scp', - TODAY + '-pgdump.sql', - 'root@build.discours.io:/root/.' - ]) + pgurl = DB_URL + if not pgurl: + raise Exception("\n\nYou should set DATABASE_URL enviroment variable\n") + subprocess.call( + ["pg_dump", pgurl, "-f", TODAY + "-pgdump.sql"], stderr=subprocess.STDOUT + ) + subprocess.call(["scp", TODAY + "-pgdump.sql", "root@build.discours.io:/root/."]) async def handle_auto(): - print('[migration] no command given, auto mode') - url = os.getenv('MONGODB_URL') - if url: mongo_download(url) - bson_handle() - await all_handle(data_load(), sys.argv) - create_pgdump() + print("[migration] no command given, auto mode") + url = os.getenv("MONGODB_URL") + if url: + mongo_download(url) + bson_handle() + await all_handle(data_load(), sys.argv) + create_pgdump() + async def main(): - if len(sys.argv) > 1: - cmd=sys.argv[1] - if type(cmd) == str: print('[migration] command: ' + cmd) - await handle_auto() - else: - print('[migration] usage: python server.py migrate') + if len(sys.argv) > 1: + cmd = sys.argv[1] + if type(cmd) == str: + print("[migration] command: " + cmd) + await handle_auto() + else: + print("[migration] usage: python server.py migrate") + def migrate(): - loop = asyncio.get_event_loop() - loop.run_until_complete(main()) - -if __name__ == '__main__': - migrate() \ No newline at end of file + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) + + +if __name__ == "__main__": + migrate() diff --git a/migration/bson2json.py b/migration/bson2json.py index a7d51b5d..82ff2281 100644 --- a/migration/bson2json.py +++ b/migration/bson2json.py @@ -4,25 +4,27 @@ import json from .utils import DateTimeEncoder -def json_tables(): - print('[migration] unpack dump/discours/*.bson to migration/data/*.json') - data = { - "content_items": [], - "content_item_categories": [], - "tags": [], - "email_subscriptions": [], - "users": [], - "comments": [] - } - for table in data.keys(): - lc = [] - with open('dump/discours/'+table+'.bson', 'rb') as f: - bs = f.read() - f.close() - base = 0 - while base < len(bs): - base, d = bson.decode_document(bs, base) - lc.append(d) - data[table] = lc - open(os.getcwd() + '/migration/data/'+table+'.json', 'w').write(json.dumps(lc,cls=DateTimeEncoder)) +def json_tables(): + print("[migration] unpack dump/discours/*.bson to migration/data/*.json") + data = { + "content_items": [], + "content_item_categories": [], + "tags": [], + "email_subscriptions": [], + "users": [], + "comments": [], + } + for table in data.keys(): + lc = [] + with open("dump/discours/" + table + ".bson", "rb") as f: + bs = f.read() + f.close() + base = 0 + while base < len(bs): + base, d = bson.decode_document(bs, base) + lc.append(d) + data[table] = lc + open(os.getcwd() + "/migration/data/" + table + ".json", "w").write( + json.dumps(lc, cls=DateTimeEncoder) + ) diff --git a/migration/export.py b/migration/export.py index 47f2c8c3..e8f28b58 100644 --- a/migration/export.py +++ b/migration/export.py @@ -1,4 +1,3 @@ - from datetime import datetime import json import os @@ -6,100 +5,150 @@ import frontmatter from .extract import extract_html, prepare_html_body from .utils import DateTimeEncoder -OLD_DATE = '2016-03-05 22:22:00.350000' -EXPORT_DEST = '../discoursio-web/data/' -parentDir = '/'.join(os.getcwd().split('/')[:-1]) -contentDir = parentDir + '/discoursio-web/content/' +OLD_DATE = "2016-03-05 22:22:00.350000" +EXPORT_DEST = "../discoursio-web/data/" +parentDir = "/".join(os.getcwd().split("/")[:-1]) +contentDir = parentDir + "/discoursio-web/content/" ts = datetime.now() + def get_metadata(r): - authors = [] - for a in r['authors']: - authors.append({ # a short version for public listings - 'slug': a.slug or 'discours', - 'name': a.name or 'Дискурс', - 'userpic': a.userpic or 'https://discours.io/static/img/discours.png' - }) - metadata = {} - metadata['title'] = r.get('title', '').replace('{', '(').replace('}', ')') - metadata['authors'] = authors - metadata['createdAt'] = r.get('createdAt', ts) - metadata['layout'] = r['layout'] - metadata['topics'] = [topic for topic in r['topics']] - metadata['topics'].sort() - if r.get('cover', False): metadata['cover'] = r.get('cover') - return metadata - + authors = [] + for a in r["authors"]: + authors.append( + { # a short version for public listings + "slug": a.slug or "discours", + "name": a.name or "Дискурс", + "userpic": a.userpic or "https://discours.io/static/img/discours.png", + } + ) + metadata = {} + metadata["title"] = r.get("title", "").replace("{", "(").replace("}", ")") + metadata["authors"] = authors + metadata["createdAt"] = r.get("createdAt", ts) + metadata["layout"] = r["layout"] + metadata["topics"] = [topic for topic in r["topics"]] + metadata["topics"].sort() + if r.get("cover", False): + metadata["cover"] = r.get("cover") + return metadata + + def export_mdx(r): - # print('[export] mdx %s' % r['slug']) - content = '' - metadata = get_metadata(r) - content = frontmatter.dumps(frontmatter.Post(r['body'], **metadata)) - ext = 'mdx' - filepath = contentDir + r['slug'] - bc = bytes(content,'utf-8').decode('utf-8','ignore') - open(filepath + '.' + ext, 'w').write(bc) + # print('[export] mdx %s' % r['slug']) + content = "" + metadata = get_metadata(r) + content = frontmatter.dumps(frontmatter.Post(r["body"], **metadata)) + ext = "mdx" + filepath = contentDir + r["slug"] + bc = bytes(content, "utf-8").decode("utf-8", "ignore") + open(filepath + "." + ext, "w").write(bc) + def export_body(shout, storage): - entry = storage['content_items']['by_oid'][shout['oid']] - if entry: - shout['body'] = prepare_html_body(entry) # prepare_md_body(entry) - export_mdx(shout) - print('[export] html for %s' % shout['slug']) - body = extract_html(entry) - open(contentDir + shout['slug'] + '.html', 'w').write(body) - else: - raise Exception('no content_items entry found') + entry = storage["content_items"]["by_oid"][shout["oid"]] + if entry: + shout["body"] = prepare_html_body(entry) # prepare_md_body(entry) + export_mdx(shout) + print("[export] html for %s" % shout["slug"]) + body = extract_html(entry) + open(contentDir + shout["slug"] + ".html", "w").write(body) + else: + raise Exception("no content_items entry found") + def export_slug(slug, storage): - shout = storage['shouts']['by_slug'][slug] - shout = storage['shouts']['by_slug'].get(slug) - assert shout, '[export] no shout found by slug: %s ' % slug - author = shout['authors'][0] - assert author, '[export] no author error' - export_body(shout, storage) + shout = storage["shouts"]["by_slug"][slug] + shout = storage["shouts"]["by_slug"].get(slug) + assert shout, "[export] no shout found by slug: %s " % slug + author = shout["authors"][0] + assert author, "[export] no author error" + export_body(shout, storage) + def export_email_subscriptions(): - email_subscriptions_data = json.loads(open('migration/data/email_subscriptions.json').read()) - for data in email_subscriptions_data: - # migrate_email_subscription(data) - pass - print('[migration] ' + str(len(email_subscriptions_data)) + ' email subscriptions exported') + email_subscriptions_data = json.loads( + open("migration/data/email_subscriptions.json").read() + ) + for data in email_subscriptions_data: + # migrate_email_subscription(data) + pass + print( + "[migration] " + + str(len(email_subscriptions_data)) + + " email subscriptions exported" + ) + def export_shouts(storage): - # update what was just migrated or load json again - if len(storage['users']['by_slugs'].keys()) == 0: - storage['users']['by_slugs'] = json.loads(open(EXPORT_DEST + 'authors.json').read()) - print('[migration] ' + str(len(storage['users']['by_slugs'].keys())) + ' exported authors ') - if len(storage['shouts']['by_slugs'].keys()) == 0: - storage['shouts']['by_slugs'] = json.loads(open(EXPORT_DEST + 'articles.json').read()) - print('[migration] ' + str(len(storage['shouts']['by_slugs'].keys())) + ' exported articles ') - for slug in storage['shouts']['by_slugs'].keys(): export_slug(slug, storage) + # update what was just migrated or load json again + if len(storage["users"]["by_slugs"].keys()) == 0: + storage["users"]["by_slugs"] = json.loads( + open(EXPORT_DEST + "authors.json").read() + ) + print( + "[migration] " + + str(len(storage["users"]["by_slugs"].keys())) + + " exported authors " + ) + if len(storage["shouts"]["by_slugs"].keys()) == 0: + storage["shouts"]["by_slugs"] = json.loads( + open(EXPORT_DEST + "articles.json").read() + ) + print( + "[migration] " + + str(len(storage["shouts"]["by_slugs"].keys())) + + " exported articles " + ) + for slug in storage["shouts"]["by_slugs"].keys(): + export_slug(slug, storage) -def export_json(export_articles = {}, export_authors = {}, export_topics = {}, export_comments = {}): - open(EXPORT_DEST + 'authors.json', 'w').write(json.dumps(export_authors, - cls=DateTimeEncoder, - indent=4, - sort_keys=True, - ensure_ascii=False)) - print('[migration] ' + str(len(export_authors.items())) + ' authors exported') - open(EXPORT_DEST + 'topics.json', 'w').write(json.dumps(export_topics, - cls=DateTimeEncoder, - indent=4, - sort_keys=True, - ensure_ascii=False)) - print('[migration] ' + str(len(export_topics.keys())) + ' topics exported') - - open(EXPORT_DEST + 'articles.json', 'w').write(json.dumps(export_articles, - cls=DateTimeEncoder, - indent=4, - sort_keys=True, - ensure_ascii=False)) - print('[migration] ' + str(len(export_articles.items())) + ' articles exported') - open(EXPORT_DEST + 'comments.json', 'w').write(json.dumps(export_comments, - cls=DateTimeEncoder, - indent=4, - sort_keys=True, - ensure_ascii=False)) - print('[migration] ' + str(len(export_comments.items())) + ' exported articles with comments') +def export_json( + export_articles={}, export_authors={}, export_topics={}, export_comments={} +): + open(EXPORT_DEST + "authors.json", "w").write( + json.dumps( + export_authors, + cls=DateTimeEncoder, + indent=4, + sort_keys=True, + ensure_ascii=False, + ) + ) + print("[migration] " + str(len(export_authors.items())) + " authors exported") + open(EXPORT_DEST + "topics.json", "w").write( + json.dumps( + export_topics, + cls=DateTimeEncoder, + indent=4, + sort_keys=True, + ensure_ascii=False, + ) + ) + print("[migration] " + str(len(export_topics.keys())) + " topics exported") + + open(EXPORT_DEST + "articles.json", "w").write( + json.dumps( + export_articles, + cls=DateTimeEncoder, + indent=4, + sort_keys=True, + ensure_ascii=False, + ) + ) + print("[migration] " + str(len(export_articles.items())) + " articles exported") + open(EXPORT_DEST + "comments.json", "w").write( + json.dumps( + export_comments, + cls=DateTimeEncoder, + indent=4, + sort_keys=True, + ensure_ascii=False, + ) + ) + print( + "[migration] " + + str(len(export_comments.items())) + + " exported articles with comments" + ) diff --git a/migration/extract.py b/migration/extract.py index eff2f22f..96913b3d 100644 --- a/migration/extract.py +++ b/migration/extract.py @@ -3,322 +3,397 @@ import re import base64 from .html2text import html2text -TOOLTIP_REGEX = r'(\/\/\/(.+)\/\/\/)' -contentDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..', 'discoursio-web', 'content') -s3 = 'https://discours-io.s3.amazonaws.com/' -cdn = 'https://assets.discours.io' +TOOLTIP_REGEX = r"(\/\/\/(.+)\/\/\/)" +contentDir = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "..", "..", "discoursio-web", "content" +) +s3 = "https://discours-io.s3.amazonaws.com/" +cdn = "https://assets.discours.io" -def replace_tooltips(body): - # change if you prefer regexp - newbody = body - matches = list(re.finditer(TOOLTIP_REGEX, body, re.IGNORECASE | re.MULTILINE))[1:] - for match in matches: - newbody = body.replace(match.group(1), '') # NOTE: doesn't work - if len(matches) > 0: - print('[extract] found %d tooltips' % len(matches)) - return newbody + +def replace_tooltips(body): + # change if you prefer regexp + newbody = body + matches = list(re.finditer(TOOLTIP_REGEX, body, re.IGNORECASE | re.MULTILINE))[1:] + for match in matches: + newbody = body.replace( + match.group(1), '' + ) # NOTE: doesn't work + if len(matches) > 0: + print("[extract] found %d tooltips" % len(matches)) + return newbody def place_tooltips(body): - parts = body.split('&&&') - l = len(parts) - newparts = list(parts) - placed = False - if l & 1: - if l > 1: - i = 1 - print('[extract] found %d tooltips' % (l-1)) - for part in parts[1:]: - if i & 1: - placed = True - if 'a class="footnote-url" href=' in part: - print('[extract] footnote: ' + part) - fn = 'a class="footnote-url" href="' - link = part.split(fn,1)[1].split('"', 1)[0] - extracted_part = part.split(fn,1)[0] + ' ' + part.split('/', 1)[-1] - newparts[i] = '' + extracted_part + '' - else: - newparts[i] = '%s' % part - # print('[extract] ' + newparts[i]) - else: - # print('[extract] ' + part[:10] + '..') - newparts[i] = part - i += 1 - return (''.join(newparts), placed) + parts = body.split("&&&") + l = len(parts) + newparts = list(parts) + placed = False + if l & 1: + if l > 1: + i = 1 + print("[extract] found %d tooltips" % (l - 1)) + for part in parts[1:]: + if i & 1: + placed = True + if 'a class="footnote-url" href=' in part: + print("[extract] footnote: " + part) + fn = 'a class="footnote-url" href="' + link = part.split(fn, 1)[1].split('"', 1)[0] + extracted_part = ( + part.split(fn, 1)[0] + " " + part.split("/", 1)[-1] + ) + newparts[i] = ( + "" + + extracted_part + + "" + ) + else: + newparts[i] = "%s" % part + # print('[extract] ' + newparts[i]) + else: + # print('[extract] ' + part[:10] + '..') + newparts[i] = part + i += 1 + return ("".join(newparts), placed) + IMG_REGEX = r"\!\[(.*?)\]\((data\:image\/(png|jpeg|jpg);base64\,((?:[A-Za-z\d+\/]{4})*(?:[A-Za-z\d+\/]{3}=|[A-Za-z\d+\/]{2}==)))\)" -parentDir = '/'.join(os.getcwd().split('/')[:-1]) -public = parentDir + '/discoursio-web/public' +parentDir = "/".join(os.getcwd().split("/")[:-1]) +public = parentDir + "/discoursio-web/public" cache = {} -def reextract_images(body, oid): - # change if you prefer regexp - matches = list(re.finditer(IMG_REGEX, body, re.IGNORECASE | re.MULTILINE))[1:] - i = 0 - for match in matches: - print('[extract] image ' + match.group(1)) - ext = match.group(3) - name = oid + str(i) - link = public + '/upload/image-' + name + '.' + ext - img = match.group(4) - title = match.group(1) # NOTE: this is not the title - if img not in cache: - content = base64.b64decode(img + '==') - print(str(len(img)) + ' image bytes been written') - open('../' + link, 'wb').write(content) - cache[img] = name - i += 1 - else: - print('[extract] image cached ' + cache[img]) - body.replace(str(match), '![' + title + '](' + cdn + link + ')') # WARNING: this does not work - return body +def reextract_images(body, oid): + # change if you prefer regexp + matches = list(re.finditer(IMG_REGEX, body, re.IGNORECASE | re.MULTILINE))[1:] + i = 0 + for match in matches: + print("[extract] image " + match.group(1)) + ext = match.group(3) + name = oid + str(i) + link = public + "/upload/image-" + name + "." + ext + img = match.group(4) + title = match.group(1) # NOTE: this is not the title + if img not in cache: + content = base64.b64decode(img + "==") + print(str(len(img)) + " image bytes been written") + open("../" + link, "wb").write(content) + cache[img] = name + i += 1 + else: + print("[extract] image cached " + cache[img]) + body.replace( + str(match), "![" + title + "](" + cdn + link + ")" + ) # WARNING: this does not work + return body + IMAGES = { - 'data:image/png': 'png', - 'data:image/jpg': 'jpg', - 'data:image/jpeg': 'jpg', + "data:image/png": "png", + "data:image/jpg": "jpg", + "data:image/jpeg": "jpg", } -b64 = ';base64,' +b64 = ";base64," + def extract_imageparts(bodyparts, prefix): - # recursive loop - newparts = list(bodyparts) - for current in bodyparts: - i = bodyparts.index(current) - for mime in IMAGES.keys(): - if mime == current[-len(mime):] and (i + 1 < len(bodyparts)): - print('[extract] ' + mime) - next = bodyparts[i+1] - ext = IMAGES[mime] - b64end = next.index(')') - b64encoded = next[:b64end] - name = prefix + '-' + str(len(cache)) - link = '/upload/image-' + name + '.' + ext - print('[extract] name: ' + name) - print('[extract] link: ' + link) - print('[extract] %d bytes' % len(b64encoded)) - if b64encoded not in cache: - try: - content = base64.b64decode(b64encoded + '==') - open(public + link, 'wb').write(content) - print('[extract] ' +str(len(content)) + ' image bytes been written') - cache[b64encoded] = name - except: - raise Exception - # raise Exception('[extract] error decoding image %r' %b64encoded) - else: - print('[extract] cached link ' + cache[b64encoded]) - name = cache[b64encoded] - link = cdn + '/upload/image-' + name + '.' + ext - newparts[i] = current[:-len(mime)] + current[-len(mime):] + link + next[-b64end:] - newparts[i+1] = next[:-b64end] - break - return extract_imageparts(newparts[i] + newparts[i+1] + b64.join(bodyparts[i+2:]), prefix) \ - if len(bodyparts) > (i + 1) else ''.join(newparts) + # recursive loop + newparts = list(bodyparts) + for current in bodyparts: + i = bodyparts.index(current) + for mime in IMAGES.keys(): + if mime == current[-len(mime) :] and (i + 1 < len(bodyparts)): + print("[extract] " + mime) + next = bodyparts[i + 1] + ext = IMAGES[mime] + b64end = next.index(")") + b64encoded = next[:b64end] + name = prefix + "-" + str(len(cache)) + link = "/upload/image-" + name + "." + ext + print("[extract] name: " + name) + print("[extract] link: " + link) + print("[extract] %d bytes" % len(b64encoded)) + if b64encoded not in cache: + try: + content = base64.b64decode(b64encoded + "==") + open(public + link, "wb").write(content) + print( + "[extract] " + + str(len(content)) + + " image bytes been written" + ) + cache[b64encoded] = name + except: + raise Exception + # raise Exception('[extract] error decoding image %r' %b64encoded) + else: + print("[extract] cached link " + cache[b64encoded]) + name = cache[b64encoded] + link = cdn + "/upload/image-" + name + "." + ext + newparts[i] = ( + current[: -len(mime)] + + current[-len(mime) :] + + link + + next[-b64end:] + ) + newparts[i + 1] = next[:-b64end] + break + return ( + extract_imageparts( + newparts[i] + newparts[i + 1] + b64.join(bodyparts[i + 2 :]), prefix + ) + if len(bodyparts) > (i + 1) + else "".join(newparts) + ) + def extract_dataimages(parts, prefix): - newparts = list(parts) - for part in parts: - i = parts.index(part) - if part.endswith(']('): - [ext, rest] = parts[i+1].split(b64) - name = prefix + '-' + str(len(cache)) - if ext == '/jpeg': ext = 'jpg' - else: ext = ext.replace('/', '') - link = '/upload/image-' + name + '.' + ext - print('[extract] filename: ' + link) - b64end = rest.find(')') - if b64end !=-1: - b64encoded = rest[:b64end] - print('[extract] %d text bytes' % len(b64encoded)) - # write if not cached - if b64encoded not in cache: - try: - content = base64.b64decode(b64encoded + '==') - open(public + link, 'wb').write(content) - print('[extract] ' +str(len(content)) + ' image bytes') - cache[b64encoded] = name - except: - raise Exception - # raise Exception('[extract] error decoding image %r' %b64encoded) - else: - print('[extract] 0 image bytes, cached for ' + cache[b64encoded]) - name = cache[b64encoded] + newparts = list(parts) + for part in parts: + i = parts.index(part) + if part.endswith("]("): + [ext, rest] = parts[i + 1].split(b64) + name = prefix + "-" + str(len(cache)) + if ext == "/jpeg": + ext = "jpg" + else: + ext = ext.replace("/", "") + link = "/upload/image-" + name + "." + ext + print("[extract] filename: " + link) + b64end = rest.find(")") + if b64end != -1: + b64encoded = rest[:b64end] + print("[extract] %d text bytes" % len(b64encoded)) + # write if not cached + if b64encoded not in cache: + try: + content = base64.b64decode(b64encoded + "==") + open(public + link, "wb").write(content) + print("[extract] " + str(len(content)) + " image bytes") + cache[b64encoded] = name + except: + raise Exception + # raise Exception('[extract] error decoding image %r' %b64encoded) + else: + print("[extract] 0 image bytes, cached for " + cache[b64encoded]) + name = cache[b64encoded] - # update link with CDN - link = cdn + '/upload/image-' + name + '.' + ext - - # patch newparts - newparts[i+1] = link + rest[b64end:] - else: - raise Exception('cannot find the end of base64 encoded string') - else: - print('[extract] dataimage skipping part ' + str(i)) - continue - return ''.join(newparts) + # update link with CDN + link = cdn + "/upload/image-" + name + "." + ext + + # patch newparts + newparts[i + 1] = link + rest[b64end:] + else: + raise Exception("cannot find the end of base64 encoded string") + else: + print("[extract] dataimage skipping part " + str(i)) + continue + return "".join(newparts) + + +di = "data:image" -di = 'data:image' def extract_md_images(body, oid): - newbody = '' - body = body\ - .replace('\n! []('+di, '\n ![]('+di)\ - .replace('\n[]('+di, '\n![]('+di)\ - .replace(' []('+di, ' ![]('+di) - parts = body.split(di) - i = 0 - if len(parts) > 1: newbody = extract_dataimages(parts, oid) - else: newbody = body - return newbody + newbody = "" + body = ( + body.replace("\n! [](" + di, "\n ![](" + di) + .replace("\n[](" + di, "\n![](" + di) + .replace(" [](" + di, " ![](" + di) + ) + parts = body.split(di) + i = 0 + if len(parts) > 1: + newbody = extract_dataimages(parts, oid) + else: + newbody = body + return newbody def cleanup(body): - newbody = body\ - .replace('<', '').replace('>', '')\ - .replace('{', '(').replace('}', ')')\ - .replace('…', '...')\ - .replace(' __ ', ' ')\ - .replace('_ _', ' ')\ - .replace('****', '')\ - .replace('\u00a0', ' ')\ - .replace('\u02c6', '^')\ - .replace('\u00a0',' ')\ - .replace('\ufeff', '')\ - .replace('\u200b', '')\ - .replace('\u200c', '')\ - # .replace('\u2212', '-') - return newbody + newbody = ( + body.replace("<", "") + .replace(">", "") + .replace("{", "(") + .replace("}", ")") + .replace("…", "...") + .replace(" __ ", " ") + .replace("_ _", " ") + .replace("****", "") + .replace("\u00a0", " ") + .replace("\u02c6", "^") + .replace("\u00a0", " ") + .replace("\ufeff", "") + .replace("\u200b", "") + .replace("\u200c", "") + ) # .replace('\u2212', '-') + return newbody + def extract_md(body, oid): - newbody = body - if newbody: - newbody = extract_md_images(newbody, oid) - if not newbody: raise Exception('extract_images error') - newbody = cleanup(newbody) - if not newbody: raise Exception('cleanup error') - newbody, placed = place_tooltips(newbody) - if not newbody: raise Exception('place_tooltips error') - if placed: - newbody = 'import Tooltip from \'$/components/Article/Tooltip\'\n\n' + newbody - return newbody + newbody = body + if newbody: + newbody = extract_md_images(newbody, oid) + if not newbody: + raise Exception("extract_images error") + newbody = cleanup(newbody) + if not newbody: + raise Exception("cleanup error") + newbody, placed = place_tooltips(newbody) + if not newbody: + raise Exception("place_tooltips error") + if placed: + newbody = "import Tooltip from '$/components/Article/Tooltip'\n\n" + newbody + return newbody + def prepare_md_body(entry): - # body modifications - body = '' - kind = entry.get('type') - addon = '' - if kind == 'Video': - addon = '' - for m in entry.get('media', []): - if 'youtubeId' in m: addon += '\n' - elif 'vimeoId' in m: addon += '\n' - else: - print('[extract] media is not supported') - print(m) - body = 'import VideoPlayer from \'$/components/Article/VideoPlayer\'\n\n' + addon - - elif kind == 'Music': - addon = '' - for m in entry.get('media', []): - artist = m.get('performer') - trackname = '' - if artist: trackname += artist + ' - ' - if 'title' in m: trackname += m.get('title','') - addon += '\n' - body = 'import MusicPlayer from \'$/components/Article/MusicPlayer\'\n\n' + addon + # body modifications + body = "" + kind = entry.get("type") + addon = "" + if kind == "Video": + addon = "" + for m in entry.get("media", []): + if "youtubeId" in m: + addon += "\n" + elif "vimeoId" in m: + addon += "\n" + else: + print("[extract] media is not supported") + print(m) + body = "import VideoPlayer from '$/components/Article/VideoPlayer'\n\n" + addon + + elif kind == "Music": + addon = "" + for m in entry.get("media", []): + artist = m.get("performer") + trackname = "" + if artist: + trackname += artist + " - " + if "title" in m: + trackname += m.get("title", "") + addon += ( + '\n' + ) + body = "import MusicPlayer from '$/components/Article/MusicPlayer'\n\n" + addon + + body_orig = extract_html(entry) + if body_orig: + body += extract_md(html2text(body_orig), entry["_id"]) + if not body: + print("[extract] empty MDX body") + return body - body_orig = extract_html(entry) - if body_orig: body += extract_md(html2text(body_orig), entry['_id']) - if not body: print('[extract] empty MDX body') - return body def prepare_html_body(entry): - # body modifications - body = '' - kind = entry.get('type') - addon = '' - if kind == 'Video': - addon = '' - for m in entry.get('media', []): - if 'youtubeId' in m: - addon += '\n' - elif 'vimeoId' in m: - addon += '' - else: - print('[extract] media is not supported') - print(m) - body += addon - - elif kind == 'Music': - addon = '' - for m in entry.get('media', []): - artist = m.get('performer') - trackname = '' - if artist: trackname += artist + ' - ' - if 'title' in m: trackname += m.get('title','') - addon += '
' - addon += trackname - addon += '
' - body += addon + # body modifications + body = "" + kind = entry.get("type") + addon = "" + if kind == "Video": + addon = "" + for m in entry.get("media", []): + if "youtubeId" in m: + addon += '