diff --git a/.gitignore b/.gitignore index b8332f4b..10b8c415 100644 --- a/.gitignore +++ b/.gitignore @@ -148,3 +148,4 @@ dump *dump.sql *.csv dev-server-status.txt +/resetdb.sh diff --git a/CHECKS b/CHECKS index 5affecfc..e1253bd6 100644 --- a/CHECKS +++ b/CHECKS @@ -1,5 +1,5 @@ -WAIT=30 +WAIT=10 TIMEOUT=10 -ATTEMPTS=60 # 60 * 30 = 30 min +ATTEMPTS=30 # 10 * 30 = 5 min / Playground diff --git a/auth/authenticate.py b/auth/authenticate.py index 2b0142f4..9821133b 100644 --- a/auth/authenticate.py +++ b/auth/authenticate.py @@ -2,11 +2,14 @@ from functools import wraps from typing import Optional, Tuple from graphql.type import GraphQLResolveInfo +from sqlalchemy.orm import joinedload, exc from starlette.authentication import AuthenticationBackend from starlette.requests import HTTPConnection from auth.credentials import AuthCredentials, AuthUser -from services.auth.users import UserStorage +from base.orm import local_session +from orm import User, Role + from settings import SESSION_TOKEN_HEADER from auth.tokenstorage import SessionToken from base.exceptions import InvalidToken, OperationNotAllowed, Unauthorized @@ -32,10 +35,26 @@ class JWTAuthenticate(AuthenticationBackend): payload = await SessionToken.verify(token) if payload is None: return AuthCredentials(scopes=[]), AuthUser(user_id=None) - user = await UserStorage.get_user(payload.user_id) + + with local_session() as session: + try: + user = ( + session.query(User).options( + joinedload(User.roles), + joinedload(Role.permissions), + joinedload(User.ratings) + ).filter( + User.id == id + ).one() + ) + except exc.NoResultFound: + user = None + if not user: return AuthCredentials(scopes=[]), AuthUser(user_id=None) - scopes = await user.get_permission() + + scopes = user.get_permission() + return ( AuthCredentials( user_id=payload.user_id, @@ -46,10 +65,10 @@ class JWTAuthenticate(AuthenticationBackend): ) else: InvalidToken("please try again") - except Exception as exc: + except Exception as e: print("[auth.authenticate] session token verify error") - print(exc) - return AuthCredentials(scopes=[], error_message=str(exc)), AuthUser(user_id=None) + print(e) + return AuthCredentials(scopes=[], error_message=str(e)), AuthUser(user_id=None) def login_required(func): diff --git a/main.py b/main.py index c354efea..7b8fcee9 100644 --- a/main.py +++ b/main.py @@ -16,12 +16,8 @@ from base.redis import redis from base.resolvers import resolvers from resolvers.auth import confirm_email_handler from services.main import storages_init -# from services.stat.reacted import ReactedStorage -from services.stat.topicstat import TopicStat from services.stat.viewed import ViewedStorage -from services.zine.topics import TopicStorage from services.zine.gittask import GitTask -from services.zine.shoutauthor import ShoutAuthorStorage from settings import DEV_SERVER_STATUS_FILE_NAME import_module("resolvers") @@ -37,16 +33,8 @@ async def start_up(): init_tables() await redis.connect() await storages_init() - topics_random_work = asyncio.create_task(TopicStorage().worker()) - print(topics_random_work) views_stat_task = asyncio.create_task(ViewedStorage().worker()) print(views_stat_task) - # reacted_storage_task = asyncio.create_task(ReactedStorage.worker()) - # print(reacted_storage_task) - shout_author_task = asyncio.create_task(ShoutAuthorStorage.worker()) - print(shout_author_task) - topic_stat_task = asyncio.create_task(TopicStat.worker()) - print(topic_stat_task) git_task = asyncio.create_task(GitTask.git_task_worker()) print(git_task) diff --git a/migration/__init__.py b/migration/__init__.py index 043b0972..7995cec6 100644 --- a/migration/__init__.py +++ b/migration/__init__.py @@ -96,16 +96,16 @@ async def shouts_handle(storage, args): continue # migrate - shout = await migrateShout(entry, storage) - if shout: - storage["shouts"]["by_oid"][entry["_id"]] = shout - storage["shouts"]["by_slug"][shout["slug"]] = shout + shout_dict = await migrateShout(entry, storage) + if shout_dict: + storage["shouts"]["by_oid"][entry["_id"]] = shout_dict + storage["shouts"]["by_slug"][shout_dict["slug"]] = shout_dict # shouts.topics - if not shout["topics"]: + if not shout_dict["topics"]: print("[migration] no topics!") # with author - author: str = shout["authors"][0].dict() + author = shout_dict["authors"][0] if author["slug"] == "discours": discours_author += 1 if author["slug"] == "anonymous": @@ -114,19 +114,20 @@ async def shouts_handle(storage, args): if entry.get("published"): if "mdx" in args: - export_mdx(shout) + export_mdx(shout_dict) pub_counter += 1 # print main counter counter += 1 - line = str(counter + 1) + ": " + shout["slug"] + " @" + author["slug"] - print(line) + print('[migration] shouts_handle %d: %s @%s' % ( + (counter + 1), shout_dict["slug"], author["slug"] + )) - b = bs4.BeautifulSoup(shout["body"], "html.parser") - texts = [shout["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")] + b = bs4.BeautifulSoup(shout_dict["body"], "html.parser") + texts = [shout_dict["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")] texts = texts + b.findAll(text=True) topics_dataset_bodies.append(" ".join([x.strip().lower() for x in texts])) - topics_dataset_tlist.append(shout["topics"]) + topics_dataset_tlist.append(shout_dict["topics"]) else: ignored += 1 @@ -134,9 +135,7 @@ async def shouts_handle(storage, args): # ', fmt='%s') print("[migration] " + str(counter) + " content items were migrated") - print("[migration] " + str(ignored) + " content items were ignored") print("[migration] " + str(pub_counter) + " have been published") - print("[migration] " + str(discours_author) + " authored by @discours") print("[migration] " + str(anonymous_author) + " authored by @anonymous") diff --git a/migration/tables/comments.py b/migration/tables/comments.py index 5350f00c..4fde9569 100644 --- a/migration/tables/comments.py +++ b/migration/tables/comments.py @@ -8,12 +8,13 @@ from orm.reaction import Reaction, ReactionKind from orm.shout import ShoutReactionsFollower from orm.topic import TopicFollower from orm.user import User +from orm.shout import Shout # from services.stat.reacted import ReactedStorage ts = datetime.now(tz=timezone.utc) -def auto_followers(session, shout_dict, reaction_dict): +def auto_followers(session, topics, reaction_dict): # creating shout's reactions following for reaction author following1 = session.query( ShoutReactionsFollower @@ -30,18 +31,18 @@ def auto_followers(session, shout_dict, reaction_dict): ) session.add(following1) # creating topics followings for reaction author - for t in shout_dict["topics"]: + for t in topics: tf = session.query( TopicFollower ).where( TopicFollower.follower == reaction_dict["createdBy"] ).filter( - TopicFollower.topic == t + TopicFollower.topic == t['id'] ).first() if not tf: topic_following = TopicFollower.create( follower=reaction_dict["createdBy"], - topic=t, + topic=t['id'], auto=True ) session.add(topic_following) @@ -60,7 +61,7 @@ def migrate_ratings(session, entry, reaction_dict): "kind": ReactionKind.LIKE if comment_rating_old["value"] > 0 else ReactionKind.DISLIKE, - "createdBy": rater.slug if rater else "anonymous", + "createdBy": rater.id if rater else 1, } cts = comment_rating_old.get("createdAt") if cts: @@ -108,9 +109,7 @@ async def migrate(entry, storage): "updatedAt": "2020-05-27 19:22:57.091000+00:00", "updatedBy": "0" } - -> - type Reaction { id: Int! shout: Shout! @@ -143,30 +142,41 @@ async def migrate(entry, storage): raise Exception return else: + stage = "started" + reaction = None with local_session() as session: author = session.query(User).filter(User.oid == entry["createdBy"]).first() - shout_dict = storage["shouts"]["by_oid"][shout_oid] - if shout_dict: - reaction_dict["shout"] = shout_dict["slug"] - reaction_dict["createdBy"] = author.slug if author else "discours" - reaction_dict["kind"] = ReactionKind.COMMENT - - # creating reaction from old comment - reaction = Reaction.create(**reaction_dict) - session.add(reaction) - # await ReactedStorage.react(reaction) - - reaction_dict = reaction.dict() - - auto_followers(session, shout_dict, reaction_dict) - - migrate_ratings(session, shout_dict, reaction_dict) + old_shout = storage["shouts"]["by_oid"].get(shout_oid) + if not old_shout: + raise Exception("no old shout in storage") else: - print( - "[migration] error: cannot find shout for comment %r" - % reaction_dict - ) - return reaction + stage = "author and old id found" + try: + shout = session.query( + Shout + ).where(Shout.slug == old_shout["slug"]).one() + if shout: + reaction_dict["shout"] = shout.id + reaction_dict["createdBy"] = author.id if author else 1 + reaction_dict["kind"] = ReactionKind.COMMENT + + # creating reaction from old comment + reaction = Reaction.create(**reaction_dict) + session.add(reaction) + # session.commit() + stage = "new reaction commited" + reaction_dict = reaction.dict() + topics = [t.dict() for t in shout.topics] + auto_followers(session, topics, reaction_dict) + + migrate_ratings(session, entry, reaction_dict) + + return reaction + except Exception as e: + print(e) + print(reaction) + raise Exception(stage) + return def migrate_2stage(old_comment, idmap): diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index 81dd40ff..e1503994 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -8,9 +8,10 @@ from migration.extract import extract_html, extract_media from orm.reaction import Reaction, ReactionKind from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower from orm.user import User -from orm.topic import TopicFollower +from orm.topic import TopicFollower, Topic # from services.stat.reacted import ReactedStorage from services.stat.viewed import ViewedStorage +import re OLD_DATE = "2016-03-05 22:22:00.350000" ts = datetime.now(tz=timezone.utc) @@ -22,6 +23,8 @@ type2layout = { "Image": "image", } +anondict = {"slug": "anonymous", "id": 1, "name": "Аноним"} + def get_shout_slug(entry): slug = entry.get("slug", "") @@ -30,6 +33,7 @@ def get_shout_slug(entry): slug = friend.get("slug", "") if slug: break + slug = re.sub('[^0-9a-zA-Z]+', '-', slug) return slug @@ -40,13 +44,8 @@ def create_author_from_app(app): user = session.query(User).where(User.email == app['email']).first() if not user: name = app.get('name') - slug = ( - translit(name, "ru", reversed=True) - .replace(" ", "-") - .replace("'", "") - .replace(".", "-") - .lower() - ) + slug = translit(name, "ru", reversed=True).lower() + slug = re.sub('[^0-9a-zA-Z]+', '-', slug) # check if nameslug is used user = session.query(User).where(User.slug == slug).first() # get slug from email @@ -82,18 +81,33 @@ def create_author_from_app(app): return userdata -async def create_shout(shout_dict, userslug): +async def create_shout(shout_dict, user): s = Shout.create(**shout_dict) with local_session() as session: srf = session.query(ShoutReactionsFollower).where( - ShoutReactionsFollower.shout == s.slug + ShoutReactionsFollower.shout == s.id ).filter( - ShoutReactionsFollower.follower == userslug + ShoutReactionsFollower.follower == user.id ).first() if not srf: - srf = ShoutReactionsFollower.create(shout=s.slug, follower=userslug, auto=True) + srf = ShoutReactionsFollower.create(shout=s.id, follower=user.id, auto=True) session.add(srf) session.commit() + return s + + +def get_userdata(entry, storage): + user_oid = entry.get("createdBy", "") + userdata = None + app = entry.get("application") + if app: + userdata = create_author_from_app(app) or anondict + else: + userdata = storage["users"]["by_oid"].get(user_oid) or anondict + slug = userdata.get("slug") + slug = re.sub('[^0-9a-zA-Z]+', '-', slug) + userdata["slug"] = slug + return userdata, user_oid def get_userdata(entry, storage): @@ -109,12 +123,12 @@ def get_userdata(entry, storage): async def migrate(entry, storage): - userslug, userdata, user_oid = get_userdata(entry, storage) - user = await get_user(userslug, userdata, storage, user_oid) + userdata, user_oid = get_userdata(entry, storage) + user = await get_user(userdata, storage, user_oid) r = { "layout": type2layout[entry["type"]], "title": entry["title"], - "authors": [userslug, ], + "authors": [userdata["slug"], ], "slug": get_shout_slug(entry), "cover": ( "https://assets.discours.io/unsafe/1600x/" + @@ -125,7 +139,7 @@ async def migrate(entry, storage): "deletedAt": date_parse(entry.get("deletedAt")) if entry.get("deletedAt") else None, "createdAt": date_parse(entry.get("createdAt", OLD_DATE)), "updatedAt": date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts, - "topics": await add_topics_follower(entry, storage, userslug), + "topics": await add_topics_follower(entry, storage, user), "body": extract_html(entry) } @@ -136,7 +150,7 @@ async def migrate(entry, storage): if entry.get("published"): with local_session() as session: # update user.emailConfirmed if published - author = session.query(User).where(User.slug == userslug).first() + author = session.query(User).where(User.slug == userdata["slug"]).first() author.emailConfirmed = True session.add(author) session.commit() @@ -153,13 +167,18 @@ async def migrate(entry, storage): del shout_dict["topics"] try: # save shout to db - await create_shout(shout_dict, userslug) + shout_dict["oid"] = entry.get("_id", "") + shout = await create_shout(shout_dict, user) except IntegrityError as e: - print(e) - await resolve_create_shout(shout_dict, userslug) + print('[migration] create_shout integrity error', e) + shout = await resolve_create_shout(shout_dict, userdata["slug"]) except Exception as e: raise Exception(e) + # udpate data + shout_dict = shout.dict() + shout_dict["authors"] = [user.dict(), ] + # shout topics aftermath shout_dict["topics"] = await topics_aftermath(r, storage) @@ -170,13 +189,12 @@ async def migrate(entry, storage): await ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1)) # del shout_dict['ratings'] - shout_dict["oid"] = entry.get("_id", "") storage["shouts"]["by_oid"][entry["_id"]] = shout_dict storage["shouts"]["by_slug"][shout_dict["slug"]] = shout_dict return shout_dict -async def add_topics_follower(entry, storage, userslug): +async def add_topics_follower(entry, storage, user): topics = set([]) category = entry.get("category") topics_by_oid = storage["topics"]["by_oid"] @@ -188,25 +206,26 @@ async def add_topics_follower(entry, storage, userslug): ttt = list(topics) # add author as TopicFollower with local_session() as session: - for tpc in topics: + for tpcslug in topics: try: + tpc = session.query(Topic).where(Topic.slug == tpcslug).first() tf = session.query( TopicFollower ).where( - TopicFollower.follower == userslug + TopicFollower.follower == user.id ).filter( - TopicFollower.topic == tpc + TopicFollower.topic == tpc.id ).first() if not tf: tf = TopicFollower.create( - topic=tpc, - follower=userslug, + topic=tpc.id, + follower=user.id, auto=True ) session.add(tf) session.commit() except IntegrityError: - print('[migration.shout] hidden by topic ' + tpc) + print('[migration.shout] hidden by topic ' + tpc.slug) # main topic maintopic = storage["replacements"].get(topics_by_oid.get(category, {}).get("slug")) if maintopic in ttt: @@ -215,23 +234,28 @@ async def add_topics_follower(entry, storage, userslug): return ttt -async def get_user(userslug, userdata, storage, oid): +async def get_user(userdata, storage, oid): user = None with local_session() as session: - if not user and userslug: - user = session.query(User).filter(User.slug == userslug).first() - if not user and userdata: + uid = userdata.get("id") + if uid: + user = session.query(User).filter(User.id == uid).first() + elif userdata: try: - userdata["slug"] = userdata["slug"].lower().strip().replace(" ", "-") + slug = userdata["slug"].lower().strip() + slug = re.sub('[^0-9a-zA-Z]+', '-', slug) + userdata["slug"] = slug user = User.create(**userdata) session.add(user) session.commit() except IntegrityError: - print("[migration] user error: " + userdata) - userdata["id"] = user.id - userdata["createdAt"] = user.createdAt - storage["users"]["by_slug"][userdata["slug"]] = userdata - storage["users"]["by_oid"][oid] = userdata + print("[migration] user creating with slug %s" % userdata["slug"]) + print("[migration] from userdata: %r" % userdata) + raise Exception("[migration] cannot create user in content_items.get_user()") + userdata["id"] = user.id + userdata["createdAt"] = user.createdAt + storage["users"]["by_slug"][userdata["slug"]] = userdata + storage["users"]["by_oid"][oid] = userdata if not user: raise Exception("could not get a user") return user @@ -269,6 +293,7 @@ async def resolve_create_shout(shout_dict, userslug): print("[migration] something went wrong with shout: \n%r" % shout_dict) raise Exception("") session.commit() + return s async def topics_aftermath(entry, storage): @@ -276,27 +301,35 @@ async def topics_aftermath(entry, storage): for tpc in filter(lambda x: bool(x), entry["topics"]): oldslug = tpc newslug = storage["replacements"].get(oldslug, oldslug) + if newslug: with local_session() as session: + shout = session.query(Shout).where(Shout.slug == entry["slug"]).one() + new_topic = session.query(Topic).where(Topic.slug == newslug).one() + shout_topic_old = ( session.query(ShoutTopic) - .filter(ShoutTopic.shout == entry["slug"]) - .filter(ShoutTopic.topic == oldslug) + .join(Shout) + .join(Topic) + .filter(Shout.slug == entry["slug"]) + .filter(Topic.slug == oldslug) .first() ) if shout_topic_old: - shout_topic_old.update({"slug": newslug}) + shout_topic_old.update({"topic": new_topic.id}) else: shout_topic_new = ( session.query(ShoutTopic) - .filter(ShoutTopic.shout == entry["slug"]) - .filter(ShoutTopic.topic == newslug) + .join(Shout) + .join(Topic) + .filter(Shout.slug == entry["slug"]) + .filter(Topic.slug == newslug) .first() ) if not shout_topic_new: try: ShoutTopic.create( - **{"shout": entry["slug"], "topic": newslug} + **{"shout": shout.id, "topic": new_topic.id} ) except Exception: print("[migration] shout topic error: " + newslug) @@ -318,14 +351,15 @@ async def content_ratings_to_reactions(entry, slug): .filter(User.oid == content_rating["createdBy"]) .first() ) or User.default_user + shout = session.query(Shout).where(Shout.slug == slug).first() cts = content_rating.get("createdAt") reaction_dict = { "createdAt": date_parse(cts) if cts else None, "kind": ReactionKind.LIKE if content_rating["value"] > 0 else ReactionKind.DISLIKE, - "createdBy": rater.slug, - "shout": slug + "createdBy": rater.id, + "shout": shout.id } reaction = ( session.query(Reaction) diff --git a/migration/tables/topics.py b/migration/tables/topics.py index 15fcf245..3287adb7 100644 --- a/migration/tables/topics.py +++ b/migration/tables/topics.py @@ -9,9 +9,10 @@ def migrate(entry): topic_dict = { "slug": entry["slug"], "oid": entry["_id"], - "title": entry["title"].replace(" ", " ") + "title": entry["title"].replace(" ", " "), + "body": extract_md(html2text(body_orig), entry["_id"]) } - topic_dict["body"] = extract_md(html2text(body_orig), entry["_id"]) + with local_session() as session: slug = topic_dict["slug"] topic = session.query(Topic).filter(Topic.slug == slug).first() or Topic.create( diff --git a/migration/tables/users.py b/migration/tables/users.py index 6db7a243..cb3e87bc 100644 --- a/migration/tables/users.py +++ b/migration/tables/users.py @@ -1,7 +1,7 @@ from dateutil.parser import parse from sqlalchemy.exc import IntegrityError from bs4 import BeautifulSoup - +import re from base.orm import local_session from orm.user import AuthorFollower, User, UserRating @@ -23,17 +23,18 @@ def migrate(entry): "notifications": [], "links": [], "name": "anonymous", + "password": entry["services"]["password"].get("bcrypt") } - user_dict["password"] = entry["services"]["password"].get("bcrypt") + if "updatedAt" in entry: user_dict["updatedAt"] = parse(entry["updatedAt"]) if "wasOnineAt" in entry: user_dict["lastSeen"] = parse(entry["wasOnlineAt"]) if entry.get("profile"): # slug - user_dict["slug"] = ( - entry["profile"].get("path").lower().replace(" ", "-").strip() - ) + slug = entry["profile"].get("path").lower() + slug = re.sub('[^0-9a-zA-Z]+', '-', slug).strip() + user_dict["slug"] = slug bio = BeautifulSoup(entry.get("profile").get("bio") or "", features="lxml").text if bio.startswith('<'): print('[migration] bio! ' + bio) @@ -114,18 +115,23 @@ def migrate_2stage(entry, id_map): continue oid = entry["_id"] author_slug = id_map.get(oid) - user_rating_dict = { - "value": rating_entry["value"], - "rater": rater_slug, - "user": author_slug, - } + with local_session() as session: try: + rater = session.query(User).where(User.slug == rater_slug).one() + user = session.query(User).where(User.slug == author_slug).one() + + user_rating_dict = { + "value": rating_entry["value"], + "raterId": rater.id, + "user": user.id, + } + user_rating = UserRating.create(**user_rating_dict) if user_rating_dict['value'] > 0: af = AuthorFollower.create( - author=user_rating_dict['user'], - follower=user_rating_dict['rater'], + author=user.id, + follower=rater.id, auto=True ) session.add(af) diff --git a/orm/__init__.py b/orm/__init__.py index c8251256..b83e8ad8 100644 --- a/orm/__init__.py +++ b/orm/__init__.py @@ -21,7 +21,7 @@ __all__ = [ "TopicFollower", "Notification", "Reaction", - "UserRating" + "UserRating", "ViewedEntry" ] diff --git a/orm/collab.py b/orm/collab.py index a159f1d7..a568daf4 100644 --- a/orm/collab.py +++ b/orm/collab.py @@ -11,23 +11,16 @@ class CollabAuthor(Base): id = None # type: ignore collab = Column(ForeignKey("collab.id"), primary_key=True) - author = Column(ForeignKey("user.slug"), primary_key=True) - invitedBy = Column(ForeignKey("user.slug")) - - -class CollabInvited(Base): - __tablename__ = "collab_invited" - - id = None # type: ignore - collab = Column(ForeignKey("collab.id"), primary_key=True) - author = Column(ForeignKey("user.slug"), primary_key=True) - invitedBy = Column(ForeignKey("user.slug")) + author = Column(ForeignKey("user.id"), primary_key=True) + accepted = Column(Boolean, default=False) class Collab(Base): __tablename__ = "collab" - shout = Column(ForeignKey("shout.id"), primary_key=True) + title = Column(String, nullable=True, comment="Title") + body = Column(String, nullable=True, comment="Body") + pic = Column(String, nullable=True, comment="Picture") authors = relationship(lambda: User, secondary=CollabAuthor.__tablename__) invites = relationship(lambda: User, secondary=CollabInvited.__tablename__) createdAt = Column(DateTime, default=datetime.now, comment="Created At") diff --git a/orm/collection.py b/orm/collection.py index 8d8b74eb..27dc52e4 100644 --- a/orm/collection.py +++ b/orm/collection.py @@ -9,8 +9,8 @@ class ShoutCollection(Base): __tablename__ = "shout_collection" id = None # type: ignore - shout = Column(ForeignKey("shout.slug"), primary_key=True) - collection = Column(ForeignKey("collection.slug"), primary_key=True) + shout = Column(ForeignKey("shout.id"), primary_key=True) + collectionId = Column(ForeignKey("collection.id"), primary_key=True) class Collection(Base): diff --git a/orm/community.py b/orm/community.py index 8c339714..66ea5891 100644 --- a/orm/community.py +++ b/orm/community.py @@ -8,8 +8,8 @@ class CommunityFollower(Base): __tablename__ = "community_followers" id = None # type: ignore - follower = Column(ForeignKey("user.slug"), primary_key=True) - community = Column(ForeignKey("community.slug"), primary_key=True) + follower = Column(ForeignKey("user.id"), primary_key=True) + communityId = Column(ForeignKey("community.id"), primary_key=True) joinedAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" ) diff --git a/orm/rbac.py b/orm/rbac.py index ad288798..aa49804c 100644 --- a/orm/rbac.py +++ b/orm/rbac.py @@ -122,7 +122,7 @@ class Operation(Base): class Resource(Base): __tablename__ = "resource" - resource_class = Column( + resourceClass = Column( String, nullable=False, unique=True, comment="Resource class" ) name = Column(String, nullable=False, unique=True, comment="Resource name") @@ -134,7 +134,7 @@ class Resource(Base): for res in ["shout", "topic", "reaction", "chat", "message", "invite", "community", "user"]: r = session.query(Resource).filter(Resource.name == res).first() if not r: - r = Resource.create(name=res, resource_class=res) + r = Resource.create(name=res, resourceClass=res) session.add(r) session.commit() @@ -142,19 +142,19 @@ class Resource(Base): class Permission(Base): __tablename__ = "permission" __table_args__ = ( - UniqueConstraint("role_id", "operation_id", "resource_id"), + UniqueConstraint("roleId", "operationId", "resourceId"), {"extend_existing": True}, ) - role_id = Column( + roleId = Column( ForeignKey("role.id", ondelete="CASCADE"), nullable=False, comment="Role" ) - operation_id = Column( + operationId = Column( ForeignKey("operation.id", ondelete="CASCADE"), nullable=False, comment="Operation", ) - resource_id = Column( + resourceId = Column( ForeignKey("resource.id", ondelete="CASCADE"), nullable=False, comment="Resource", @@ -164,11 +164,11 @@ class Permission(Base): if __name__ == "__main__": Base.metadata.create_all(engine) ops = [ - Permission(role_id=1, operation_id=1, resource_id=1), - Permission(role_id=1, operation_id=2, resource_id=1), - Permission(role_id=1, operation_id=3, resource_id=1), - Permission(role_id=1, operation_id=4, resource_id=1), - Permission(role_id=2, operation_id=4, resource_id=1), + Permission(roleId=1, operationId=1, resourceId=1), + Permission(roleId=1, operationId=2, resourceId=1), + Permission(roleId=1, operationId=3, resourceId=1), + Permission(roleId=1, operationId=4, resourceId=1), + Permission(roleId=2, operationId=4, resourceId=1), ] global_session.add_all(ops) global_session.commit() diff --git a/orm/reaction.py b/orm/reaction.py index 123cfb22..f484808a 100644 --- a/orm/reaction.py +++ b/orm/reaction.py @@ -28,12 +28,12 @@ class Reaction(Base): createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" ) - createdBy = Column(ForeignKey("user.slug"), nullable=False, comment="Sender") + createdBy = Column(ForeignKey("user.id"), nullable=False, index=True, comment="Sender") updatedAt = Column(DateTime, nullable=True, comment="Updated at") - updatedBy = Column(ForeignKey("user.slug"), nullable=True, comment="Last Editor") + updatedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Last Editor") deletedAt = Column(DateTime, nullable=True, comment="Deleted at") - deletedBy = Column(ForeignKey("user.slug"), nullable=True, comment="Deleted by") - shout = Column(ForeignKey("shout.slug"), nullable=False) + deletedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Deleted by") + shout = Column(ForeignKey("shout.id"), nullable=False, index=True) replyTo = Column( ForeignKey("reaction.id"), nullable=True, comment="Reply to reaction ID" ) diff --git a/orm/shout.py b/orm/shout.py index f65b1268..55435d5f 100644 --- a/orm/shout.py +++ b/orm/shout.py @@ -13,16 +13,16 @@ class ShoutTopic(Base): __tablename__ = "shout_topic" id = None # type: ignore - shout = Column(ForeignKey("shout.slug"), primary_key=True) - topic = Column(ForeignKey("topic.slug"), primary_key=True) + shout = Column(ForeignKey("shout.id"), primary_key=True, index=True) + topic = Column(ForeignKey("topic.id"), primary_key=True, index=True) class ShoutReactionsFollower(Base): __tablename__ = "shout_reactions_followers" id = None # type: ignore - follower = Column(ForeignKey("user.slug"), primary_key=True) - shout = Column(ForeignKey("shout.slug"), primary_key=True) + follower = Column(ForeignKey("user.id"), primary_key=True, index=True) + shout = Column(ForeignKey("shout.id"), primary_key=True, index=True) auto = Column(Boolean, nullable=False, default=False) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" @@ -34,8 +34,8 @@ class ShoutAuthor(Base): __tablename__ = "shout_author" id = None # type: ignore - shout = Column(ForeignKey("shout.slug"), primary_key=True) - user = Column(ForeignKey("user.slug"), primary_key=True) + shout = Column(ForeignKey("shout.id"), primary_key=True, index=True) + user = Column(ForeignKey("user.id"), primary_key=True, index=True) caption = Column(String, nullable=True, default="") @@ -55,7 +55,7 @@ class Shout(Base): topics = relationship(lambda: Topic, secondary=ShoutTopic.__tablename__) reactions = relationship(lambda: Reaction) visibility = Column(String, nullable=True) # owner authors community public - versionOf = Column(ForeignKey("shout.slug"), nullable=True) + versionOf = Column(ForeignKey("shout.id"), nullable=True) oid = Column(String, nullable=True) media = Column(JSON, nullable=True) diff --git a/orm/topic.py b/orm/topic.py index f4d094b7..a37dc69a 100644 --- a/orm/topic.py +++ b/orm/topic.py @@ -9,8 +9,8 @@ class TopicFollower(Base): __tablename__ = "topic_followers" id = None # type: ignore - follower = Column(ForeignKey("user.slug"), primary_key=True) - topic = Column(ForeignKey("topic.slug"), primary_key=True) + follower = Column(ForeignKey("user.id"), primary_key=True, index=True) + topic = Column(ForeignKey("topic.id"), primary_key=True, index=True) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" ) diff --git a/orm/user.py b/orm/user.py index 5b3b8484..0c236b56 100644 --- a/orm/user.py +++ b/orm/user.py @@ -6,13 +6,12 @@ from sqlalchemy.orm import relationship from base.orm import Base, local_session from orm.rbac import Role -from services.auth.roles import RoleStorage class UserNotifications(Base): __tablename__ = "user_notifications" # id auto - user_id = Column(Integer, ForeignKey("user.id")) + user = Column(Integer, ForeignKey("user.id")) kind = Column(String, ForeignKey("notification.kind")) values = Column(JSONType, nullable=True) # [ , .. ] @@ -21,8 +20,8 @@ class UserRating(Base): __tablename__ = "user_rating" id = None # type: ignore - rater = Column(ForeignKey("user.slug"), primary_key=True) - user = Column(ForeignKey("user.slug"), primary_key=True) + raterId = Column(ForeignKey("user.id"), primary_key=True, index=True) + user = Column(ForeignKey("user.id"), primary_key=True, index=True) value = Column(Integer) @staticmethod @@ -34,16 +33,16 @@ class UserRole(Base): __tablename__ = "user_role" id = None # type: ignore - user_id = Column(ForeignKey("user.id"), primary_key=True) - role_id = Column(ForeignKey("role.id"), primary_key=True) + user = Column(ForeignKey("user.id"), primary_key=True, index=True) + roleId = Column(ForeignKey("role.id"), primary_key=True, index=True) class AuthorFollower(Base): __tablename__ = "author_follower" id = None # type: ignore - follower = Column(ForeignKey("user.slug"), primary_key=True) - author = Column(ForeignKey("user.slug"), primary_key=True) + follower = Column(ForeignKey("user.id"), primary_key=True, index=True) + author = Column(ForeignKey("user.id"), primary_key=True, index=True) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" ) @@ -103,12 +102,12 @@ class User(Base): async def get_permission(self): scope = {} - for user_role in self.roles: - role: Role = await RoleStorage.get_role(user_role.id) # type: ignore + for role in self.roles: for p in role.permissions: - if p.resource_id not in scope: - scope[p.resource_id] = set() - scope[p.resource_id].add(p.operation_id) + if p.resourceId not in scope: + scope[p.resourceId] = set() + scope[p.resourceId].add(p.operationId) + return scope diff --git a/orm/viewed.py b/orm/viewed.py index 6d089234..be8a88e8 100644 --- a/orm/viewed.py +++ b/orm/viewed.py @@ -6,8 +6,8 @@ from base.orm import Base, local_session class ViewedEntry(Base): __tablename__ = "viewed" - viewer = Column(ForeignKey("user.slug"), default='anonymous') - shout = Column(ForeignKey("shout.slug"), default="genesis-block") + viewerId = Column(ForeignKey("user.id"), index=True, default=1) + shout = Column(ForeignKey("shout.id"), index=True, default=1) amount = Column(Integer, default=1) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" diff --git a/resolvers/__init__.py b/resolvers/__init__.py index da8800a2..0a1b096f 100644 --- a/resolvers/__init__.py +++ b/resolvers/__init__.py @@ -15,7 +15,8 @@ from resolvers.create.editor import create_shout, delete_shout, update_shout from resolvers.zine.profile import ( load_authors_by, rate_user, - update_profile + update_profile, + get_authors_all ) from resolvers.zine.reactions import ( diff --git a/resolvers/create/editor.py b/resolvers/create/editor.py index 78f4d4d1..e125d2c2 100644 --- a/resolvers/create/editor.py +++ b/resolvers/create/editor.py @@ -1,13 +1,13 @@ from datetime import datetime, timezone +from sqlalchemy import and_ + from auth.authenticate import login_required from base.orm import local_session from base.resolvers import mutation from orm.rbac import Resource from orm.shout import Shout, ShoutAuthor, ShoutTopic -from orm.collab import Collab -from services.inbox import MessagesStorage -from orm.topic import TopicFollower +from orm.topic import TopicFollower, Topic from orm.user import User from resolvers.zine.reactions import reactions_follow, reactions_unfollow from services.zine.gittask import GitTask @@ -49,7 +49,7 @@ async def create_shout(_, info, inp): session.add(new_collab) # NOTE: shout made by one first author - sa = ShoutAuthor.create(shout=new_shout.slug, user=user.slug) + sa = ShoutAuthor.create(shout=new_shout.id, user=user.id) session.add(sa) reactions_follow(user, new_shout.slug, True) @@ -58,11 +58,16 @@ async def create_shout(_, info, inp): topic_slugs.append(inp["mainTopic"]) for slug in topic_slugs: - st = ShoutTopic.create(shout=new_shout.slug, topic=slug) + topic = session.query(Topic).where(Topic.slug == slug).one() + + st = ShoutTopic.create(shout=new_shout.id, topic=topic.id) session.add(st) - tf = session.query(TopicFollower).where(follower=user.slug, topic=slug) + tf = session.query(TopicFollower).where( + and_(TopicFollower.follower == user.id, TopicFollower.topic == topic.id) + ) + if not tf: - tf = TopicFollower.create(follower=user.slug, topic=slug, auto=True) + tf = TopicFollower.create(follower=user.id, topic=topic.id, auto=True) session.add(tf) new_shout.topic_slugs = topic_slugs @@ -70,7 +75,7 @@ async def create_shout(_, info, inp): session.commit() - GitTask(inp, user.username, user.email, "new shout %s" % (new_shout.slug)) + GitTask(inp, user.username, user.email, "new shout %s" % new_shout.slug) return {"shout": new_shout} @@ -92,7 +97,7 @@ async def update_shout(_, info, inp): if user_id not in authors: scopes = auth.scopes print(scopes) - if Resource.shout_id not in scopes: + if Resource.shout not in scopes: return {"error": "access denied"} else: shout.update(inp) @@ -100,7 +105,7 @@ async def update_shout(_, info, inp): session.add(shout) if inp.get("topics"): # remove old links - links = session.query(ShoutTopic).where(ShoutTopic.shout == slug).all() + links = session.query(ShoutTopic).where(ShoutTopic.shout == shout.id).all() for topiclink in links: session.delete(topiclink) # add new topic links diff --git a/resolvers/inbox/search.py b/resolvers/inbox/search.py index 914af4eb..5a1289ac 100644 --- a/resolvers/inbox/search.py +++ b/resolvers/inbox/search.py @@ -4,7 +4,7 @@ from auth.authenticate import login_required from base.redis import redis from base.resolvers import query from base.orm import local_session -from orm.user import AuthorFollower +from orm.user import AuthorFollower, User @query.field("searchRecipients") @@ -30,13 +30,19 @@ async def search_recipients(_, info, query: str, limit: int = 50, offset: int = with local_session() as session: # followings - result += session.query(AuthorFollower.author).where(AuthorFollower.follower.startswith(query))\ - .offset(offset + len(result)).limit(more_amount) + result += session.query(AuthorFollower.author).join( + User, User.id == AuthorFollower.follower + ).where( + User.slug.startswith(query) + ).offset(offset + len(result)).limit(more_amount) more_amount = limit # followers - result += session.query(AuthorFollower.follower).where(AuthorFollower.author.startswith(query))\ - .offset(offset + len(result)).limit(offset + len(result) + limit) + result += session.query(AuthorFollower.follower).join( + User, User.id == AuthorFollower.author + ).where( + User.slug.startswith(query) + ).offset(offset + len(result)).limit(offset + len(result) + limit) return { "members": list(result), "error": None diff --git a/resolvers/zine/_common.py b/resolvers/zine/_common.py new file mode 100644 index 00000000..7046be92 --- /dev/null +++ b/resolvers/zine/_common.py @@ -0,0 +1,31 @@ +from sqlalchemy import func, case +from sqlalchemy.orm import aliased +from orm.reaction import Reaction, ReactionKind + + +def add_common_stat_columns(q): + aliased_reaction = aliased(Reaction) + + q = q.outerjoin(aliased_reaction).add_columns( + func.sum( + aliased_reaction.id + ).label('reacted_stat'), + func.sum( + case( + (aliased_reaction.body.is_not(None), 1), + else_=0 + ) + ).label('commented_stat'), + func.sum(case( + (aliased_reaction.kind == ReactionKind.AGREE, 1), + (aliased_reaction.kind == ReactionKind.DISAGREE, -1), + (aliased_reaction.kind == ReactionKind.PROOF, 1), + (aliased_reaction.kind == ReactionKind.DISPROOF, -1), + (aliased_reaction.kind == ReactionKind.ACCEPT, 1), + (aliased_reaction.kind == ReactionKind.REJECT, -1), + (aliased_reaction.kind == ReactionKind.LIKE, 1), + (aliased_reaction.kind == ReactionKind.DISLIKE, -1), + else_=0) + ).label('rating_stat')) + + return q diff --git a/resolvers/zine/load.py b/resolvers/zine/load.py index 53b1841a..63896348 100644 --- a/resolvers/zine/load.py +++ b/resolvers/zine/load.py @@ -1,51 +1,31 @@ from datetime import datetime, timedelta, timezone -import sqlalchemy as sa from sqlalchemy.orm import joinedload, aliased -from sqlalchemy.sql.expression import desc, asc, select, case +from sqlalchemy.sql.expression import desc, asc, select, func from base.orm import local_session from base.resolvers import query from orm import ViewedEntry -from orm.shout import Shout -from orm.reaction import Reaction, ReactionKind -from services.zine.shoutauthor import ShoutAuthorStorage -from services.stat.viewed import ViewedStorage +from orm.shout import Shout, ShoutAuthor +from orm.reaction import Reaction +from resolvers.zine._common import add_common_stat_columns -def calc_reactions(q): - aliased_reaction = aliased(Reaction) - return q.join(aliased_reaction).add_columns( - sa.func.sum(case( - (aliased_reaction.kind == ReactionKind.AGREE, 1), - (aliased_reaction.kind == ReactionKind.DISAGREE, -1), - (aliased_reaction.kind == ReactionKind.PROOF, 1), - (aliased_reaction.kind == ReactionKind.DISPROOF, -1), - (aliased_reaction.kind == ReactionKind.ACCEPT, 1), - (aliased_reaction.kind == ReactionKind.REJECT, -1), - (aliased_reaction.kind == ReactionKind.LIKE, 1), - (aliased_reaction.kind == ReactionKind.DISLIKE, -1), - else_=0) - ).label('rating'), - sa.func.sum( - case( - (aliased_reaction.body.is_not(None), 1), - else_=0 - ) - ).label('commented'), - sa.func.sum( - aliased_reaction.id - ).label('reacted') - ) +def add_stat_columns(q): + q = q.outerjoin(ViewedEntry).add_columns(func.sum(ViewedEntry.amount).label('viewed_stat')) + + return add_common_stat_columns(q) def apply_filters(q, filters, user=None): - filters = {} if filters is None else filters + if filters.get("reacted") and user: - q.join(Reaction, Reaction.createdBy == user.slug) + q.join(Reaction, Reaction.createdBy == user.id) + v = filters.get("visibility") if v == "public": q = q.filter(Shout.visibility == filters.get("visibility")) if v == "community": q = q.filter(Shout.visibility.in_(["public", "community"])) + if filters.get("layout"): q = q.filter(Shout.layout == filters.get("layout")) if filters.get("author"): @@ -59,6 +39,7 @@ def apply_filters(q, filters, user=None): if filters.get("days"): before = datetime.now(tz=timezone.utc) - timedelta(days=int(filters.get("days")) or 30) q = q.filter(Shout.createdAt > before) + return q @@ -69,24 +50,27 @@ async def load_shout(_, info, slug): joinedload(Shout.authors), joinedload(Shout.topics), ) - q = calc_reactions(q) + q = add_stat_columns(q) q = q.filter( Shout.slug == slug ).filter( Shout.deletedAt.is_(None) ).group_by(Shout.id) - [shout, rating, commented, reacted] = session.execute(q).unique().one() - for a in shout.authors: - a.caption = await ShoutAuthorStorage.get_author_caption(shout.slug, a.slug) - viewed = await ViewedStorage.get_shout(shout.slug) + [shout, viewed_stat, reacted_stat, commented_stat, rating_stat] = session.execute(q).unique().one() + shout.stat = { - "rating": rating, - "viewed": viewed, - "commented": commented, - "reacted": reacted + "viewed": viewed_stat, + "reacted": reacted_stat, + "commented": commented_stat, + "rating": rating_stat } + for author_caption in session.query(ShoutAuthor).join(Shout).where(Shout.slug == slug): + for author in shout.authors: + if author.id == author_caption.user: + author.caption = author_caption.caption + return shout @@ -105,7 +89,7 @@ async def load_shouts_by(_, info, options): } offset: 0 limit: 50 - order_by: 'createdAt' + order_by: 'createdAt' | 'commented' | 'reacted' | 'rating' order_by_desc: true } @@ -118,47 +102,36 @@ async def load_shouts_by(_, info, options): ).where( Shout.deletedAt.is_(None) ) + + q = add_stat_columns(q) + user = info.context["request"].user - q = apply_filters(q, options.get("filters"), user) - q = calc_reactions(q) + q = apply_filters(q, options.get("filters", {}), user) - o = options.get("order_by") - if o: - if o == 'comments': - q = q.add_columns(sa.func.count(Reaction.id).label(o)) - q = q.join(Reaction, Shout.slug == Reaction.shout) - q = q.filter(Reaction.body.is_not(None)) - elif o == 'reacted': - q = q.join( - Reaction - ).add_columns( - sa.func.max(Reaction.createdAt).label(o) - ) - elif o == 'views': - q = q.join(ViewedEntry) - q = q.add_columns(sa.func.sum(ViewedEntry.amount).label(o)) - order_by = o - else: - order_by = Shout.createdAt + order_by = options.get("order_by", Shout.createdAt) + if order_by == 'reacted': + aliased_reaction = aliased(Reaction) + q.outerjoin(aliased_reaction).add_columns(func.max(aliased_reaction.createdAt).label('reacted')) - order_by_desc = True if options.get('order_by_desc') is None else options.get('order_by_desc') + order_by_desc = options.get('order_by_desc', True) query_order_by = desc(order_by) if order_by_desc else asc(order_by) offset = options.get("offset", 0) limit = options.get("limit", 10) + q = q.group_by(Shout.id).order_by(query_order_by).limit(limit).offset(offset) - shouts = [] with local_session() as session: - for [shout, rating, commented, reacted] in session.execute(q).unique(): - shout.stat = { - "rating": rating, - "viewed": await ViewedStorage.get_shout(shout.slug), - "commented": commented, - "reacted": reacted - } - # NOTE: no need authors captions in arrays - # for author in shout.authors: - # author.caption = await ShoutAuthorStorage.get_author_caption(shout.slug, author.slug) + shouts = [] + + for [shout, viewed_stat, reacted_stat, commented_stat, rating_stat] in session.execute(q).unique(): shouts.append(shout) + + shout.stat = { + "viewed": viewed_stat, + "reacted": reacted_stat, + "commented": commented_stat, + "rating": rating_stat + } + return shouts diff --git a/resolvers/zine/profile.py b/resolvers/zine/profile.py index a133c2af..652a602a 100644 --- a/resolvers/zine/profile.py +++ b/resolvers/zine/profile.py @@ -1,25 +1,82 @@ from typing import List from datetime import datetime, timedelta, timezone -from sqlalchemy import and_, func -from sqlalchemy.orm import selectinload +from sqlalchemy import and_, func, distinct, select, literal +from sqlalchemy.orm import aliased, joinedload from auth.authenticate import login_required from base.orm import local_session from base.resolvers import mutation, query from orm.reaction import Reaction -from orm.shout import ShoutAuthor -from orm.topic import Topic, TopicFollower +from orm.shout import ShoutAuthor, ShoutTopic +from orm.topic import Topic from orm.user import AuthorFollower, Role, User, UserRating, UserRole -from services.stat.topicstat import TopicStat # from .community import followed_communities from resolvers.inbox.unread import get_total_unread_counter -from .topics import get_topic_stat +from resolvers.zine.topics import followed_by_user + + +def add_author_stat_columns(q): + author_followers = aliased(AuthorFollower) + author_following = aliased(AuthorFollower) + shout_author_aliased = aliased(ShoutAuthor) + user_rating_aliased = aliased(UserRating) + + q = q.outerjoin(shout_author_aliased).add_columns( + func.count(distinct(shout_author_aliased.shout)).label('shouts_stat') + ) + q = q.outerjoin(author_followers, author_followers.author == User.id).add_columns( + func.count(distinct(author_followers.follower)).label('followers_stat') + ) + + q = q.outerjoin(author_following, author_following.follower == User.id).add_columns( + func.count(distinct(author_following.author)).label('followings_stat') + ) + + q = q.add_columns(literal(0).label('rating_stat')) + # FIXME + # q = q.outerjoin(user_rating_aliased, user_rating_aliased.user == User.id).add_columns( + # # TODO: check + # func.sum(user_rating_aliased.value).label('rating_stat') + # ) + + q = q.add_columns(literal(0).label('commented_stat')) + # FIXME + # q = q.outerjoin(Reaction, and_(Reaction.createdBy == User.id, Reaction.body.is_not(None))).add_columns( + # func.count(distinct(Reaction.id)).label('commented_stat') + # ) + + q = q.group_by(User.id) + + return q + + +def add_stat(author, stat_columns): + [shouts_stat, followers_stat, followings_stat, rating_stat, commented_stat] = stat_columns + author.stat = { + "shouts": shouts_stat, + "followers": followers_stat, + "followings": followings_stat, + "rating": rating_stat, + "commented": commented_stat + } + + return author + + +def get_authors_from_query(q): + authors = [] + with local_session() as session: + for [author, *stat_columns] in session.execute(q): + author = add_stat(author, stat_columns) + authors.append(author) + + return authors async def user_subscriptions(slug: str): return { - "unread": await get_total_unread_counter(slug), # unread inbox messages counter + "unread": await get_total_unread_counter(slug), # unread inbox messages counter "topics": [t.slug for t in await followed_topics(slug)], # followed topics slugs "authors": [a.slug for a in await followed_authors(slug)], # followed authors slugs "reactions": await followed_reactions(slug) @@ -27,23 +84,6 @@ async def user_subscriptions(slug: str): } -async def get_author_stat(slug): - with local_session() as session: - return { - "shouts": session.query(ShoutAuthor).where(ShoutAuthor.user == slug).count(), - "followers": session.query(AuthorFollower).where(AuthorFollower.author == slug).count(), - "followings": session.query(AuthorFollower).where(AuthorFollower.follower == slug).count(), - "rating": session.query(func.sum(UserRating.value)).where(UserRating.user == slug).first(), - "commented": session.query( - Reaction.id - ).where( - Reaction.createdBy == slug - ).filter( - Reaction.body.is_not(None) - ).count() - } - - # @query.field("userFollowedDiscussions") @login_required async def followed_discussions(_, info, slug) -> List[Topic]: @@ -56,7 +96,7 @@ async def followed_reactions(slug): return session.query( Reaction.shout ).where( - Reaction.createdBy == slug + Reaction.createdBy == user.id ).filter( Reaction.createdAt > user.lastSeen ).all() @@ -69,17 +109,7 @@ async def get_followed_topics(_, info, slug) -> List[Topic]: async def followed_topics(slug): - topics = [] - with local_session() as session: - topics = ( - session.query(Topic) - .join(TopicFollower) - .where(TopicFollower.follower == slug) - .all() - ) - for topic in topics: - topic.stat = await get_topic_stat(topic.slug) - return topics + return followed_by_user(slug) @query.field("userFollowedAuthors") @@ -88,29 +118,26 @@ async def get_followed_authors(_, _info, slug) -> List[User]: async def followed_authors(slug) -> List[User]: - authors = [] - with local_session() as session: - authors = ( - session.query(User) - .join(AuthorFollower, User.slug == AuthorFollower.author) - .where(AuthorFollower.follower == slug) - .all() - ) - for author in authors: - author.stat = await get_author_stat(author.slug) - return authors + q = select(User) + q = add_author_stat_columns(q) + q = q.join(AuthorFollower).join(User, User.id == AuthorFollower.follower).where(User.slug == slug) + + return get_authors_from_query(q) @query.field("userFollowers") async def user_followers(_, _info, slug) -> List[User]: - with local_session() as session: - users = ( - session.query(User) - .join(AuthorFollower, User.slug == AuthorFollower.follower) - .where(AuthorFollower.author == slug) - .all() - ) - return users + q = select(User) + q = add_author_stat_columns(q) + + aliased_user = aliased(User) + q = q.join(AuthorFollower).join( + aliased_user, aliased_user.id == AuthorFollower.author + ).where( + aliased_user.slug == slug + ) + + return get_authors_from_query(q) async def get_user_roles(slug): @@ -118,11 +145,12 @@ async def get_user_roles(slug): user = session.query(User).where(User.slug == slug).first() roles = ( session.query(Role) - .options(selectinload(Role.permissions)) - .join(UserRole) - .where(UserRole.user_id == user.id) - .all() + .options(joinedload(Role.permissions)) + .join(UserRole) + .where(UserRole.user == user.id) + .all() ) + return roles @@ -147,8 +175,8 @@ async def rate_user(_, info, rated_userslug, value): with local_session() as session: rating = ( session.query(UserRating) - .filter(and_(UserRating.rater == user.slug, UserRating.user == rated_userslug)) - .first() + .filter(and_(UserRating.rater == user.slug, UserRating.user == rated_userslug)) + .first() ) if rating: rating.value = value @@ -164,7 +192,8 @@ async def rate_user(_, info, rated_userslug, value): # for mutation.field("follow") def author_follow(user, slug): with local_session() as session: - af = AuthorFollower.create(follower=user.slug, author=slug) + author = session.query(User).where(User.slug == slug).one() + af = AuthorFollower.create(follower=user.id, author=author.id) session.add(af) session.commit() @@ -173,13 +202,13 @@ def author_follow(user, slug): def author_unfollow(user, slug): with local_session() as session: flw = ( - session.query(AuthorFollower) - .filter( + session.query( + AuthorFollower + ).join(User, User.id == AuthorFollower.author).filter( and_( - AuthorFollower.follower == user.slug, AuthorFollower.author == slug + AuthorFollower.follower == user.id, User.slug == slug ) - ) - .first() + ).first() ) if not flw: raise Exception("[resolvers.profile] follower not exist, cant unfollow") @@ -190,49 +219,41 @@ def author_unfollow(user, slug): @query.field("authorsAll") async def get_authors_all(_, _info): - with local_session() as session: - authors = session.query(User).join(ShoutAuthor).all() - for author in authors: - author.stat = await get_author_stat(author.slug) - return authors + q = select(User) + q = add_author_stat_columns(q) + q = q.join(ShoutAuthor, User.id == ShoutAuthor.user) + + return get_authors_from_query(q) @query.field("getAuthor") async def get_author(_, _info, slug): - with local_session() as session: - author = session.query(User).where(User.slug == slug).first() - author.stat = await get_author_stat(author.slug) - return author + q = select(User).where(User.slug == slug) + q = add_author_stat_columns(q) + + authors = get_authors_from_query(q) + return authors[0] @query.field("loadAuthorsBy") async def load_authors_by(_, info, by, limit, offset): - authors = [] - with local_session() as session: - aq = session.query(User) - if by.get("slug"): - aq = aq.filter(User.slug.ilike(f"%{by['slug']}%")) - elif by.get("name"): - aq = aq.filter(User.name.ilike(f"%{by['name']}%")) - elif by.get("topic"): - aaa = list(map(lambda a: a.slug, TopicStat.authors_by_topic.get(by["topic"]))) - aq = aq.filter(User.name._in(aaa)) - if by.get("lastSeen"): # in days - days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["lastSeen"]) - aq = aq.filter(User.lastSeen > days_before) - elif by.get("createdAt"): # in days - days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["createdAt"]) - aq = aq.filter(User.createdAt > days_before) - aq = aq.group_by( - User.id - ).order_by( - by.get("order") or "createdAt" - ).limit(limit).offset(offset) - print(aq) - authors = list(map(lambda r: r.User, session.execute(aq))) - if by.get("stat"): - for a in authors: - a.stat = await get_author_stat(a.slug) - authors = list(set(authors)) - # authors = sorted(authors, key=lambda a: a["stat"].get(by.get("stat"))) - return authors + q = select(User) + q = add_author_stat_columns(q) + if by.get("slug"): + q = q.filter(User.slug.ilike(f"%{by['slug']}%")) + elif by.get("name"): + q = q.filter(User.name.ilike(f"%{by['name']}%")) + elif by.get("topic"): + q = q.join(ShoutAuthor).join(ShoutTopic).join(Topic).where(Topic.slug == by["topic"]) + if by.get("lastSeen"): # in days + days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["lastSeen"]) + q = q.filter(User.lastSeen > days_before) + elif by.get("createdAt"): # in days + days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["createdAt"]) + q = q.filter(User.createdAt > days_before) + + q = q.order_by( + by.get("order", User.createdAt) + ).limit(limit).offset(offset) + + return get_authors_from_query(q) diff --git a/resolvers/zine/reactions.py b/resolvers/zine/reactions.py index b77d4e5e..c0648fcd 100644 --- a/resolvers/zine/reactions.py +++ b/resolvers/zine/reactions.py @@ -1,29 +1,34 @@ from datetime import datetime, timedelta, timezone from sqlalchemy import and_, asc, desc, select, text, func from sqlalchemy.orm import aliased - from auth.authenticate import login_required from base.orm import local_session from base.resolvers import mutation, query from orm.reaction import Reaction, ReactionKind from orm.shout import Shout, ShoutReactionsFollower from orm.user import User -# from services.stat.reacted import ReactedStorage -from resolvers.zine.load import calc_reactions +from resolvers.zine._common import add_common_stat_columns + + +def add_reaction_stat_columns(q): + return add_common_stat_columns(q) def reactions_follow(user: User, slug: str, auto=False): with local_session() as session: + shout = session.query(Shout).where(Shout.slug == slug).one() + following = ( session.query(ShoutReactionsFollower).where(and_( - ShoutReactionsFollower.follower == user.slug, - ShoutReactionsFollower.shout == slug + ShoutReactionsFollower.follower == user.id, + ShoutReactionsFollower.shout == shout.id, )).first() ) + if not following: following = ShoutReactionsFollower.create( - follower=user.slug, - shout=slug, + follower=user.id, + shout=shout.id, auto=auto ) session.add(following) @@ -32,12 +37,15 @@ def reactions_follow(user: User, slug: str, auto=False): def reactions_unfollow(user, slug): with local_session() as session: + shout = session.query(Shout).where(Shout.slug == slug).one() + following = ( session.query(ShoutReactionsFollower).where(and_( - ShoutReactionsFollower.follower == user.slug, - ShoutReactionsFollower.shout == slug + ShoutReactionsFollower.follower == user.id, + ShoutReactionsFollower.shout == shout.id )).first() ) + if following: session.delete(following) session.commit() @@ -134,7 +142,6 @@ async def create_reaction(_, info, inp): elif check_to_publish(session, user, reaction): set_published(session, reaction.shout, reaction.createdBy) - # ReactedStorage.react(reaction) try: reactions_follow(user, inp["shout"], True) except Exception as e: @@ -157,9 +164,9 @@ async def update_reaction(_, info, inp): with local_session() as session: user = session.query(User).where(User.id == user_id).first() q = select(Reaction).filter(Reaction.id == inp.id) - q = calc_reactions(q) + q = add_reaction_stat_columns(q) - [reaction, rating, commented, reacted] = session.execute(q).unique().one() + [reaction, reacted_stat, commented_stat, rating_stat] = session.execute(q).unique().one() if not reaction: return {"error": "invalid reaction id"} @@ -175,9 +182,9 @@ async def update_reaction(_, info, inp): reaction.range = inp.get("range") session.commit() reaction.stat = { - "commented": commented, - "reacted": reacted, - "rating": rating + "commented": commented_stat, + "reacted": reacted_stat, + "rating": rating_stat } return {"reaction": reaction} @@ -199,6 +206,7 @@ async def delete_reaction(_, info, rid): session.commit() return {} + @query.field("loadReactionsBy") async def load_reactions_by(_, _info, by, limit=50, offset=0): """ @@ -222,28 +230,33 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0): q = select( Reaction, CreatedByUser, ReactedShout ).join( - CreatedByUser, Reaction.createdBy == CreatedByUser.slug + CreatedByUser, Reaction.createdBy == CreatedByUser.id ).join( - ReactedShout, Reaction.shout == ReactedShout.slug + ReactedShout, Reaction.shout == ReactedShout.id ) if by.get("shout"): - q = q.filter(Reaction.shout == by["shout"]) + aliased_shout = aliased(Shout) + q = q.join(aliased_shout).filter(aliased_shout.slug == by["shout"]) elif by.get("shouts"): - q = q.filter(Reaction.shout.in_(by["shouts"])) + aliased_shout = aliased(Shout) + q = q.join(aliased_shout).filter(aliased_shout.shout.in_(by["shouts"])) if by.get("createdBy"): - q = q.filter(Reaction.createdBy == by.get("createdBy")) + aliased_user = aliased(User) + q = q.join(aliased_user).filter(aliased_user.slug == by.get("createdBy")) if by.get("topic"): + # TODO: check q = q.filter(Shout.topics.contains(by["topic"])) if by.get("comment"): q = q.filter(func.length(Reaction.body) > 0) - if by.get('search', 0) > 2: + if len(by.get('search', '')) > 2: q = q.filter(Reaction.body.ilike(f'%{by["body"]}%')) if by.get("days"): after = datetime.now(tz=timezone.utc) - timedelta(days=int(by["days"]) or 30) q = q.filter(Reaction.createdAt > after) order_way = asc if by.get("sort", "").startswith("-") else desc + # replace "-" -> "" ? order_field = by.get("sort") or Reaction.createdAt q = q.group_by( @@ -252,23 +265,24 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0): order_way(order_field) ) - q = calc_reactions(q) + q = add_reaction_stat_columns(q) q = q.where(Reaction.deletedAt.is_(None)) q = q.limit(limit).offset(offset) reactions = [] with local_session() as session: - for [reaction, user, shout, rating, commented, reacted] in session.execute(q): + for [reaction, user, shout, reacted_stat, commented_stat, rating_stat] in session.execute(q): reaction.createdBy = user reaction.shout = shout reaction.stat = { - "rating": rating, - "commented": commented, - "reacted": reacted + "rating": rating_stat, + "commented": commented_stat, + "reacted": reacted_stat } reactions.append(reaction) + # ? if by.get("stat"): reactions.sort(lambda r: r.stat.get(by["stat"]) or r.createdAt) diff --git a/resolvers/zine/topics.py b/resolvers/zine/topics.py index 474fc211..2ee6809b 100644 --- a/resolvers/zine/topics.py +++ b/resolvers/zine/topics.py @@ -1,56 +1,91 @@ -from sqlalchemy import and_ +from sqlalchemy import and_, select, distinct, func from auth.authenticate import login_required from base.orm import local_session from base.resolvers import mutation, query +from orm.shout import ShoutTopic, ShoutAuthor from orm.topic import Topic, TopicFollower -from services.zine.topics import TopicStorage -from services.stat.topicstat import TopicStat +from orm import Shout, User -# from services.stat.viewed import ViewedStorage +def add_topic_stat_columns(q): + q = q.outerjoin(ShoutTopic, Topic.id == ShoutTopic.topic).add_columns( + func.count(distinct(ShoutTopic.shout)).label('shouts_stat') + ).outerjoin(ShoutAuthor, ShoutTopic.shout == ShoutAuthor.shout).add_columns( + func.count(distinct(ShoutAuthor.user)).label('authors_stat') + ).outerjoin(TopicFollower, + and_( + TopicFollower.topic == Topic.id, + TopicFollower.follower == ShoutAuthor.id + )).add_columns( + func.count(distinct(TopicFollower.follower)).label('followers_stat') + ) + + q = q.group_by(Topic.id) + + return q -async def get_topic_stat(slug): - return { - "shouts": len(TopicStat.shouts_by_topic.get(slug, {}).keys()), - "authors": len(TopicStat.authors_by_topic.get(slug, {}).keys()), - "followers": len(TopicStat.followers_by_topic.get(slug, {}).keys()) +def add_stat(topic, stat_columns): + [shouts_stat, authors_stat, followers_stat] = stat_columns + topic.stat = { + "shouts": shouts_stat, + "authors": authors_stat, + "followers": followers_stat } + return topic + + +def get_topics_from_query(q): + topics = [] + with local_session() as session: + for [topic, *stat_columns] in session.execute(q): + topic = add_stat(topic, stat_columns) + topics.append(topic) + + return topics + + +def followed_by_user(user_slug): + q = select(Topic) + q = add_topic_stat_columns(q) + q = q.join(User).where(User.slug == user_slug) + + return get_topics_from_query(q) + @query.field("topicsAll") async def topics_all(_, _info): - topics = await TopicStorage.get_topics_all() - for topic in topics: - topic.stat = await get_topic_stat(topic.slug) - return topics + q = select(Topic) + q = add_topic_stat_columns(q) + + return get_topics_from_query(q) @query.field("topicsByCommunity") async def topics_by_community(_, info, community): - topics = await TopicStorage.get_topics_by_community(community) - for topic in topics: - topic.stat = await get_topic_stat(topic.slug) - return topics + q = select(Topic).where(Topic.community == community) + q = add_topic_stat_columns(q) + + return get_topics_from_query(q) @query.field("topicsByAuthor") async def topics_by_author(_, _info, author): - shouts = TopicStorage.get_topics_by_author(author) - author_topics = set() - for s in shouts: - for tpc in s.topics: - tpc = await TopicStorage.topics[tpc.slug] - tpc.stat = await get_topic_stat(tpc.slug) - author_topics.add(tpc) - return list(author_topics) + q = select(Topic) + q = add_topic_stat_columns(q) + q = q.join(User).where(User.slug == author) + + return get_topics_from_query(q) @query.field("getTopic") async def get_topic(_, _info, slug): - t = TopicStorage.topics[slug] - t.stat = await get_topic_stat(slug) - return t + q = select(Topic).where(Topic.slug == slug) + q = add_topic_stat_columns(q) + + topics = get_topics_from_query(q) + return topics[0] @mutation.field("createTopic") @@ -61,7 +96,7 @@ async def create_topic(_, _info, inp): new_topic = Topic.create(**inp) session.add(new_topic) session.commit() - await TopicStorage.update_topic(new_topic) + return {"topic": new_topic} @@ -76,25 +111,26 @@ async def update_topic(_, _info, inp): else: topic.update(**inp) session.commit() - await TopicStorage.update_topic(topic.slug) + return {"topic": topic} async def topic_follow(user, slug): with local_session() as session: - following = TopicFollower.create(topic=slug, follower=user.slug) + topic = session.query(Topic).where(Topic.slug == slug).one() + + following = TopicFollower.create(topic=topic.id, follower=user.id) session.add(following) session.commit() - await TopicStorage.update_topic(slug) async def topic_unfollow(user, slug): with local_session() as session: sub = ( - session.query(TopicFollower).filter( + session.query(TopicFollower).join(Topic).filter( and_( - TopicFollower.follower == user.slug, - TopicFollower.topic == slug + TopicFollower.follower == user.id, + Topic.slug == slug ) ).first() ) @@ -103,9 +139,13 @@ async def topic_unfollow(user, slug): else: session.delete(sub) session.commit() - await TopicStorage.update_topic(slug) @query.field("topicsRandom") async def topics_random(_, info, amount=12): - return TopicStorage.get_random_topics(amount) + q = select(Topic) + q = add_topic_stat_columns(q) + q = q.join(Shout, ShoutTopic.shout == Shout.id).group_by(Topic.id).having(func.count(Shout.id) > 2) + q = q.order_by(func.random()).limit(amount) + + return get_topics_from_query(q) diff --git a/schema.graphql b/schema.graphql index 3bc88d19..d4f3b20c 100644 --- a/schema.graphql +++ b/schema.graphql @@ -321,8 +321,8 @@ type Operation { } type Permission { - operation_id: Int! - resource_id: Int! + operationId: Int! + resourceId: Int! } type Role { @@ -373,6 +373,23 @@ type User { oid: String } +<<<<<<< HEAD +======= +type Draft { + title: String + body: String + createdBy: Int +} + +type Collab { + authors: [String]! + invites: [String] + createdAt: DateTime! + title: String + body: String +} + +>>>>>>> migation-fix2 enum ReactionKind { LIKE DISLIKE diff --git a/services/auth/roles.py b/services/auth/roles.py deleted file mode 100644 index b5924dc5..00000000 --- a/services/auth/roles.py +++ /dev/null @@ -1,35 +0,0 @@ -import asyncio - -from sqlalchemy.orm import selectinload - -from orm.rbac import Role - - -class RoleStorage: - roles = {} - lock = asyncio.Lock() - - @staticmethod - def init(session): - self = RoleStorage - roles = session.query(Role).options(selectinload(Role.permissions)).all() - self.roles = dict([(role.id, role) for role in roles]) - print("[auth.roles] %d precached" % len(roles)) - - @staticmethod - async def get_role(id): - self = RoleStorage - async with self.lock: - return self.roles.get(id) - - @staticmethod - async def add_role(role): - self = RoleStorage - async with self.lock: - self.roles[id] = role - - @staticmethod - async def del_role(id): - self = RoleStorage - async with self.lock: - del self.roles[id] diff --git a/services/auth/users.py b/services/auth/users.py deleted file mode 100644 index 2ba636bc..00000000 --- a/services/auth/users.py +++ /dev/null @@ -1,72 +0,0 @@ -import asyncio -from sqlalchemy.orm import selectinload, exc -from orm.user import User -from base.orm import local_session - - -class UserStorage: - users = {} - lock = asyncio.Lock() - - @staticmethod - def init(session): - self = UserStorage - users = ( - session.query(User) - .options(selectinload(User.roles), selectinload(User.ratings)) - .all() - ) - self.users = dict([(user.id, user) for user in users]) - print("[auth.users] %d precached" % len(self.users)) - - @staticmethod - async def get_user(id): - with local_session() as session: - try: - user = ( - session.query(User).options( - selectinload(User.roles), - selectinload(User.ratings) - ).filter( - User.id == id - ).one() - ) - return user - except exc.NoResultFound: - return None - - @staticmethod - async def get_all_users(): - self = UserStorage - async with self.lock: - aaa = list(self.users.values()) - aaa.sort(key=lambda user: user.createdAt) - return aaa - - @staticmethod - async def get_top_users(): - self = UserStorage - async with self.lock: - aaa = list(self.users.values()) - aaa.sort(key=lambda user: user.rating) - return aaa - - @staticmethod - async def get_user_by_slug(slug): - self = UserStorage - async with self.lock: - for user in self.users.values(): - if user.slug == slug: - return user - - @staticmethod - async def add_user(user): - self = UserStorage - async with self.lock: - self.users[user.id] = user - - @staticmethod - async def del_user(id): - self = UserStorage - async with self.lock: - del self.users[id] diff --git a/services/main.py b/services/main.py index 1099e90e..10301b86 100644 --- a/services/main.py +++ b/services/main.py @@ -1,7 +1,3 @@ -# from services.stat.reacted import ReactedStorage -from services.auth.roles import RoleStorage -from services.auth.users import UserStorage -from services.zine.topics import TopicStorage from services.search import SearchService from services.stat.viewed import ViewedStorage from base.orm import local_session @@ -9,11 +5,9 @@ from base.orm import local_session async def storages_init(): with local_session() as session: - print('[main] initialize storages') - # ReactedStorage.init(session) - RoleStorage.init(session) - UserStorage.init(session) - TopicStorage.init(session) + print('[main] initialize SearchService') await SearchService.init(session) - session.commit() + print('[main] SearchService initialized') + print('[main] initialize storages') await ViewedStorage.init() + print('[main] storages initialized') diff --git a/services/stat/topicstat.py b/services/stat/topicstat.py deleted file mode 100644 index 05a61388..00000000 --- a/services/stat/topicstat.py +++ /dev/null @@ -1,72 +0,0 @@ -import asyncio -import time -from base.orm import local_session -from orm.shout import Shout, ShoutTopic, ShoutAuthor -from orm.topic import TopicFollower -# from sqlalchemy.sql.expression import select - - -class TopicStat: - # by slugs - shouts_by_topic = {} # Shout object stored - authors_by_topic = {} # User - followers_by_topic = {} # User - # - lock = asyncio.Lock() - period = 30 * 60 # sec - - @staticmethod - async def load_stat(session): - print("[stat.topics] ⎧ loading stat -------") - ts = time.time() - self = TopicStat - shout_topics = session.query(ShoutTopic, Shout).join(Shout).all() # ~ 10 secs - print("[stat.topics] ⎪ shout topics joined query took %fs " % (time.time() - ts)) - print("[stat.topics] ⎪ indexing %d links..." % len(shout_topics)) - for [shout_topic, shout] in shout_topics: - tpc = shout_topic.topic - self.shouts_by_topic[tpc] = self.shouts_by_topic.get(tpc, dict()) - self.shouts_by_topic[tpc][shout.slug] = shout - self.authors_by_topic[tpc] = self.authors_by_topic.get(tpc, dict()) - authors = session.query( - ShoutAuthor.user, ShoutAuthor.caption - ).filter( - ShoutAuthor.shout == shout.slug - ).all() - for a in authors: - self.authors_by_topic[tpc][a[0]] = a[1] - - self.followers_by_topic = {} - followings = session.query(TopicFollower).all() - print("[stat.topics] ⎪ indexing %d followings..." % len(followings)) - for flw in followings: - topic = flw.topic - userslug = flw.follower - self.followers_by_topic[topic] = self.followers_by_topic.get(topic, dict()) - self.followers_by_topic[topic][userslug] = userslug - - @staticmethod - async def get_shouts(topic): - self = TopicStat - async with self.lock: - return self.shouts_by_topic.get(topic, dict()) - - @staticmethod - async def worker(): - self = TopicStat - first_run = True - while True: - try: - with local_session() as session: - ts = time.time() - async with self.lock: - await self.load_stat(session) - print("[stat.topics] ⎩ load_stat took %fs " % (time.time() - ts)) - except Exception as err: - raise Exception(err) - if first_run: - # sleep for period + 1 min after first run - # to distribute load on server by workers with the same period - await asyncio.sleep(60) - first_run = False - await asyncio.sleep(self.period) diff --git a/services/stat/viewed.py b/services/stat/viewed.py index 2b9678b5..34808166 100644 --- a/services/stat/viewed.py +++ b/services/stat/viewed.py @@ -5,7 +5,9 @@ from gql import Client, gql from gql.transport.aiohttp import AIOHTTPTransport from base.orm import local_session from sqlalchemy import func -from orm.shout import ShoutTopic + +from orm import User, Topic +from orm.shout import ShoutTopic, Shout from orm.viewed import ViewedEntry from ssl import create_default_context from os import environ, path @@ -113,6 +115,7 @@ class ViewedStorage: async with self.lock: return self.client.execute_async(load_facts) + # unused yet @staticmethod async def get_shout(shout_slug): """ getting shout views metric by slug """ @@ -123,8 +126,9 @@ class ViewedStorage: shout_views = 0 with local_session() as session: try: + shout = session.query(Shout).where(Shout.slug == shout_slug).one() shout_views = session.query(func.sum(ViewedEntry.amount)).where( - ViewedEntry.shout == shout_slug + ViewedEntry.shout == shout.id ).all()[0][0] self.by_shouts[shout_slug] = shout_views self.update_topics(session, shout_slug) @@ -147,11 +151,12 @@ class ViewedStorage: def update_topics(session, shout_slug): """ updates topics counters by shout slug """ self = ViewedStorage - for t in session.query(ShoutTopic).where(ShoutTopic.shout == shout_slug).all(): - tpc = t.topic - if not self.by_topics.get(tpc): - self.by_topics[tpc] = {} - self.by_topics[tpc][shout_slug] = self.by_shouts[shout_slug] + for [shout_topic, topic] in session.query(ShoutTopic, Topic).join(Topic).join(Shout).where( + Shout.slug == shout_slug + ).all(): + if not self.by_topics.get(topic.slug): + self.by_topics[topic.slug] = {} + self.by_topics[topic.slug][shout_slug] = self.by_shouts[shout_slug] @staticmethod async def increment(shout_slug, amount=1, viewer='anonymous'): @@ -159,9 +164,12 @@ class ViewedStorage: self = ViewedStorage async with self.lock: with local_session() as session: + shout = session.query(Shout).where(Shout.slug == shout_slug).one() + viewer = session.query(User).where(User.slug == viewer).one() + viewed = ViewedEntry.create(**{ - "viewer": viewer, - "shout": shout_slug, + "viewerId": viewer.id, + "shout": shout.id, "amount": amount }) session.add(viewed) diff --git a/services/zine/shoutauthor.py b/services/zine/shoutauthor.py deleted file mode 100644 index 7d2a3d1d..00000000 --- a/services/zine/shoutauthor.py +++ /dev/null @@ -1,49 +0,0 @@ -import asyncio -import time -from base.orm import local_session -from orm.shout import ShoutAuthor - - -class ShoutAuthorStorage: - authors_by_shout = {} - lock = asyncio.Lock() - # period = 30 * 60 # sec - - @staticmethod - async def load_captions(session): - self = ShoutAuthorStorage - sas = session.query(ShoutAuthor).all() - for sa in sas: - self.authors_by_shout[sa.shout] = self.authors_by_shout.get(sa.shout, {}) - self.authors_by_shout[sa.shout][sa.user] = sa.caption - print("[zine.authors] ⎧ %d shouts indexed by authors" % len(self.authors_by_shout)) - - @staticmethod - async def get_author_caption(shout, author): - self = ShoutAuthorStorage - async with self.lock: - return self.authors_by_shout.get(shout, {}).get(author) - - @staticmethod - async def set_author_caption(shout, author, caption): - self = ShoutAuthorStorage - async with self.lock: - self.authors_by_shout[shout] = self.authors_by_shout.get(shout, {}) - self.authors_by_shout[shout][author] = caption - return { - "error": None, - } - - @staticmethod - async def worker(): - self = ShoutAuthorStorage - async with self.lock: - # while True: - try: - with local_session() as session: - ts = time.time() - await self.load_captions(session) - print("[zine.authors] ⎩ load_captions took %fs " % (time.time() - ts)) - except Exception as err: - print("[zine.authors] ⎩ error indexing by author: %s" % (err)) - # await asyncio.sleep(self.period) diff --git a/services/zine/topics.py b/services/zine/topics.py deleted file mode 100644 index 0aa6af7a..00000000 --- a/services/zine/topics.py +++ /dev/null @@ -1,97 +0,0 @@ -import asyncio -from base.orm import local_session -from orm.topic import Topic -from orm.shout import Shout -import sqlalchemy as sa -from sqlalchemy import select - - -class TopicStorage: - topics = {} - lock = asyncio.Lock() - random_topics = [] - - @staticmethod - def init(session): - self = TopicStorage - topics = session.query(Topic) - self.topics = dict([(topic.slug, topic) for topic in topics]) - for tpc in self.topics.values(): - # self.load_parents(tpc) - pass - - print("[zine.topics] %d precached" % len(self.topics.keys())) - - # @staticmethod - # def load_parents(topic): - # self = TopicStorage - # parents = [] - # for parent in self.topics.values(): - # if topic.slug in parent.children: - # parents.append(parent.slug) - # topic.parents = parents - # return topic - - @staticmethod - def get_random_topics(amount): - return TopicStorage.random_topics[0:amount] - - @staticmethod - def renew_topics_random(): - with local_session() as session: - q = select(Topic).join(Shout).group_by(Topic.id).having(sa.func.count(Shout.id) > 2).order_by( - sa.func.random()).limit(50) - TopicStorage.random_topics = list(map( - lambda result_item: result_item.Topic, session.execute(q) - )) - - @staticmethod - async def worker(): - self = TopicStorage - async with self.lock: - while True: - try: - self.renew_topics_random() - except Exception as err: - print("[zine.topics] error %s" % (err)) - await asyncio.sleep(300) # 5 mins - - @staticmethod - async def get_topics_all(): - self = TopicStorage - async with self.lock: - return list(self.topics.values()) - - @staticmethod - async def get_topics_by_slugs(slugs): - self = TopicStorage - async with self.lock: - if not slugs: - return self.topics.values() - topics = filter(lambda topic: topic.slug in slugs, self.topics.values()) - return list(topics) - - @staticmethod - async def get_topics_by_community(community): - self = TopicStorage - async with self.lock: - topics = filter( - lambda topic: topic.community == community, self.topics.values() - ) - return list(topics) - - @staticmethod - async def get_topics_by_author(author): - self = TopicStorage - async with self.lock: - topics = filter( - lambda topic: topic.community == author, self.topics.values() - ) - return list(topics) - - @staticmethod - async def update_topic(topic): - self = TopicStorage - async with self.lock: - self.topics[topic.slug] = topic - # self.load_parents(topic)