From ab9d03aac6b3b71cbd86b9d3c331ded341a97ec3 Mon Sep 17 00:00:00 2001 From: tonyrewin Date: Sun, 14 Aug 2022 15:48:35 +0300 Subject: [PATCH] resolvers, orm, migration, schema fixes --- .gitignore | 2 +- base/resolvers.py | 8 +- migration/tables/content_items.py | 5 +- migration/tables/users.py | 7 +- orm/reaction.py | 61 +-------- orm/shout.py | 20 +-- queries/shout-by-slug.sql | 4 + resolvers/reactions.py | 14 +- resolvers/zine.py | 220 ++++++++++++++++-------------- schema.graphql | 2 +- services/stat/reacted.py | 48 +++++-- services/stat/topicstat.py | 6 +- services/zine/shoutauthor.py | 21 +-- 13 files changed, 211 insertions(+), 207 deletions(-) create mode 100644 queries/shout-by-slug.sql diff --git a/.gitignore b/.gitignore index 41b0b7ec..650774ab 100644 --- a/.gitignore +++ b/.gitignore @@ -145,5 +145,5 @@ migration/content/**/*.md .DS_Store dump .vscode -*.sql +*dump.sql *.csv \ No newline at end of file diff --git a/base/resolvers.py b/base/resolvers.py index 68414507..e6296549 100644 --- a/base/resolvers.py +++ b/base/resolvers.py @@ -1,15 +1,13 @@ from ariadne import MutationType, QueryType, SubscriptionType, ScalarType -query = QueryType() -mutation = MutationType() -subscription = SubscriptionType() - - datetime_scalar = ScalarType("DateTime") @datetime_scalar.serializer def serialize_datetime(value): return value.isoformat() +query = QueryType() +mutation = MutationType() +subscription = SubscriptionType() resolvers = [query, mutation, subscription, datetime_scalar] diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index 470ed1b8..e279956a 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -123,12 +123,15 @@ def migrate(entry, storage): #del shout_dict['ratings'] email = userdata.get('email') slug = userdata.get('slug') + if not slug: raise Exception with local_session() as session: # c = session.query(Community).all().pop() if email: user = session.query(User).filter(User.email == email).first() if not user and slug: user = session.query(User).filter(User.slug == slug).first() if not user and userdata: - try: user = User.create(**userdata) + try: + userdata['slug'] = userdata['slug'].lower().strip().replace(' ', '-') + user = User.create(**userdata) except sqlalchemy.exc.IntegrityError: print('[migration] user error: ' + userdata) userdata['id'] = user.id diff --git a/migration/tables/users.py b/migration/tables/users.py index d8574c74..bb21c927 100644 --- a/migration/tables/users.py +++ b/migration/tables/users.py @@ -29,7 +29,7 @@ def migrate(entry): if 'wasOnineAt' in entry: user_dict['wasOnlineAt'] = parse(entry['wasOnlineAt']) if entry.get('profile'): # slug - user_dict['slug'] = entry['profile'].get('path') + user_dict['slug'] = entry['profile'].get('path').lower().replace(' ', '-').strip() user_dict['bio'] = html2text(entry.get('profile').get('bio') or '') # userpic @@ -41,10 +41,10 @@ def migrate(entry): # name fn = entry['profile'].get('firstName', '') ln = entry['profile'].get('lastName', '') - name = user_dict['slug'] if user_dict['slug'] else 'noname' + name = user_dict['slug'] if user_dict['slug'] else 'anonymous' name = fn if fn else name name = (name + ' ' + ln) if ln else name - name = entry['profile']['path'].lower().replace(' ', '-') if len(name) < 2 else name + name = entry['profile']['path'].lower().strip().replace(' ', '-') if len(name) < 2 else name user_dict['name'] = name # links @@ -63,6 +63,7 @@ def migrate(entry): user_dict['slug'] = user_dict.get('slug', user_dict['email'].split('@')[0]) oid = user_dict['oid'] + user_dict['slug'] = user_dict['slug'].lower().strip().replace(' ', '-') try: user = User.create(**user_dict.copy()) except sqlalchemy.exc.IntegrityError: print('[migration] cannot create user ' + user_dict['slug']) diff --git a/orm/reaction.py b/orm/reaction.py index d862b23f..ba62abc7 100644 --- a/orm/reaction.py +++ b/orm/reaction.py @@ -1,52 +1,10 @@ from datetime import datetime from sqlalchemy import Column, String, ForeignKey, DateTime -from base.orm import Base, local_session -import enum +from base.orm import Base from sqlalchemy import Enum +from services.stat.reacted import ReactedStorage, ReactionKind from services.stat.viewed import ViewedStorage -class ReactionKind(enum.Enum): - AGREE = 1 # +1 - DISAGREE = 2 # -1 - PROOF = 3 # +1 - DISPROOF = 4 # -1 - ASK = 5 # +0 bookmark - PROPOSE = 6 # +0 - QUOTE = 7 # +0 bookmark - COMMENT = 8 # +0 - ACCEPT = 9 # +1 - REJECT = 0 # -1 - LIKE = 11 # +1 - DISLIKE = 12 # -1 - # TYPE = # rating diff - -def kind_to_rate(kind) -> int: - if kind in [ - ReactionKind.AGREE, - ReactionKind.LIKE, - ReactionKind.PROOF, - ReactionKind.ACCEPT - ]: return 1 - elif kind in [ - ReactionKind.DISAGREE, - ReactionKind.DISLIKE, - ReactionKind.DISPROOF, - ReactionKind.REJECT - ]: return -1 - else: return 0 - -def get_bookmarked(reactions): - c = 0 - for r in reactions: - c += 1 if r.kind in [ ReactionKind.QUOTE, ReactionKind.ASK] else 0 - return c - -def get_rating(reactions): - rating = 0 - for r in reactions: - rating += kind_to_rate(r.kind) - return rating - class Reaction(Base): __tablename__ = 'reaction' body: str = Column(String, nullable=True, comment="Reaction Body") @@ -64,15 +22,10 @@ class Reaction(Base): @property async def stat(self): - reacted = [] - try: - with local_session() as session: - reacted = session.query(Reaction).filter(Reaction.replyTo == self.id).all() - except Exception as e: - print(e) + rrr = await ReactedStorage.get_reaction(self.id) + print(rrr[0]) return { "viewed": await ViewedStorage.get_reaction(self.id), - "reacted": reacted.count(), - "rating": get_rating(reacted), - "bookmarked": get_bookmarked(reacted) - } \ No newline at end of file + "reacted": len(rrr), + "rating": await ReactedStorage.get_reaction_rating(self.id) + } diff --git a/orm/shout.py b/orm/shout.py index cc4d0cf0..77f74071 100644 --- a/orm/shout.py +++ b/orm/shout.py @@ -3,10 +3,10 @@ from sqlalchemy import Column, Integer, String, ForeignKey, DateTime, Boolean from sqlalchemy.orm import relationship from orm.user import User from orm.topic import Topic, ShoutTopic -from orm.reaction import Reaction, get_bookmarked -from services.stat.reacted import ReactedStorage +from orm.reaction import Reaction +from services.stat.reacted import ReactedStorage, ReactionKind from services.stat.viewed import ViewedStorage -from base.orm import Base, local_session +from base.orm import Base class ShoutReactionsFollower(Base): @@ -63,15 +63,9 @@ class Shout(Base): @property async def stat(self): - reacted = [] - try: - with local_session() as session: - reacted = session.query(Reaction).where(Reaction.shout == self.slug).all() - except Exception as e: - print(e) + rrr = await ReactedStorage.get_shout(self.slug) return { "viewed": await ViewedStorage.get_shout(self.slug), - "reacted": await ReactedStorage.get_shout(self.slug), - "rating": await ReactedStorage.get_rating(self.slug), - "bookmarked": get_bookmarked(reacted) - } + "reacted": len(rrr), + "rating": await ReactedStorage.get_rating(self.slug) + } diff --git a/queries/shout-by-slug.sql b/queries/shout-by-slug.sql new file mode 100644 index 00000000..4c274abe --- /dev/null +++ b/queries/shout-by-slug.sql @@ -0,0 +1,4 @@ +SELECT s.*, a.*, sa.* FROM shout s +JOIN shout_author sa ON s.slug = sa.shout +JOIN user a ON a.slug = sa.user +WHERE sa.slug = a.slug AND a.slug = %s; \ No newline at end of file diff --git a/resolvers/reactions.py b/resolvers/reactions.py index 8c9f742a..5e3d045f 100644 --- a/resolvers/reactions.py +++ b/resolvers/reactions.py @@ -6,6 +6,7 @@ from orm.user import User from base.resolvers import mutation, query from auth.authenticate import login_required from datetime import datetime +from services.auth.users import UserStorage from services.stat.reacted import ReactedStorage def reactions_follow(user, slug, auto=False): @@ -103,11 +104,15 @@ async def delete_reaction(_, info, id): return {} @query.field("reactionsByShout") -def get_shout_reactions(_, info, slug, page, size): +async def get_shout_reactions(_, info, slug, page, size): offset = page * size reactions = [] with local_session() as session: - reactions = session.query(Reaction).filter(Reaction.shout == slug).limit(size).offset(offset).all() + reactions = session.query(Reaction).\ + filter(Reaction.shout == slug).\ + limit(size).offset(offset).all() + for r in reactions: + r.createdBy = await UserStorage.get_user(r.createdBy) return reactions @@ -116,12 +121,13 @@ def get_all_reactions(_, info, page=1, size=10): offset = page * size reactions = [] with local_session() as session: - stmt = session.query(Reaction).\ + # raw sql: statement = text(open('queries/reactions-all.sql', 'r').read())) + statement = session.query(Reaction).\ filter(Reaction.deletedAt == None).\ order_by(desc("createdAt")).\ offset(offset).limit(size) reactions = [] - for row in session.execute(stmt): + for row in session.execute(statement): reaction = row.Reaction reactions.append(reaction) reactions.sort(key=lambda x: x.createdAt, reverse=True) diff --git a/resolvers/zine.py b/resolvers/zine.py index 5d412616..7471fde8 100644 --- a/resolvers/zine.py +++ b/resolvers/zine.py @@ -3,6 +3,7 @@ from orm.shout import Shout, ShoutAuthor, ShoutTopic from orm.topic import Topic from base.orm import local_session from base.resolvers import mutation, query +from services.zine.shoutauthor import ShoutAuthorStorage from services.zine.shoutscache import ShoutsCache from services.stat.viewed import ViewedStorage from resolvers.profile import author_follow, author_unfollow @@ -10,152 +11,171 @@ from resolvers.topics import topic_follow, topic_unfollow from resolvers.community import community_follow, community_unfollow from resolvers.reactions import reactions_follow, reactions_unfollow from auth.authenticate import login_required -from sqlalchemy import select, desc, and_ -from sqlalchemy.orm import selectinload, joinedload - +from sqlalchemy import select, desc, and_, text +from sqlalchemy.orm import selectinload +from sqlalchemy.dialects import postgresql @query.field("topViewed") async def top_viewed(_, info, page, size): - async with ShoutsCache.lock: - return ShoutsCache.top_viewed[(page - 1) * size : page * size] + async with ShoutsCache.lock: + return ShoutsCache.top_viewed[(page - 1) * size : page * size] @query.field("topMonth") async def top_month(_, info, page, size): - async with ShoutsCache.lock: - return ShoutsCache.top_month[(page - 1) * size : page * size] + async with ShoutsCache.lock: + return ShoutsCache.top_month[(page - 1) * size : page * size] @query.field("topOverall") async def top_overall(_, info, page, size): - async with ShoutsCache.lock: - return ShoutsCache.top_overall[(page - 1) * size : page * size] + async with ShoutsCache.lock: + return ShoutsCache.top_overall[(page - 1) * size : page * size] @query.field("recentPublished") async def recent_published(_, info, page, size): - async with ShoutsCache.lock: - return ShoutsCache.recent_published[(page - 1) * size : page * size] + async with ShoutsCache.lock: + return ShoutsCache.recent_published[(page - 1) * size : page * size] @query.field("recentAll") async def recent_all(_, info, page, size): - async with ShoutsCache.lock: - return ShoutsCache.recent_all[(page - 1) * size : page * size] + async with ShoutsCache.lock: + return ShoutsCache.recent_all[(page - 1) * size : page * size] @query.field("recentReacted") async def recent_reacted(_, info, page, size): - async with ShoutsCache.lock: - return ShoutsCache.recent_reacted[(page - 1) * size : page * size] + async with ShoutsCache.lock: + return ShoutsCache.recent_reacted[(page - 1) * size : page * size] @mutation.field("viewShout") async def view_shout(_, info, slug): - await ViewedStorage.inc_shout(slug) - return {"error" : ""} + await ViewedStorage.inc_shout(slug) + return {"error" : ""} @query.field("getShoutBySlug") async def get_shout_by_slug(_, info, slug): - shout = None - # FIXME: append captions anyhow - with local_session() as session: - shout = session.query(Shout, ShoutAuthor.caption.label("author_caption")).\ - options([ - selectinload(Shout.topics), - selectinload(Shout.reactions), - joinedload(Shout.authors), - selectinload(ShoutAuthor.caption) - ]).\ - join(ShoutAuthor.shout == slug ).\ - filter(Shout.slug == slug).first() + all_fields = [node.name.value for node in info.field_nodes[0].selection_set.selections] + selected_fields = set(["authors", "topics"]).intersection(all_fields) + select_options = [selectinload(getattr(Shout, field)) for field in selected_fields] - if not shout: - print(f"[resolvers.zine] error: shout with slug {slug} not exist") - return {"error" : "shout not found"} - - return shout + with local_session() as session: + try: s = text(open('src/queries/shout-by-slug.sql', 'r').read() % slug) + except: pass + shout_q = session.query(Shout).\ + options(select_options).\ + filter(Shout.slug == slug) + + print(shout_q.statement) + + shout = shout_q.first() + for a in shout.authors: + a.caption = await ShoutAuthorStorage.get_author_caption(slug, a.slug) + + if not shout: + print(f"shout with slug {slug} not exist") + return {"error" : "shout not found"} + return shout @query.field("shoutsByTopics") async def shouts_by_topics(_, info, slugs, page, size): - page = page - 1 - with local_session() as session: - shouts = session.query(Shout).\ - join(ShoutTopic).\ - where(and_(ShoutTopic.topic.in_(slugs), Shout.publishedAt != None)).\ - order_by(desc(Shout.publishedAt)).\ - limit(size).\ - offset(page * size) - return shouts + page = page - 1 + with local_session() as session: + shouts = session.query(Shout).\ + join(ShoutTopic).\ + where(and_(ShoutTopic.topic.in_(slugs), Shout.publishedAt != None)).\ + order_by(desc(Shout.publishedAt)).\ + limit(size).\ + offset(page * size) + + for s in shouts: + for a in s.authors: + a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug) + return shouts @query.field("shoutsByCollection") async def shouts_by_topics(_, info, collection, page, size): - page = page - 1 - with local_session() as session: - shouts = session.query(Shout).\ - join(ShoutCollection, ShoutCollection.collection == collection).\ - where(and_(ShoutCollection.shout == Shout.slug, Shout.publishedAt != None)).\ - order_by(desc(Shout.publishedAt)).\ - limit(size).\ - offset(page * size) - return shouts + page = page - 1 + shouts = [] + with local_session() as session: + shouts = session.query(Shout).\ + join(ShoutCollection, ShoutCollection.collection == collection).\ + where(and_(ShoutCollection.shout == Shout.slug, Shout.publishedAt != None)).\ + order_by(desc(Shout.publishedAt)).\ + limit(size).\ + offset(page * size) + for s in shouts: + for a in s.authors: + a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug) + return shouts @query.field("shoutsByAuthors") async def shouts_by_authors(_, info, slugs, page, size): - page = page - 1 - with local_session() as session: + page = page - 1 + with local_session() as session: - shouts = session.query(Shout).\ - join(ShoutAuthor).\ - where(and_(ShoutAuthor.user.in_(slugs), Shout.publishedAt != None)).\ - order_by(desc(Shout.publishedAt)).\ - limit(size).\ - offset(page * size) - return shouts + shouts = session.query(Shout).\ + join(ShoutAuthor).\ + where(and_(ShoutAuthor.user.in_(slugs), Shout.publishedAt != None)).\ + order_by(desc(Shout.publishedAt)).\ + limit(size).\ + offset(page * size) + + for s in shouts: + for a in s.authors: + a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug) + return shouts @query.field("shoutsByCommunities") async def shouts_by_communities(_, info, slugs, page, size): - page = page - 1 - with local_session() as session: - #TODO fix postgres high load - shouts = session.query(Shout).distinct().\ - join(ShoutTopic).\ - where(and_(Shout.publishedAt != None,\ - ShoutTopic.topic.in_(\ - select(Topic.slug).where(Topic.community.in_(slugs))\ - ))).\ - order_by(desc(Shout.publishedAt)).\ - limit(size).\ - offset(page * size) - return shouts + page = page - 1 + with local_session() as session: + #TODO fix postgres high load + shouts = session.query(Shout).distinct().\ + join(ShoutTopic).\ + where(and_(Shout.publishedAt != None,\ + ShoutTopic.topic.in_(\ + select(Topic.slug).where(Topic.community.in_(slugs))\ + ))).\ + order_by(desc(Shout.publishedAt)).\ + limit(size).\ + offset(page * size) + + for s in shouts: + for a in s.authors: + a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug) + return shouts @mutation.field("follow") @login_required async def follow(_, info, what, slug): - user = info.context["request"].user - try: - if what == "AUTHOR": - author_follow(user, slug) - elif what == "TOPIC": - topic_follow(user, slug) - elif what == "COMMUNITY": - community_follow(user, slug) - elif what == "REACTIONS": - reactions_follow(user, slug) - except Exception as e: - return {"error" : str(e)} + user = info.context["request"].user + try: + if what == "AUTHOR": + author_follow(user, slug) + elif what == "TOPIC": + topic_follow(user, slug) + elif what == "COMMUNITY": + community_follow(user, slug) + elif what == "REACTIONS": + reactions_follow(user, slug) + except Exception as e: + return {"error" : str(e)} - return {} + return {} @mutation.field("unfollow") @login_required async def unfollow(_, info, what, slug): - user = info.context["request"].user + user = info.context["request"].user - try: - if what == "AUTHOR": - author_unfollow(user, slug) - elif what == "TOPIC": - topic_unfollow(user, slug) - elif what == "COMMUNITY": - community_unfollow(user, slug) - elif what == "REACTIONS": - reactions_unfollow(user, slug) - except Exception as e: - return {"error" : str(e)} + try: + if what == "AUTHOR": + author_unfollow(user, slug) + elif what == "TOPIC": + topic_unfollow(user, slug) + elif what == "COMMUNITY": + community_unfollow(user, slug) + elif what == "REACTIONS": + reactions_unfollow(user, slug) + except Exception as e: + return {"error" : str(e)} - return {} + return {} diff --git a/schema.graphql b/schema.graphql index 7686df07..c55468fc 100644 --- a/schema.graphql +++ b/schema.graphql @@ -241,7 +241,7 @@ type Query { # reactons reactionsAll(page: Int!, size: Int!): [Reaction]! reactionsByAuthor(slug: String!, page: Int!, size: Int!): [Reaction]! - reactionsByShout(slug: String!): [Reaction]! + reactionsByShout(slug: String!, page: Int!, size: Int!): [Reaction]! # collab inviteAuthor(slug: String!, author: String!): Result! diff --git a/services/stat/reacted.py b/services/stat/reacted.py index 03714a39..d17e35e5 100644 --- a/services/stat/reacted.py +++ b/services/stat/reacted.py @@ -1,12 +1,42 @@ import asyncio from datetime import datetime from sqlalchemy.types import Enum -from sqlalchemy import Column, DateTime, ForeignKey, Integer +from sqlalchemy import Column, DateTime, ForeignKey # from sqlalchemy.orm.attributes import flag_modified +from sqlalchemy import Enum +import enum from base.orm import Base, local_session -from orm.reaction import Reaction, ReactionKind, kind_to_rate from orm.topic import ShoutTopic +class ReactionKind(enum.Enum): + AGREE = 1 # +1 + DISAGREE = 2 # -1 + PROOF = 3 # +1 + DISPROOF = 4 # -1 + ASK = 5 # +0 bookmark + PROPOSE = 6 # +0 + QUOTE = 7 # +0 bookmark + COMMENT = 8 # +0 + ACCEPT = 9 # +1 + REJECT = 0 # -1 + LIKE = 11 # +1 + DISLIKE = 12 # -1 + # TYPE = # rating diff + +def kind_to_rate(kind) -> int: + if kind in [ + ReactionKind.AGREE, + ReactionKind.LIKE, + ReactionKind.PROOF, + ReactionKind.ACCEPT + ]: return 1 + elif kind in [ + ReactionKind.DISAGREE, + ReactionKind.DISLIKE, + ReactionKind.DISPROOF, + ReactionKind.REJECT + ]: return -1 + else: return 0 class ReactedByDay(Base): __tablename__ = "reacted_by_day" @@ -37,12 +67,8 @@ class ReactedStorage: def init(session): self = ReactedStorage all_reactions = session.query(ReactedByDay).all() - all_reactions2 = session.query(Reaction).filter(Reaction.deletedAt == None).all() - print('[stat.reacted] %d reactions total' % len(all_reactions or all_reactions2)) - rrr = (all_reactions or all_reactions2) - create = False - if not all_reactions: create = True - for reaction in rrr: + print('[stat.reacted] %d reactions total' % len(all_reactions)) + for reaction in all_reactions: shout = reaction.shout topics = session.query(ShoutTopic.topic).where(ShoutTopic.shout == shout).all() kind = reaction.kind @@ -64,12 +90,6 @@ class ReactedStorage: print('[stat.reacted] %d topics reacted' % len(ttt)) print('[stat.reacted] %d shouts reacted' % len(self.reacted['shouts'])) print('[stat.reacted] %d reactions reacted' % len(self.reacted['reactions'])) - - if len(all_reactions) == 0 and len(all_reactions2) != 0: - with local_session() as session: - for r in all_reactions2: - session.add(ReactedByDay(reaction=r.id, shout=r.shout, reply=r.replyTo, kind=r.kind, day=r.createdAt.replace(hour=0, minute=0, second=0))) - session.commit() @staticmethod async def get_shout(shout_slug): diff --git a/services/stat/topicstat.py b/services/stat/topicstat.py index 146f6a81..676b9cb0 100644 --- a/services/stat/topicstat.py +++ b/services/stat/topicstat.py @@ -65,9 +65,9 @@ class TopicStat: "shouts" : len(shouts), "authors" : len(authors), "followers" : len(followers), - "viewed": ViewedStorage.get_topic(topic), - "reacted" : ReactedStorage.get_topic(topic), - "rating" : ReactedStorage.get_topic_rating(topic), + "viewed": await ViewedStorage.get_topic(topic), + "reacted" : await ReactedStorage.get_topic(topic), + "rating" : await ReactedStorage.get_topic_rating(topic), } @staticmethod diff --git a/services/zine/shoutauthor.py b/services/zine/shoutauthor.py index 4ae20405..98ea9bec 100644 --- a/services/zine/shoutauthor.py +++ b/services/zine/shoutauthor.py @@ -12,14 +12,10 @@ class ShoutAuthorStorage: @staticmethod async def load(session): self = ShoutAuthorStorage - authors = session.query(ShoutAuthor).all() - for author in authors: - user = author.user - shout = author.shout - if shout in self.authors_by_shout: - self.authors_by_shout[shout].append(user) - else: - self.authors_by_shout[shout] = [user] + sas = session.query(ShoutAuthor).all() + for sa in sas: + self.authors_by_shout[sa.shout] = self.authors_by_shout.get(sa.shout, []) + self.authors_by_shout[sa.shout].append([sa.user, sa.caption]) print('[zine.authors] %d shouts preprocessed' % len(self.authors_by_shout)) @staticmethod @@ -28,6 +24,15 @@ class ShoutAuthorStorage: async with self.lock: return self.authors_by_shout.get(shout, []) + @staticmethod + async def get_author_caption(shout, author): + self = ShoutAuthorStorage + async with self.lock: + for a in self.authors_by_shout.get(shout, []): + if author in a: + return a[1] + return { "error": "author caption not found" } + @staticmethod async def worker(): self = ShoutAuthorStorage