From 96e9728d6b4617f64c2720dedc6165095c122fd0 Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Fri, 25 Nov 2022 15:59:10 +0100 Subject: [PATCH 01/12] WIP --- main.py | 2 - migration/tables/comments.py | 1 - migration/tables/content_items.py | 1 - resolvers/zine/load.py | 11 +- resolvers/zine/profile.py | 2 - resolvers/zine/reactions.py | 1 - resolvers/zine/topics.py | 2 - services/main.py | 4 - services/stat/reacted.py | 207 ------------------------------ services/stat/topicstat.py | 73 ----------- services/zine/topics.py | 68 ---------- 11 files changed, 4 insertions(+), 368 deletions(-) delete mode 100644 services/stat/reacted.py delete mode 100644 services/stat/topicstat.py delete mode 100644 services/zine/topics.py diff --git a/main.py b/main.py index 3e1deaae..4384016d 100644 --- a/main.py +++ b/main.py @@ -16,8 +16,6 @@ from base.redis import redis from base.resolvers import resolvers from resolvers.auth import confirm_email_handler from services.main import storages_init -from services.stat.reacted import ReactedStorage -from services.stat.topicstat import TopicStat from services.stat.viewed import ViewedStorage from services.zine.gittask import GitTask from settings import DEV_SERVER_STATUS_FILE_NAME diff --git a/migration/tables/comments.py b/migration/tables/comments.py index c93e3d63..944cad1c 100644 --- a/migration/tables/comments.py +++ b/migration/tables/comments.py @@ -8,7 +8,6 @@ from orm.reaction import Reaction, ReactionKind from orm.shout import ShoutReactionsFollower from orm.topic import TopicFollower from orm.user import User -from services.stat.reacted import ReactedStorage ts = datetime.now(tz=timezone.utc) diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index af5f99d5..926e28d3 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -9,7 +9,6 @@ from orm.reaction import Reaction, ReactionKind from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower from orm.user import User from orm.topic import TopicFollower -from services.stat.reacted import ReactedStorage from services.stat.viewed import ViewedStorage OLD_DATE = "2016-03-05 22:22:00.350000" diff --git a/resolvers/zine/load.py b/resolvers/zine/load.py index fe13d431..d25167ed 100644 --- a/resolvers/zine/load.py +++ b/resolvers/zine/load.py @@ -7,7 +7,6 @@ from base.resolvers import query from orm import ViewedEntry from orm.shout import Shout, ShoutAuthor from orm.reaction import Reaction, ReactionKind -from services.stat.reacted import ReactedStorage def add_rating_column(q): @@ -135,17 +134,15 @@ async def load_shouts_by(_, info, options): q = q.group_by(Shout.id).order_by(query_order_by).limit(limit).offset(offset) with local_session() as session: - shouts = list(map(map_result_item, session.execute(q).unique())) - - for shout in shouts: - shout.stat = await ReactedStorage.get_shout_stat(shout.slug, shout.rating) - del shout.rating + results = session.execute(q).unique() + for [shout, rating] in results: + shout.stat = await ReactedStorage.get_shout_stat(shout.slug, rating) author_captions = {} if with_author_captions: author_captions_result = session.query(ShoutAuthor).where( - ShoutAuthor.shout.in_(map(lambda s: s.slug, shouts))).all() + ShoutAuthor.shout.in_(map(lambda result_item: result_item[0].slug, results))).all() for author_captions_result_item in author_captions_result: if author_captions.get(author_captions_result_item.shout) is None: diff --git a/resolvers/zine/profile.py b/resolvers/zine/profile.py index d0e9d3af..e43d3884 100644 --- a/resolvers/zine/profile.py +++ b/resolvers/zine/profile.py @@ -10,8 +10,6 @@ from orm.reaction import Reaction from orm.shout import ShoutAuthor from orm.topic import Topic, TopicFollower from orm.user import AuthorFollower, Role, User, UserRating, UserRole -from services.stat.reacted import ReactedStorage -from services.stat.topicstat import TopicStat # from .community import followed_communities from resolvers.inbox.unread import get_total_unread_counter diff --git a/resolvers/zine/reactions.py b/resolvers/zine/reactions.py index 646de03a..e3fcf416 100644 --- a/resolvers/zine/reactions.py +++ b/resolvers/zine/reactions.py @@ -8,7 +8,6 @@ from base.resolvers import mutation, query from orm.reaction import Reaction, ReactionKind from orm.shout import Shout, ShoutReactionsFollower from orm.user import User -from services.stat.reacted import ReactedStorage async def get_reaction_stat(reaction_id): diff --git a/resolvers/zine/topics.py b/resolvers/zine/topics.py index 81db4f91..6c1327f7 100644 --- a/resolvers/zine/topics.py +++ b/resolvers/zine/topics.py @@ -5,9 +5,7 @@ from base.orm import local_session from base.resolvers import mutation, query from orm import Shout from orm.topic import Topic, TopicFollower -from services.zine.topics import TopicStorage # from services.stat.reacted import ReactedStorage -from services.stat.topicstat import TopicStat # from services.stat.viewed import ViewedStorage diff --git a/services/main.py b/services/main.py index 636f01db..f87f8afe 100644 --- a/services/main.py +++ b/services/main.py @@ -1,7 +1,5 @@ -from services.stat.reacted import ReactedStorage from services.auth.roles import RoleStorage from services.auth.users import UserStorage -from services.zine.topics import TopicStorage from services.search import SearchService from services.stat.viewed import ViewedStorage from base.orm import local_session @@ -10,10 +8,8 @@ from base.orm import local_session async def storages_init(): with local_session() as session: print('[main] initialize storages') - ReactedStorage.init(session) RoleStorage.init(session) UserStorage.init(session) - TopicStorage.init(session) await SearchService.init(session) session.commit() await ViewedStorage.init() diff --git a/services/stat/reacted.py b/services/stat/reacted.py deleted file mode 100644 index ef5a2953..00000000 --- a/services/stat/reacted.py +++ /dev/null @@ -1,207 +0,0 @@ -import asyncio -import time -from base.orm import local_session -from orm.reaction import ReactionKind, Reaction -from services.zine.topics import TopicStorage -from services.stat.viewed import ViewedStorage - - -def kind_to_rate(kind) -> int: - if kind in [ - ReactionKind.AGREE, - ReactionKind.LIKE, - ReactionKind.PROOF, - ReactionKind.ACCEPT, - ]: - return 1 - elif kind in [ - ReactionKind.DISAGREE, - ReactionKind.DISLIKE, - ReactionKind.DISPROOF, - ReactionKind.REJECT, - ]: - return -1 - else: - return 0 - - -class ReactedStorage: - reacted = {"shouts": {}, "topics": {}, "reactions": {}, "authors": {}} - rating = {"shouts": {}, "topics": {}, "reactions": {}} - reactions = [] - to_flush = [] - period = 30 * 60 # sec - lock = asyncio.Lock() - modified_shouts = set([]) - - @staticmethod - async def get_shout_stat(slug, rating): - viewed = int(await ViewedStorage.get_shout(slug)) - # print(viewed) - return { - "viewed": viewed, - "reacted": len(await ReactedStorage.get_shout(slug)), - "commented": len(await ReactedStorage.get_comments(slug)), - # "rating": await ReactedStorage.get_rating(slug), - "rating": rating - } - - @staticmethod - async def get_shout(shout_slug): - self = ReactedStorage - async with self.lock: - return self.reacted["shouts"].get(shout_slug, []) - - @staticmethod - async def get_author(user_slug): - self = ReactedStorage - async with self.lock: - return self.reacted["authors"].get(user_slug, []) - - @staticmethod - async def get_shouts_by_author(user_slug): - self = ReactedStorage - async with self.lock: - author_reactions = self.reacted["authors"].get(user_slug, []) - shouts = [] - for r in author_reactions: - if r.shout not in shouts: - shouts.append(r.shout) - return shouts - - @staticmethod - async def get_topic(topic_slug): - self = ReactedStorage - async with self.lock: - return self.reacted["topics"].get(topic_slug, []) - - @staticmethod - async def get_comments(shout_slug): - self = ReactedStorage - async with self.lock: - return list( - filter(lambda r: bool(r.body), self.reacted["shouts"].get(shout_slug, {})) - ) - - @staticmethod - async def get_topic_comments(topic_slug): - self = ReactedStorage - async with self.lock: - return list( - filter(lambda r: bool(r.body), self.reacted["topics"].get(topic_slug, [])) - ) - - @staticmethod - async def get_reaction_comments(reaction_id): - self = ReactedStorage - async with self.lock: - return list( - filter( - lambda r: bool(r.body), self.reacted["reactions"].get(reaction_id, {}) - ) - ) - - @staticmethod - async def get_reaction(reaction_id): - self = ReactedStorage - async with self.lock: - return self.reacted["reactions"].get(reaction_id, []) - - @staticmethod - async def get_rating(shout_slug): - self = ReactedStorage - rating = 0 - async with self.lock: - for r in self.reacted["shouts"].get(shout_slug, []): - rating = rating + kind_to_rate(r.kind) - return rating - - @staticmethod - async def get_topic_rating(topic_slug): - self = ReactedStorage - rating = 0 - async with self.lock: - for r in self.reacted["topics"].get(topic_slug, []): - rating = rating + kind_to_rate(r.kind) - return rating - - @staticmethod - async def get_reaction_rating(reaction_id): - self = ReactedStorage - rating = 0 - async with self.lock: - for r in self.reacted["reactions"].get(reaction_id, []): - rating = rating + kind_to_rate(r.kind) - return rating - - @staticmethod - async def react(reaction): - ReactedStorage.modified_shouts.add(reaction.shout) - - @staticmethod - async def recount(reactions): - self = ReactedStorage - for r in reactions: - # renew reactions by shout - self.reacted["shouts"][r.shout] = self.reacted["shouts"].get(r.shout, []) - self.reacted["shouts"][r.shout].append(r) - # renew reactions by author - self.reacted["authors"][r.createdBy] = self.reacted["authors"].get(r.createdBy, []) - self.reacted["authors"][r.createdBy].append(r) - # renew reactions by topic - shout_topics = await TopicStorage.get_topics_by_slugs([r.shout, ]) - for t in shout_topics: - self.reacted["topics"][t] = self.reacted["topics"].get(t, []) - self.reacted["topics"][t].append(r) - self.rating["topics"][t] = \ - self.rating["topics"].get(t, 0) + kind_to_rate(r.kind) - if r.replyTo: - # renew reactions replies - self.reacted["reactions"][r.replyTo] = \ - self.reacted["reactions"].get(r.replyTo, []) - self.reacted["reactions"][r.replyTo].append(r) - self.rating["reactions"][r.replyTo] = \ - self.rating["reactions"].get(r.replyTo, 0) + kind_to_rate(r.kind) - else: - # renew shout rating - self.rating["shouts"][r.shout] = \ - self.rating["shouts"].get(r.shout, 0) + kind_to_rate(r.kind) - - @staticmethod - def init(session): - self = ReactedStorage - all_reactions = session.query(Reaction).all() - self.modified_shouts = list(set([r.shout for r in all_reactions])) - print("[stat.reacted] %d shouts with reactions" % len(self.modified_shouts)) - - @staticmethod - async def recount_changed(session): - start = time.time() - self = ReactedStorage - async with self.lock: - sss = list(self.modified_shouts) - c = 0 - for slug in sss: - siblings = session.query(Reaction).where(Reaction.shout == slug).all() - c += len(siblings) - await self.recount(siblings) - - print("[stat.reacted] %d reactions recounted" % c) - print("[stat.reacted] %d shouts modified" % len(self.modified_shouts)) - print("[stat.reacted] %d topics" % len(self.reacted["topics"].values())) - print("[stat.reacted] %d authors" % len(self.reacted["authors"].values())) - print("[stat.reacted] %d replies" % len(self.reacted["reactions"])) - self.modified_shouts = set([]) - - end = time.time() - print("[stat.reacted] recount_changed took %fs " % (end - start)) - - @staticmethod - async def worker(): - while True: - try: - with local_session() as session: - await ReactedStorage.recount_changed(session) - except Exception as err: - print("[stat.reacted] recount error %s" % (err)) - await asyncio.sleep(ReactedStorage.period) diff --git a/services/stat/topicstat.py b/services/stat/topicstat.py deleted file mode 100644 index c95d0850..00000000 --- a/services/stat/topicstat.py +++ /dev/null @@ -1,73 +0,0 @@ -import asyncio -import time -from base.orm import local_session -from orm.shout import Shout, ShoutTopic, ShoutAuthor -from orm.topic import TopicFollower -from sqlalchemy.sql.expression import select - - -class TopicStat: - # by slugs - shouts_by_topic = {} # Shout object stored - authors_by_topic = {} # User - followers_by_topic = {} # User - # - lock = asyncio.Lock() - period = 30 * 60 # sec - - @staticmethod - async def load_stat(session): - start = time.time() - self = TopicStat - shout_topics = session.query(ShoutTopic, Shout).join(Shout).all() - all_shout_authors = session.query(ShoutAuthor).all() - print("[stat.topics] %d links for shouts" % len(shout_topics)) - for [shout_topic, shout] in shout_topics: - tpc = shout_topic.topic - # shouts by topics - # shout = session.query(Shout).where(Shout.slug == shout_topic.shout).first() - self.shouts_by_topic[tpc] = self.shouts_by_topic.get(tpc, dict()) - self.shouts_by_topic[tpc][shout.slug] = shout - - # authors by topics - shout_authors = filter(lambda asa: asa.shout == shout.slug, all_shout_authors) - - self.authors_by_topic[tpc] = self.authors_by_topic.get(tpc, dict()) - for sa in shout_authors: - self.authors_by_topic[tpc][sa.shout] = sa.caption - - self.followers_by_topic = {} - followings = session.query(TopicFollower).all() - print("[stat.topics] %d followings by users" % len(followings)) - for flw in followings: - topic = flw.topic - userslug = flw.follower - self.followers_by_topic[topic] = self.followers_by_topic.get(topic, dict()) - self.followers_by_topic[topic][userslug] = userslug - - end = time.time() - print("[stat.topics] load_stat took %fs " % (end - start)) - - @staticmethod - async def get_shouts(topic): - self = TopicStat - async with self.lock: - return self.shouts_by_topic.get(topic, dict()) - - @staticmethod - async def worker(): - self = TopicStat - first_run = True - while True: - try: - with local_session() as session: - async with self.lock: - await self.load_stat(session) - except Exception as err: - raise Exception(err) - if first_run: - # sleep for period + 1 min after first run - # to distribute load on server by workers with the same period - await asyncio.sleep(60) - first_run = False - await asyncio.sleep(self.period) diff --git a/services/zine/topics.py b/services/zine/topics.py deleted file mode 100644 index 13889899..00000000 --- a/services/zine/topics.py +++ /dev/null @@ -1,68 +0,0 @@ -import asyncio -from orm.topic import Topic - - -class TopicStorage: - topics = {} - lock = asyncio.Lock() - - @staticmethod - def init(session): - self = TopicStorage - topics = session.query(Topic) - self.topics = dict([(topic.slug, topic) for topic in topics]) - for tpc in self.topics.values(): - # self.load_parents(tpc) - pass - - print("[zine.topics] %d precached" % len(self.topics.keys())) - - # @staticmethod - # def load_parents(topic): - # self = TopicStorage - # parents = [] - # for parent in self.topics.values(): - # if topic.slug in parent.children: - # parents.append(parent.slug) - # topic.parents = parents - # return topic - - @staticmethod - async def get_topics_all(): - self = TopicStorage - async with self.lock: - return list(self.topics.values()) - - @staticmethod - async def get_topics_by_slugs(slugs): - self = TopicStorage - async with self.lock: - if not slugs: - return self.topics.values() - topics = filter(lambda topic: topic.slug in slugs, self.topics.values()) - return list(topics) - - @staticmethod - async def get_topics_by_community(community): - self = TopicStorage - async with self.lock: - topics = filter( - lambda topic: topic.community == community, self.topics.values() - ) - return list(topics) - - @staticmethod - async def get_topics_by_author(author): - self = TopicStorage - async with self.lock: - topics = filter( - lambda topic: topic.community == author, self.topics.values() - ) - return list(topics) - - @staticmethod - async def update_topic(topic): - self = TopicStorage - async with self.lock: - self.topics[topic.slug] = topic - # self.load_parents(topic) From cfc1beeec015218e9018dfbf0151226678ebad9f Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Fri, 25 Nov 2022 19:31:53 +0100 Subject: [PATCH 02/12] WIP --- CHECKS | 2 +- main.py | 5 +- migration/tables/comments.py | 2 - migration/tables/content_items.py | 1 - orm/__init__.py | 2 +- resolvers/__init__.py | 3 +- resolvers/zine/load.py | 117 ++++++++++++++++-------------- resolvers/zine/reactions.py | 1 - 8 files changed, 66 insertions(+), 67 deletions(-) diff --git a/CHECKS b/CHECKS index 5affecfc..e2d41a77 100644 --- a/CHECKS +++ b/CHECKS @@ -1,5 +1,5 @@ WAIT=30 TIMEOUT=10 -ATTEMPTS=60 # 60 * 30 = 30 min +ATTEMPTS=20 # 20 * 30 = 10 min / Playground diff --git a/main.py b/main.py index 4384016d..7b8fcee9 100644 --- a/main.py +++ b/main.py @@ -35,13 +35,10 @@ async def start_up(): await storages_init() views_stat_task = asyncio.create_task(ViewedStorage().worker()) print(views_stat_task) - reacted_storage_task = asyncio.create_task(ReactedStorage.worker()) - print(reacted_storage_task) - topic_stat_task = asyncio.create_task(TopicStat.worker()) - print(topic_stat_task) git_task = asyncio.create_task(GitTask.git_task_worker()) print(git_task) + async def dev_start_up(): if exists(DEV_SERVER_STATUS_FILE_NAME): return diff --git a/migration/tables/comments.py b/migration/tables/comments.py index 944cad1c..9fb099f7 100644 --- a/migration/tables/comments.py +++ b/migration/tables/comments.py @@ -76,7 +76,6 @@ async def migrate(entry, storage): # creating reaction from old comment reaction = Reaction.create(**reaction_dict) session.add(reaction) - await ReactedStorage.react(reaction) # creating shout's reactions following for reaction author following1 = session.query( @@ -147,7 +146,6 @@ async def migrate(entry, storage): ) session.add(following2) session.add(rr) - await ReactedStorage.react(rr) except Exception as e: print("[migration] comment rating error: %r" % re_reaction_dict) diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index 926e28d3..9f490eb8 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -335,7 +335,6 @@ async def migrate(entry, storage): else: rea = Reaction.create(**reaction_dict) session.add(rea) - await ReactedStorage.react(rea) # shout_dict['ratings'].append(reaction_dict) session.commit() diff --git a/orm/__init__.py b/orm/__init__.py index c8251256..b83e8ad8 100644 --- a/orm/__init__.py +++ b/orm/__init__.py @@ -21,7 +21,7 @@ __all__ = [ "TopicFollower", "Notification", "Reaction", - "UserRating" + "UserRating", "ViewedEntry" ] diff --git a/resolvers/__init__.py b/resolvers/__init__.py index fccca3a0..8e504ab7 100644 --- a/resolvers/__init__.py +++ b/resolvers/__init__.py @@ -15,7 +15,8 @@ from resolvers.create.editor import create_shout, delete_shout, update_shout from resolvers.zine.profile import ( load_authors_by, rate_user, - update_profile + update_profile, + get_authors_all ) from resolvers.zine.reactions import ( diff --git a/resolvers/zine/load.py b/resolvers/zine/load.py index d25167ed..531ece51 100644 --- a/resolvers/zine/load.py +++ b/resolvers/zine/load.py @@ -1,6 +1,6 @@ from datetime import datetime, timedelta, timezone import sqlalchemy as sa -from sqlalchemy.orm import joinedload +from sqlalchemy.orm import joinedload, aliased, make_transient from sqlalchemy.sql.expression import desc, asc, select, case from base.orm import local_session from base.resolvers import query @@ -9,8 +9,25 @@ from orm.shout import Shout, ShoutAuthor from orm.reaction import Reaction, ReactionKind -def add_rating_column(q): - return q.join(Reaction).add_columns(sa.func.sum(case( +def add_viewed_stat_column(q): + return q.outerjoin(ViewedEntry).add_columns(sa.func.sum(ViewedEntry.amount).label('viewed_stat')) + + +def add_reacted_stat_column(q): + aliased_reaction = aliased(Reaction) + return q.outerjoin(aliased_reaction).add_columns(sa.func.count(aliased_reaction.id).label('reacted_stat')) + + +def add_commented_stat_column(q): + aliased_reaction = aliased(Reaction) + return q.outerjoin( + aliased_reaction, + aliased_reaction.shout == Shout.slug and aliased_reaction.body.is_not(None) + ).add_columns(sa.func.count(aliased_reaction.id).label('commented_stat')) + + +def add_rating_stat_column(q): + return q.outerjoin(Reaction).add_columns(sa.func.sum(case( (Reaction.kind == ReactionKind.AGREE, 1), (Reaction.kind == ReactionKind.DISAGREE, -1), (Reaction.kind == ReactionKind.PROOF, 1), @@ -20,7 +37,7 @@ def add_rating_column(q): (Reaction.kind == ReactionKind.LIKE, 1), (Reaction.kind == ReactionKind.DISLIKE, -1), else_=0 - )).label('rating')) + )).label('rating_stat')) def apply_filters(q, filters, user=None): @@ -45,6 +62,14 @@ def apply_filters(q, filters, user=None): return q +def add_stat_columns(q): + q = add_viewed_stat_column(q) + q = add_reacted_stat_column(q) + q = add_commented_stat_column(q) + q = add_rating_stat_column(q) + return q + + @query.field("loadShout") async def load_shout(_, info, slug): with local_session() as session: @@ -53,26 +78,30 @@ async def load_shout(_, info, slug): joinedload(Shout.authors), joinedload(Shout.topics), ) - q = add_rating_column(q) + q = add_stat_columns(q) q = q.filter( Shout.slug == slug ).filter( Shout.deletedAt.is_(None) ).group_by(Shout.id) - [shout, rating] = session.execute(q).unique().one() + [shout, viewed_stat, reacted_stat, commented_stat, rating_stat] = session.execute(q).unique().one() - shout.stat = await ReactedStorage.get_shout_stat(shout.slug, rating) + shout.stat = { + "viewed": viewed_stat, + "reacted": reacted_stat, + "commented": commented_stat, + "rating": rating_stat + } + + for author_caption in session.query(ShoutAuthor).where(ShoutAuthor.shout == slug): + for author in shout.authors: + if author.slug == author_caption.user: + author.caption = author_caption.caption return shout -def map_result_item(result_item): - shout = result_item[0] - shout.rating = result_item[1] - return shout - - @query.field("loadShouts") async def load_shouts_by(_, info, options): """ @@ -88,7 +117,7 @@ async def load_shouts_by(_, info, options): } offset: 0 limit: 50 - order_by: 'createdAt' + order_by: 'createdAt' | 'commented' | 'reacted' | 'rating' order_by_desc: true } @@ -101,60 +130,36 @@ async def load_shouts_by(_, info, options): ).where( Shout.deletedAt.is_(None) ) + + q = add_stat_columns(q) + user = info.context["request"].user q = apply_filters(q, options.get("filters"), user) - q = add_rating_column(q) - o = options.get("order_by") - if o: - if o == 'comments': - q = q.add_columns(sa.func.count(Reaction.id).label(o)) - q = q.join(Reaction, Shout.slug == Reaction.shout) - q = q.filter(Reaction.body.is_not(None)) - elif o == 'reacted': - q = q.join( - Reaction - ).add_columns( - sa.func.max(Reaction.createdAt).label(o) - ) - elif o == 'views': - q = q.join(ViewedEntry) - q = q.add_columns(sa.func.sum(ViewedEntry.amount).label(o)) - order_by = o - else: - order_by = Shout.createdAt + order_by = options.get("order_by", Shout.createdAt) + if order_by == 'reacted': + aliased_reaction = aliased(Reaction) + q.outerjoin(aliased_reaction).add_columns(sa.func.max(aliased_reaction.createdAt).label('reacted')) - order_by_desc = True if options.get('order_by_desc') is None else options.get('order_by_desc') - - with_author_captions = False if options.get('with_author_captions') is None else options.get('with_author_captions') + order_by_desc = options.get('order_by_desc', True) query_order_by = desc(order_by) if order_by_desc else asc(order_by) offset = options.get("offset", 0) limit = options.get("limit", 10) + q = q.group_by(Shout.id).order_by(query_order_by).limit(limit).offset(offset) with local_session() as session: - results = session.execute(q).unique() - for [shout, rating] in results: - shout.stat = await ReactedStorage.get_shout_stat(shout.slug, rating) + shouts = [] - author_captions = {} + for [shout, viewed_stat, reacted_stat, commented_stat, rating_stat] in session.execute(q).unique(): + shouts.append(shout) - if with_author_captions: - author_captions_result = session.query(ShoutAuthor).where( - ShoutAuthor.shout.in_(map(lambda result_item: result_item[0].slug, results))).all() - - for author_captions_result_item in author_captions_result: - if author_captions.get(author_captions_result_item.shout) is None: - author_captions[author_captions_result_item.shout] = {} - - author_captions[ - author_captions_result_item.shout - ][ - author_captions_result_item.user - ] = author_captions_result_item.caption - - for author in shout.authors: - author.caption = author_captions[shout.slug][author.slug] + shout.stat = { + "viewed": viewed_stat, + "reacted": reacted_stat, + "commented": commented_stat, + "rating": rating_stat + } return shouts diff --git a/resolvers/zine/reactions.py b/resolvers/zine/reactions.py index e3fcf416..5ac648aa 100644 --- a/resolvers/zine/reactions.py +++ b/resolvers/zine/reactions.py @@ -141,7 +141,6 @@ async def create_reaction(_, info, inp): elif check_to_publish(session, user, reaction): set_published(session, reaction.shout, reaction.createdBy) - ReactedStorage.react(reaction) try: reactions_follow(user, inp["shout"], True) except Exception as e: From 49ac3e97e5b4a10235f2a4679eb67dce820f15f9 Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Mon, 28 Nov 2022 21:29:02 +0100 Subject: [PATCH 03/12] topic stat via queries --- migration/tables/topics.py | 5 +- migration/tables/users.py | 3 +- resolvers/zine/load.py | 88 +++++++++-------------------- resolvers/zine/profile.py | 67 ++++++++++------------ resolvers/zine/reactions.py | 10 +--- resolvers/zine/topics.py | 109 +++++++++++++++++++++++++----------- services/zine/topics.py | 97 -------------------------------- 7 files changed, 139 insertions(+), 240 deletions(-) delete mode 100644 services/zine/topics.py diff --git a/migration/tables/topics.py b/migration/tables/topics.py index 15fcf245..3287adb7 100644 --- a/migration/tables/topics.py +++ b/migration/tables/topics.py @@ -9,9 +9,10 @@ def migrate(entry): topic_dict = { "slug": entry["slug"], "oid": entry["_id"], - "title": entry["title"].replace(" ", " ") + "title": entry["title"].replace(" ", " "), + "body": extract_md(html2text(body_orig), entry["_id"]) } - topic_dict["body"] = extract_md(html2text(body_orig), entry["_id"]) + with local_session() as session: slug = topic_dict["slug"] topic = session.query(Topic).filter(Topic.slug == slug).first() or Topic.create( diff --git a/migration/tables/users.py b/migration/tables/users.py index 6db7a243..bca86a76 100644 --- a/migration/tables/users.py +++ b/migration/tables/users.py @@ -23,8 +23,9 @@ def migrate(entry): "notifications": [], "links": [], "name": "anonymous", + "password": entry["services"]["password"].get("bcrypt") } - user_dict["password"] = entry["services"]["password"].get("bcrypt") + if "updatedAt" in entry: user_dict["updatedAt"] = parse(entry["updatedAt"]) if "wasOnineAt" in entry: diff --git a/resolvers/zine/load.py b/resolvers/zine/load.py index 79dcffb9..1408d135 100644 --- a/resolvers/zine/load.py +++ b/resolvers/zine/load.py @@ -9,72 +9,47 @@ from orm.shout import Shout, ShoutAuthor from orm.reaction import Reaction, ReactionKind -def add_viewed_stat_column(q): - return q.outerjoin(ViewedEntry).add_columns(sa.func.sum(ViewedEntry.amount).label('viewed_stat')) +def add_stat_columns(q): + q = q.outerjoin(ViewedEntry).add_columns(sa.func.sum(ViewedEntry.amount).label('viewed_stat')) - -def add_reacted_stat_column(q): aliased_reaction = aliased(Reaction) - return q.outerjoin(aliased_reaction).add_columns(sa.func.count(aliased_reaction.id).label('reacted_stat')) + q = q.outerjoin(aliased_reaction).add_columns( + sa.func.sum( + aliased_reaction.id + ).label('reacted_stat'), + sa.func.sum( + case( + (aliased_reaction.body.is_not(None), 1), + else_=0 + ) + ).label('commented_stat'), + sa.func.sum(case( + (aliased_reaction.kind == ReactionKind.AGREE, 1), + (aliased_reaction.kind == ReactionKind.DISAGREE, -1), + (aliased_reaction.kind == ReactionKind.PROOF, 1), + (aliased_reaction.kind == ReactionKind.DISPROOF, -1), + (aliased_reaction.kind == ReactionKind.ACCEPT, 1), + (aliased_reaction.kind == ReactionKind.REJECT, -1), + (aliased_reaction.kind == ReactionKind.LIKE, 1), + (aliased_reaction.kind == ReactionKind.DISLIKE, -1), + else_=0) + ).label('rating_stat')) -def add_commented_stat_column(q): - aliased_reaction = aliased(Reaction) - return q.outerjoin( - aliased_reaction, - aliased_reaction.shout == Shout.slug and aliased_reaction.body.is_not(None) - ).add_columns(sa.func.count(aliased_reaction.id).label('commented_stat')) - - -def add_rating_stat_column(q): - return q.outerjoin(Reaction).add_columns(sa.func.sum(case( - (Reaction.kind == ReactionKind.AGREE, 1), - (Reaction.kind == ReactionKind.DISAGREE, -1), - (Reaction.kind == ReactionKind.PROOF, 1), - (Reaction.kind == ReactionKind.DISPROOF, -1), - (Reaction.kind == ReactionKind.ACCEPT, 1), - (Reaction.kind == ReactionKind.REJECT, -1), - (Reaction.kind == ReactionKind.LIKE, 1), - (Reaction.kind == ReactionKind.DISLIKE, -1), - else_=0 - )).label('rating_stat')) - - -# def calc_reactions(q): -# aliased_reaction = aliased(Reaction) -# return q.join(aliased_reaction).add_columns( -# sa.func.sum(case( -# (aliased_reaction.kind == ReactionKind.AGREE, 1), -# (aliased_reaction.kind == ReactionKind.DISAGREE, -1), -# (aliased_reaction.kind == ReactionKind.PROOF, 1), -# (aliased_reaction.kind == ReactionKind.DISPROOF, -1), -# (aliased_reaction.kind == ReactionKind.ACCEPT, 1), -# (aliased_reaction.kind == ReactionKind.REJECT, -1), -# (aliased_reaction.kind == ReactionKind.LIKE, 1), -# (aliased_reaction.kind == ReactionKind.DISLIKE, -1), -# else_=0) -# ).label('rating'), -# sa.func.sum( -# case( -# (aliased_reaction.body.is_not(None), 1), -# else_=0 -# ) -# ).label('commented'), -# sa.func.sum( -# aliased_reaction.id -# ).label('reacted') -# ) + return q def apply_filters(q, filters, user=None): - filters = {} if filters is None else filters + if filters.get("reacted") and user: q.join(Reaction, Reaction.createdBy == user.slug) + v = filters.get("visibility") if v == "public": q = q.filter(Shout.visibility == filters.get("visibility")) if v == "community": q = q.filter(Shout.visibility.in_(["public", "community"])) + if filters.get("layout"): q = q.filter(Shout.layout == filters.get("layout")) if filters.get("author"): @@ -88,14 +63,7 @@ def apply_filters(q, filters, user=None): if filters.get("days"): before = datetime.now(tz=timezone.utc) - timedelta(days=int(filters.get("days")) or 30) q = q.filter(Shout.createdAt > before) - return q - -def add_stat_columns(q): - q = add_viewed_stat_column(q) - q = add_reacted_stat_column(q) - q = add_commented_stat_column(q) - q = add_rating_stat_column(q) return q @@ -162,7 +130,7 @@ async def load_shouts_by(_, info, options): q = add_stat_columns(q) user = info.context["request"].user - q = apply_filters(q, options.get("filters"), user) + q = apply_filters(q, options.get("filters", {}), user) order_by = options.get("order_by", Shout.createdAt) if order_by == 'reacted': diff --git a/resolvers/zine/profile.py b/resolvers/zine/profile.py index 115f1af1..b28e7c24 100644 --- a/resolvers/zine/profile.py +++ b/resolvers/zine/profile.py @@ -1,22 +1,24 @@ from typing import List from datetime import datetime, timedelta, timezone -from sqlalchemy import and_, func +from sqlalchemy import and_, func, select from sqlalchemy.orm import selectinload from auth.authenticate import login_required from base.orm import local_session from base.resolvers import mutation, query from orm.reaction import Reaction -from orm.shout import ShoutAuthor +from orm.shout import ShoutAuthor, ShoutTopic from orm.topic import Topic, TopicFollower from orm.user import AuthorFollower, Role, User, UserRating, UserRole # from .community import followed_communities from resolvers.inbox.unread import get_total_unread_counter +from resolvers.zine.topics import followed_by_user + async def user_subscriptions(slug: str): return { - "unread": await get_total_unread_counter(slug), # unread inbox messages counter + "unread": await get_total_unread_counter(slug), # unread inbox messages counter "topics": [t.slug for t in await followed_topics(slug)], # followed topics slugs "authors": [a.slug for a in await followed_authors(slug)], # followed authors slugs "reactions": await followed_reactions(slug) @@ -66,17 +68,7 @@ async def get_followed_topics(_, info, slug) -> List[Topic]: async def followed_topics(slug): - topics = [] - with local_session() as session: - topics = ( - session.query(Topic) - .join(TopicFollower) - .where(TopicFollower.follower == slug) - .all() - ) - for topic in topics: - topic.stat = await get_topic_stat(topic.slug) - return topics + return followed_by_user(slug) @query.field("userFollowedAuthors") @@ -89,9 +81,9 @@ async def followed_authors(slug) -> List[User]: with local_session() as session: authors = ( session.query(User) - .join(AuthorFollower, User.slug == AuthorFollower.author) - .where(AuthorFollower.follower == slug) - .all() + .join(AuthorFollower, User.slug == AuthorFollower.author) + .where(AuthorFollower.follower == slug) + .all() ) for author in authors: author.stat = await get_author_stat(author.slug) @@ -103,9 +95,9 @@ async def user_followers(_, _info, slug) -> List[User]: with local_session() as session: users = ( session.query(User) - .join(AuthorFollower, User.slug == AuthorFollower.follower) - .where(AuthorFollower.author == slug) - .all() + .join(AuthorFollower, User.slug == AuthorFollower.follower) + .where(AuthorFollower.author == slug) + .all() ) return users @@ -115,10 +107,10 @@ async def get_user_roles(slug): user = session.query(User).where(User.slug == slug).first() roles = ( session.query(Role) - .options(selectinload(Role.permissions)) - .join(UserRole) - .where(UserRole.user_id == user.id) - .all() + .options(selectinload(Role.permissions)) + .join(UserRole) + .where(UserRole.user_id == user.id) + .all() ) return roles @@ -144,8 +136,8 @@ async def rate_user(_, info, rated_userslug, value): with local_session() as session: rating = ( session.query(UserRating) - .filter(and_(UserRating.rater == user.slug, UserRating.user == rated_userslug)) - .first() + .filter(and_(UserRating.rater == user.slug, UserRating.user == rated_userslug)) + .first() ) if rating: rating.value = value @@ -171,12 +163,12 @@ def author_unfollow(user, slug): with local_session() as session: flw = ( session.query(AuthorFollower) - .filter( + .filter( and_( AuthorFollower.follower == user.slug, AuthorFollower.author == slug ) ) - .first() + .first() ) if not flw: raise Exception("[resolvers.profile] follower not exist, cant unfollow") @@ -204,7 +196,6 @@ async def get_author(_, _info, slug): @query.field("loadAuthorsBy") async def load_authors_by(_, info, by, limit, offset): - authors = [] with local_session() as session: aq = session.query(User) if by.get("slug"): @@ -212,24 +203,26 @@ async def load_authors_by(_, info, by, limit, offset): elif by.get("name"): aq = aq.filter(User.name.ilike(f"%{by['name']}%")) elif by.get("topic"): - aaa = list(map(lambda a: a.slug, TopicStat.authors_by_topic.get(by["topic"]))) - aq = aq.filter(User.name._in(aaa)) + aq = aq.join(ShoutAuthor).join(ShoutTopic).where(ShoutTopic.topic == by["topic"]) if by.get("lastSeen"): # in days days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["lastSeen"]) aq = aq.filter(User.lastSeen > days_before) elif by.get("createdAt"): # in days days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["createdAt"]) aq = aq.filter(User.createdAt > days_before) + aq = aq.group_by( User.id ).order_by( by.get("order") or "createdAt" ).limit(limit).offset(offset) + print(aq) - authors = list(map(lambda r: r.User, session.execute(aq))) - if by.get("stat"): - for a in authors: - a.stat = await get_author_stat(a.slug) - authors = list(set(authors)) - # authors = sorted(authors, key=lambda a: a["stat"].get(by.get("stat"))) + + authors = [] + for [author] in session.execute(aq): + if by.get("stat"): + author.stat = await get_author_stat(author.slug) + authors.append(author) + return authors diff --git a/resolvers/zine/reactions.py b/resolvers/zine/reactions.py index 5647c253..dafb1db4 100644 --- a/resolvers/zine/reactions.py +++ b/resolvers/zine/reactions.py @@ -10,15 +10,6 @@ from orm.shout import Shout, ShoutReactionsFollower from orm.user import User -async def get_reaction_stat(reaction_id): - return { - # "viewed": await ViewedStorage.get_reaction(reaction_id), - "reacted": len(await ReactedStorage.get_reaction(reaction_id)), - "rating": await ReactedStorage.get_reaction_rating(reaction_id), - "commented": len(await ReactedStorage.get_reaction_comments(reaction_id)), - } - - def reactions_follow(user: User, slug: str, auto=False): with local_session() as session: following = ( @@ -205,6 +196,7 @@ async def delete_reaction(_, info, rid): session.commit() return {} + @query.field("loadReactionsBy") async def load_reactions_by(_, _info, by, limit=50, offset=0): """ diff --git a/resolvers/zine/topics.py b/resolvers/zine/topics.py index 881299b1..f448a3b2 100644 --- a/resolvers/zine/topics.py +++ b/resolvers/zine/topics.py @@ -1,52 +1,94 @@ import sqlalchemy as sa -from sqlalchemy import and_, select +from sqlalchemy import and_, select, distinct + from auth.authenticate import login_required from base.orm import local_session from base.resolvers import mutation, query +from orm.shout import ShoutTopic, ShoutAuthor from orm.topic import Topic, TopicFollower from orm import Shout -async def get_topic_stat(slug): - return { - "shouts": len(TopicStat.shouts_by_topic.get(slug, {}).keys()), - "authors": len(TopicStat.authors_by_topic.get(slug, {}).keys()), - "followers": len(TopicStat.followers_by_topic.get(slug, {}).keys()) + +def add_topic_stat_columns(q): + q = q.outerjoin(ShoutTopic, Topic.slug == ShoutTopic.topic).add_columns( + sa.func.count(distinct(ShoutTopic.shout)).label('shouts_stat') + ).outerjoin(ShoutAuthor, ShoutTopic.shout == ShoutAuthor.shout).add_columns( + sa.func.count(distinct(ShoutAuthor.user)).label('authors_stat') + ).outerjoin(TopicFollower, + and_( + TopicFollower.topic == Topic.slug, + TopicFollower.follower == ShoutAuthor.user + )).add_columns( + sa.func.count(distinct(TopicFollower.follower)).label('followers_stat') + ) + + q = q.group_by(Topic.id) + + return q + + +def add_stat(topic, stat_columns): + [shouts_stat, authors_stat, followers_stat] = stat_columns + topic.stat = { + "shouts": shouts_stat, + "authors": authors_stat, + "followers": followers_stat } + return topic + + +def get_topics_from_query(q): + topics = [] + with local_session() as session: + for [topic, *stat_columns] in session.execute(q): + topic = add_stat(topic, stat_columns) + topics.append(topic) + + return topics + + +def followed_by_user(user_slug): + q = select(Topic) + q = add_topic_stat_columns(q) + q = q.where(TopicFollower.follower == user_slug) + + return get_topics_from_query(q) + + @query.field("topicsAll") async def topics_all(_, _info): - topics = await TopicStorage.get_topics_all() - for topic in topics: - topic.stat = await get_topic_stat(topic.slug) - return topics + q = select(Topic) + q = add_topic_stat_columns(q) + + return get_topics_from_query(q) @query.field("topicsByCommunity") async def topics_by_community(_, info, community): - topics = await TopicStorage.get_topics_by_community(community) - for topic in topics: - topic.stat = await get_topic_stat(topic.slug) - return topics + q = select(Topic).where(Topic.community == community) + q = add_topic_stat_columns(q) + + return get_topics_from_query(q) @query.field("topicsByAuthor") async def topics_by_author(_, _info, author): - shouts = TopicStorage.get_topics_by_author(author) - author_topics = set() - for s in shouts: - for tpc in s.topics: - tpc = await TopicStorage.topics[tpc.slug] - tpc.stat = await get_topic_stat(tpc.slug) - author_topics.add(tpc) - return list(author_topics) + q = select(Topic) + q = add_topic_stat_columns(q) + q = q.where(ShoutAuthor.user == author) + + return get_topics_from_query(q) @query.field("getTopic") async def get_topic(_, _info, slug): - t = TopicStorage.topics[slug] - t.stat = await get_topic_stat(slug) - return t + q = select(Topic).where(Topic.slug == slug) + q = add_topic_stat_columns(q) + + topics = get_topics_from_query(q) + return topics[0] @mutation.field("createTopic") @@ -57,7 +99,7 @@ async def create_topic(_, _info, inp): new_topic = Topic.create(**inp) session.add(new_topic) session.commit() - await TopicStorage.update_topic(new_topic) + return {"topic": new_topic} @@ -72,7 +114,7 @@ async def update_topic(_, _info, inp): else: topic.update(**inp) session.commit() - await TopicStorage.update_topic(topic.slug) + return {"topic": topic} @@ -81,7 +123,6 @@ async def topic_follow(user, slug): following = TopicFollower.create(topic=slug, follower=user.slug) session.add(following) session.commit() - await TopicStorage.update_topic(slug) async def topic_unfollow(user, slug): @@ -99,13 +140,13 @@ async def topic_unfollow(user, slug): else: session.delete(sub) session.commit() - await TopicStorage.update_topic(slug) @query.field("topicsRandom") async def topics_random(_, info, amount=12): - with local_session() as session: - q = select(Topic).join(Shout).group_by(Topic.id).having(sa.func.count(Shout.id) > 2).order_by( - sa.func.random()).limit(amount) - random_topics = list(map(lambda result_item: result_item.Topic, session.execute(q))) - return random_topics + q = select(Topic) + q = add_topic_stat_columns(q) + q = q.join(Shout, ShoutTopic.shout == Shout.slug).group_by(Topic.id).having(sa.func.count(Shout.id) > 2) + q = q.order_by(sa.func.random()).limit(amount) + + return get_topics_from_query(q) diff --git a/services/zine/topics.py b/services/zine/topics.py deleted file mode 100644 index 0aa6af7a..00000000 --- a/services/zine/topics.py +++ /dev/null @@ -1,97 +0,0 @@ -import asyncio -from base.orm import local_session -from orm.topic import Topic -from orm.shout import Shout -import sqlalchemy as sa -from sqlalchemy import select - - -class TopicStorage: - topics = {} - lock = asyncio.Lock() - random_topics = [] - - @staticmethod - def init(session): - self = TopicStorage - topics = session.query(Topic) - self.topics = dict([(topic.slug, topic) for topic in topics]) - for tpc in self.topics.values(): - # self.load_parents(tpc) - pass - - print("[zine.topics] %d precached" % len(self.topics.keys())) - - # @staticmethod - # def load_parents(topic): - # self = TopicStorage - # parents = [] - # for parent in self.topics.values(): - # if topic.slug in parent.children: - # parents.append(parent.slug) - # topic.parents = parents - # return topic - - @staticmethod - def get_random_topics(amount): - return TopicStorage.random_topics[0:amount] - - @staticmethod - def renew_topics_random(): - with local_session() as session: - q = select(Topic).join(Shout).group_by(Topic.id).having(sa.func.count(Shout.id) > 2).order_by( - sa.func.random()).limit(50) - TopicStorage.random_topics = list(map( - lambda result_item: result_item.Topic, session.execute(q) - )) - - @staticmethod - async def worker(): - self = TopicStorage - async with self.lock: - while True: - try: - self.renew_topics_random() - except Exception as err: - print("[zine.topics] error %s" % (err)) - await asyncio.sleep(300) # 5 mins - - @staticmethod - async def get_topics_all(): - self = TopicStorage - async with self.lock: - return list(self.topics.values()) - - @staticmethod - async def get_topics_by_slugs(slugs): - self = TopicStorage - async with self.lock: - if not slugs: - return self.topics.values() - topics = filter(lambda topic: topic.slug in slugs, self.topics.values()) - return list(topics) - - @staticmethod - async def get_topics_by_community(community): - self = TopicStorage - async with self.lock: - topics = filter( - lambda topic: topic.community == community, self.topics.values() - ) - return list(topics) - - @staticmethod - async def get_topics_by_author(author): - self = TopicStorage - async with self.lock: - topics = filter( - lambda topic: topic.community == author, self.topics.values() - ) - return list(topics) - - @staticmethod - async def update_topic(topic): - self = TopicStorage - async with self.lock: - self.topics[topic.slug] = topic - # self.load_parents(topic) From e5b3bdc63c72eca98db3d575f6d5a5fb9cde3929 Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Mon, 28 Nov 2022 21:34:42 +0100 Subject: [PATCH 04/12] ShoutAuthor storage removed, minor fixes --- resolvers/zine/load.py | 13 +++++----- resolvers/zine/reactions.py | 31 +++++++++++++++++++++-- resolvers/zine/topics.py | 15 +++++------ services/zine/shoutauthor.py | 49 ------------------------------------ 4 files changed, 41 insertions(+), 67 deletions(-) delete mode 100644 services/zine/shoutauthor.py diff --git a/resolvers/zine/load.py b/resolvers/zine/load.py index 1408d135..cfb9a413 100644 --- a/resolvers/zine/load.py +++ b/resolvers/zine/load.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta, timezone -import sqlalchemy as sa from sqlalchemy.orm import joinedload, aliased -from sqlalchemy.sql.expression import desc, asc, select, case +from sqlalchemy.sql.expression import desc, asc, select, case, func from base.orm import local_session from base.resolvers import query from orm import ViewedEntry @@ -10,21 +9,21 @@ from orm.reaction import Reaction, ReactionKind def add_stat_columns(q): - q = q.outerjoin(ViewedEntry).add_columns(sa.func.sum(ViewedEntry.amount).label('viewed_stat')) + q = q.outerjoin(ViewedEntry).add_columns(func.sum(ViewedEntry.amount).label('viewed_stat')) aliased_reaction = aliased(Reaction) q = q.outerjoin(aliased_reaction).add_columns( - sa.func.sum( + func.sum( aliased_reaction.id ).label('reacted_stat'), - sa.func.sum( + func.sum( case( (aliased_reaction.body.is_not(None), 1), else_=0 ) ).label('commented_stat'), - sa.func.sum(case( + func.sum(case( (aliased_reaction.kind == ReactionKind.AGREE, 1), (aliased_reaction.kind == ReactionKind.DISAGREE, -1), (aliased_reaction.kind == ReactionKind.PROOF, 1), @@ -135,7 +134,7 @@ async def load_shouts_by(_, info, options): order_by = options.get("order_by", Shout.createdAt) if order_by == 'reacted': aliased_reaction = aliased(Reaction) - q.outerjoin(aliased_reaction).add_columns(sa.func.max(aliased_reaction.createdAt).label('reacted')) + q.outerjoin(aliased_reaction).add_columns(func.max(aliased_reaction.createdAt).label('reacted')) order_by_desc = options.get('order_by_desc', True) diff --git a/resolvers/zine/reactions.py b/resolvers/zine/reactions.py index dafb1db4..4c239d19 100644 --- a/resolvers/zine/reactions.py +++ b/resolvers/zine/reactions.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta, timezone -from sqlalchemy import and_, asc, desc, select, text, func +from sqlalchemy import and_, asc, desc, select, text, func, case from sqlalchemy.orm import aliased - from auth.authenticate import login_required from base.orm import local_session from base.resolvers import mutation, query @@ -10,6 +9,34 @@ from orm.shout import Shout, ShoutReactionsFollower from orm.user import User +def add_reaction_stat_columns(q): + aliased_reaction = aliased(Reaction) + + q = q.outerjoin(aliased_reaction).add_columns( + func.sum( + aliased_reaction.id + ).label('reacted_stat'), + func.sum( + case( + (aliased_reaction.body.is_not(None), 1), + else_=0 + ) + ).label('commented_stat'), + func.sum(case( + (aliased_reaction.kind == ReactionKind.AGREE, 1), + (aliased_reaction.kind == ReactionKind.DISAGREE, -1), + (aliased_reaction.kind == ReactionKind.PROOF, 1), + (aliased_reaction.kind == ReactionKind.DISPROOF, -1), + (aliased_reaction.kind == ReactionKind.ACCEPT, 1), + (aliased_reaction.kind == ReactionKind.REJECT, -1), + (aliased_reaction.kind == ReactionKind.LIKE, 1), + (aliased_reaction.kind == ReactionKind.DISLIKE, -1), + else_=0) + ).label('rating_stat')) + + return q + + def reactions_follow(user: User, slug: str, auto=False): with local_session() as session: following = ( diff --git a/resolvers/zine/topics.py b/resolvers/zine/topics.py index f448a3b2..459d95c8 100644 --- a/resolvers/zine/topics.py +++ b/resolvers/zine/topics.py @@ -1,6 +1,4 @@ -import sqlalchemy as sa -from sqlalchemy import and_, select, distinct - +from sqlalchemy import and_, select, distinct, func from auth.authenticate import login_required from base.orm import local_session from base.resolvers import mutation, query @@ -11,15 +9,15 @@ from orm import Shout def add_topic_stat_columns(q): q = q.outerjoin(ShoutTopic, Topic.slug == ShoutTopic.topic).add_columns( - sa.func.count(distinct(ShoutTopic.shout)).label('shouts_stat') + func.count(distinct(ShoutTopic.shout)).label('shouts_stat') ).outerjoin(ShoutAuthor, ShoutTopic.shout == ShoutAuthor.shout).add_columns( - sa.func.count(distinct(ShoutAuthor.user)).label('authors_stat') + func.count(distinct(ShoutAuthor.user)).label('authors_stat') ).outerjoin(TopicFollower, and_( TopicFollower.topic == Topic.slug, TopicFollower.follower == ShoutAuthor.user )).add_columns( - sa.func.count(distinct(TopicFollower.follower)).label('followers_stat') + func.count(distinct(TopicFollower.follower)).label('followers_stat') ) q = q.group_by(Topic.id) @@ -56,7 +54,6 @@ def followed_by_user(user_slug): return get_topics_from_query(q) - @query.field("topicsAll") async def topics_all(_, _info): q = select(Topic) @@ -146,7 +143,7 @@ async def topic_unfollow(user, slug): async def topics_random(_, info, amount=12): q = select(Topic) q = add_topic_stat_columns(q) - q = q.join(Shout, ShoutTopic.shout == Shout.slug).group_by(Topic.id).having(sa.func.count(Shout.id) > 2) - q = q.order_by(sa.func.random()).limit(amount) + q = q.join(Shout, ShoutTopic.shout == Shout.slug).group_by(Topic.id).having(func.count(Shout.id) > 2) + q = q.order_by(func.random()).limit(amount) return get_topics_from_query(q) diff --git a/services/zine/shoutauthor.py b/services/zine/shoutauthor.py deleted file mode 100644 index 7d2a3d1d..00000000 --- a/services/zine/shoutauthor.py +++ /dev/null @@ -1,49 +0,0 @@ -import asyncio -import time -from base.orm import local_session -from orm.shout import ShoutAuthor - - -class ShoutAuthorStorage: - authors_by_shout = {} - lock = asyncio.Lock() - # period = 30 * 60 # sec - - @staticmethod - async def load_captions(session): - self = ShoutAuthorStorage - sas = session.query(ShoutAuthor).all() - for sa in sas: - self.authors_by_shout[sa.shout] = self.authors_by_shout.get(sa.shout, {}) - self.authors_by_shout[sa.shout][sa.user] = sa.caption - print("[zine.authors] ⎧ %d shouts indexed by authors" % len(self.authors_by_shout)) - - @staticmethod - async def get_author_caption(shout, author): - self = ShoutAuthorStorage - async with self.lock: - return self.authors_by_shout.get(shout, {}).get(author) - - @staticmethod - async def set_author_caption(shout, author, caption): - self = ShoutAuthorStorage - async with self.lock: - self.authors_by_shout[shout] = self.authors_by_shout.get(shout, {}) - self.authors_by_shout[shout][author] = caption - return { - "error": None, - } - - @staticmethod - async def worker(): - self = ShoutAuthorStorage - async with self.lock: - # while True: - try: - with local_session() as session: - ts = time.time() - await self.load_captions(session) - print("[zine.authors] ⎩ load_captions took %fs " % (time.time() - ts)) - except Exception as err: - print("[zine.authors] ⎩ error indexing by author: %s" % (err)) - # await asyncio.sleep(self.period) From 99dcfca89a0a76e9af461da79fa67d5362b3f9f3 Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Mon, 28 Nov 2022 23:16:39 +0100 Subject: [PATCH 05/12] reaction stat, author stat --- resolvers/zine/_common.py | 31 +++++++ resolvers/zine/load.py | 27 +----- resolvers/zine/profile.py | 165 +++++++++++++++++++----------------- resolvers/zine/reactions.py | 51 ++++------- 4 files changed, 136 insertions(+), 138 deletions(-) create mode 100644 resolvers/zine/_common.py diff --git a/resolvers/zine/_common.py b/resolvers/zine/_common.py new file mode 100644 index 00000000..7046be92 --- /dev/null +++ b/resolvers/zine/_common.py @@ -0,0 +1,31 @@ +from sqlalchemy import func, case +from sqlalchemy.orm import aliased +from orm.reaction import Reaction, ReactionKind + + +def add_common_stat_columns(q): + aliased_reaction = aliased(Reaction) + + q = q.outerjoin(aliased_reaction).add_columns( + func.sum( + aliased_reaction.id + ).label('reacted_stat'), + func.sum( + case( + (aliased_reaction.body.is_not(None), 1), + else_=0 + ) + ).label('commented_stat'), + func.sum(case( + (aliased_reaction.kind == ReactionKind.AGREE, 1), + (aliased_reaction.kind == ReactionKind.DISAGREE, -1), + (aliased_reaction.kind == ReactionKind.PROOF, 1), + (aliased_reaction.kind == ReactionKind.DISPROOF, -1), + (aliased_reaction.kind == ReactionKind.ACCEPT, 1), + (aliased_reaction.kind == ReactionKind.REJECT, -1), + (aliased_reaction.kind == ReactionKind.LIKE, 1), + (aliased_reaction.kind == ReactionKind.DISLIKE, -1), + else_=0) + ).label('rating_stat')) + + return q diff --git a/resolvers/zine/load.py b/resolvers/zine/load.py index cfb9a413..bc76db02 100644 --- a/resolvers/zine/load.py +++ b/resolvers/zine/load.py @@ -6,36 +6,13 @@ from base.resolvers import query from orm import ViewedEntry from orm.shout import Shout, ShoutAuthor from orm.reaction import Reaction, ReactionKind +from resolvers.zine._common import add_common_stat_columns def add_stat_columns(q): q = q.outerjoin(ViewedEntry).add_columns(func.sum(ViewedEntry.amount).label('viewed_stat')) - aliased_reaction = aliased(Reaction) - - q = q.outerjoin(aliased_reaction).add_columns( - func.sum( - aliased_reaction.id - ).label('reacted_stat'), - func.sum( - case( - (aliased_reaction.body.is_not(None), 1), - else_=0 - ) - ).label('commented_stat'), - func.sum(case( - (aliased_reaction.kind == ReactionKind.AGREE, 1), - (aliased_reaction.kind == ReactionKind.DISAGREE, -1), - (aliased_reaction.kind == ReactionKind.PROOF, 1), - (aliased_reaction.kind == ReactionKind.DISPROOF, -1), - (aliased_reaction.kind == ReactionKind.ACCEPT, 1), - (aliased_reaction.kind == ReactionKind.REJECT, -1), - (aliased_reaction.kind == ReactionKind.LIKE, 1), - (aliased_reaction.kind == ReactionKind.DISLIKE, -1), - else_=0) - ).label('rating_stat')) - - return q + return add_common_stat_columns(q) def apply_filters(q, filters, user=None): diff --git a/resolvers/zine/profile.py b/resolvers/zine/profile.py index b28e7c24..be38eeff 100644 --- a/resolvers/zine/profile.py +++ b/resolvers/zine/profile.py @@ -1,14 +1,14 @@ from typing import List from datetime import datetime, timedelta, timezone -from sqlalchemy import and_, func, select -from sqlalchemy.orm import selectinload +from sqlalchemy import and_, func, distinct, select +from sqlalchemy.orm import aliased, joinedload from auth.authenticate import login_required from base.orm import local_session from base.resolvers import mutation, query from orm.reaction import Reaction from orm.shout import ShoutAuthor, ShoutTopic -from orm.topic import Topic, TopicFollower +from orm.topic import Topic from orm.user import AuthorFollower, Role, User, UserRating, UserRole # from .community import followed_communities @@ -16,6 +16,51 @@ from resolvers.inbox.unread import get_total_unread_counter from resolvers.zine.topics import followed_by_user +def add_author_stat_columns(q): + author_followers = aliased(AuthorFollower) + author_following = aliased(AuthorFollower) + + q = q.outerjoin(ShoutAuthor).add_columns( + func.count(distinct(ShoutAuthor.shout)).label('shouts_stat') + ).outerjoin(author_followers, author_followers.author == User.slug).add_columns( + func.count(distinct(author_followers.follower)).label('followers_stat') + ).outerjoin(author_following, author_following.follower == User.slug).add_columns( + func.count(distinct(author_following.author)).label('followings_stat') + ).outerjoin(UserRating).add_columns( + # TODO: check + func.sum(UserRating.value).label('rating_stat') + ).outerjoin(Reaction, and_(Reaction.createdBy == User.slug, Reaction.body.is_not(None))).add_columns( + func.count(distinct(Reaction.id)).label('commented_stat') + ) + + q = q.group_by(User.id) + + return q + + +def add_stat(author, stat_columns): + [shouts_stat, followers_stat, followings_stat, rating_stat, commented_stat] = stat_columns + author.stat = { + "shouts": shouts_stat, + "followers": followers_stat, + "followings": followings_stat, + "rating": rating_stat, + "commented": commented_stat + } + + return author + + +def get_authors_from_query(q): + authors = [] + with local_session() as session: + for [author, *stat_columns] in session.execute(q): + author = add_stat(author, stat_columns) + authors.append(author) + + return authors + + async def user_subscriptions(slug: str): return { "unread": await get_total_unread_counter(slug), # unread inbox messages counter @@ -26,23 +71,6 @@ async def user_subscriptions(slug: str): } -async def get_author_stat(slug): - with local_session() as session: - return { - "shouts": session.query(ShoutAuthor).where(ShoutAuthor.user == slug).count(), - "followers": session.query(AuthorFollower).where(AuthorFollower.author == slug).count(), - "followings": session.query(AuthorFollower).where(AuthorFollower.follower == slug).count(), - "rating": session.query(func.sum(UserRating.value)).where(UserRating.user == slug).first(), - "commented": session.query( - Reaction.id - ).where( - Reaction.createdBy == slug - ).filter( - Reaction.body.is_not(None) - ).count() - } - - # @query.field("userFollowedDiscussions") @login_required async def followed_discussions(_, info, slug) -> List[Topic]: @@ -77,29 +105,20 @@ async def get_followed_authors(_, _info, slug) -> List[User]: async def followed_authors(slug) -> List[User]: - authors = [] - with local_session() as session: - authors = ( - session.query(User) - .join(AuthorFollower, User.slug == AuthorFollower.author) - .where(AuthorFollower.follower == slug) - .all() - ) - for author in authors: - author.stat = await get_author_stat(author.slug) - return authors + q = select(User) + q = add_author_stat_columns(q) + q = q.join(AuthorFollower).where(AuthorFollower.follower == slug) + + return get_authors_from_query(q) @query.field("userFollowers") async def user_followers(_, _info, slug) -> List[User]: - with local_session() as session: - users = ( - session.query(User) - .join(AuthorFollower, User.slug == AuthorFollower.follower) - .where(AuthorFollower.author == slug) - .all() - ) - return users + q = select(User) + q = add_author_stat_columns(q) + q = q.join(AuthorFollower).where(AuthorFollower.author == slug) + + return get_authors_from_query(q) async def get_user_roles(slug): @@ -107,11 +126,12 @@ async def get_user_roles(slug): user = session.query(User).where(User.slug == slug).first() roles = ( session.query(Role) - .options(selectinload(Role.permissions)) + .options(joinedload(Role.permissions)) .join(UserRole) .where(UserRole.user_id == user.id) .all() ) + return roles @@ -179,50 +199,41 @@ def author_unfollow(user, slug): @query.field("authorsAll") async def get_authors_all(_, _info): - with local_session() as session: - authors = session.query(User).join(ShoutAuthor).all() - for author in authors: - author.stat = await get_author_stat(author.slug) - return authors + q = select(User) + q = add_author_stat_columns(q) + q = q.join(ShoutAuthor) + + return get_authors_from_query(q) @query.field("getAuthor") async def get_author(_, _info, slug): - with local_session() as session: - author = session.query(User).where(User.slug == slug).first() - author.stat = await get_author_stat(author.slug) - return author + q = select(User).where(User.slug == slug) + q = add_author_stat_columns(q) + + authors = get_authors_from_query(q) + return authors[0] @query.field("loadAuthorsBy") async def load_authors_by(_, info, by, limit, offset): - with local_session() as session: - aq = session.query(User) - if by.get("slug"): - aq = aq.filter(User.slug.ilike(f"%{by['slug']}%")) - elif by.get("name"): - aq = aq.filter(User.name.ilike(f"%{by['name']}%")) - elif by.get("topic"): - aq = aq.join(ShoutAuthor).join(ShoutTopic).where(ShoutTopic.topic == by["topic"]) - if by.get("lastSeen"): # in days - days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["lastSeen"]) - aq = aq.filter(User.lastSeen > days_before) - elif by.get("createdAt"): # in days - days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["createdAt"]) - aq = aq.filter(User.createdAt > days_before) + q = select(User) + q = add_author_stat_columns(q) + if by.get("slug"): + q = q.filter(User.slug.ilike(f"%{by['slug']}%")) + elif by.get("name"): + q = q.filter(User.name.ilike(f"%{by['name']}%")) + elif by.get("topic"): + q = q.join(ShoutAuthor).join(ShoutTopic).where(ShoutTopic.topic == by["topic"]) + if by.get("lastSeen"): # in days + days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["lastSeen"]) + q = q.filter(User.lastSeen > days_before) + elif by.get("createdAt"): # in days + days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["createdAt"]) + q = q.filter(User.createdAt > days_before) - aq = aq.group_by( - User.id - ).order_by( - by.get("order") or "createdAt" - ).limit(limit).offset(offset) + q = q.order_by( + by.get("order", User.createdAt) + ).limit(limit).offset(offset) - print(aq) - - authors = [] - for [author] in session.execute(aq): - if by.get("stat"): - author.stat = await get_author_stat(author.slug) - authors.append(author) - - return authors + return get_authors_from_query(q) diff --git a/resolvers/zine/reactions.py b/resolvers/zine/reactions.py index 4c239d19..876e4791 100644 --- a/resolvers/zine/reactions.py +++ b/resolvers/zine/reactions.py @@ -1,5 +1,5 @@ from datetime import datetime, timedelta, timezone -from sqlalchemy import and_, asc, desc, select, text, func, case +from sqlalchemy import and_, asc, desc, select, text, func from sqlalchemy.orm import aliased from auth.authenticate import login_required from base.orm import local_session @@ -7,34 +7,11 @@ from base.resolvers import mutation, query from orm.reaction import Reaction, ReactionKind from orm.shout import Shout, ShoutReactionsFollower from orm.user import User +from resolvers.zine._common import add_common_stat_columns def add_reaction_stat_columns(q): - aliased_reaction = aliased(Reaction) - - q = q.outerjoin(aliased_reaction).add_columns( - func.sum( - aliased_reaction.id - ).label('reacted_stat'), - func.sum( - case( - (aliased_reaction.body.is_not(None), 1), - else_=0 - ) - ).label('commented_stat'), - func.sum(case( - (aliased_reaction.kind == ReactionKind.AGREE, 1), - (aliased_reaction.kind == ReactionKind.DISAGREE, -1), - (aliased_reaction.kind == ReactionKind.PROOF, 1), - (aliased_reaction.kind == ReactionKind.DISPROOF, -1), - (aliased_reaction.kind == ReactionKind.ACCEPT, 1), - (aliased_reaction.kind == ReactionKind.REJECT, -1), - (aliased_reaction.kind == ReactionKind.LIKE, 1), - (aliased_reaction.kind == ReactionKind.DISLIKE, -1), - else_=0) - ).label('rating_stat')) - - return q + return add_common_stat_columns(q) def reactions_follow(user: User, slug: str, auto=False): @@ -181,9 +158,9 @@ async def update_reaction(_, info, inp): with local_session() as session: user = session.query(User).where(User.id == user_id).first() q = select(Reaction).filter(Reaction.id == inp.id) - q = calc_reactions(q) + q = add_reaction_stat_columns(q) - [reaction, rating, commented, reacted] = session.execute(q).unique().one() + [reaction, reacted_stat, commented_stat, rating_stat] = session.execute(q).unique().one() if not reaction: return {"error": "invalid reaction id"} @@ -199,9 +176,9 @@ async def update_reaction(_, info, inp): reaction.range = inp.get("range") session.commit() reaction.stat = { - "commented": commented, - "reacted": reacted, - "rating": rating + "commented": commented_stat, + "reacted": reacted_stat, + "rating": rating_stat } return {"reaction": reaction} @@ -269,6 +246,7 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0): q = q.filter(Reaction.createdAt > after) order_way = asc if by.get("sort", "").startswith("-") else desc + # replace "-" -> "" ? order_field = by.get("sort") or Reaction.createdAt q = q.group_by( @@ -277,23 +255,24 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0): order_way(order_field) ) - q = calc_reactions(q) + q = add_reaction_stat_columns(q) q = q.where(Reaction.deletedAt.is_(None)) q = q.limit(limit).offset(offset) reactions = [] with local_session() as session: - for [reaction, user, shout, rating, commented, reacted] in session.execute(q): + for [reaction, user, shout, reacted_stat, commented_stat, rating_stat] in session.execute(q): reaction.createdBy = user reaction.shout = shout reaction.stat = { - "rating": rating, - "commented": commented, - "reacted": reacted + "rating": rating_stat, + "commented": commented_stat, + "reacted": reacted_stat } reactions.append(reaction) + # ? if by.get("stat"): reactions.sort(lambda r: r.stat.get(by["stat"]) or r.createdAt) From b840823fce279e84f5969646869417c5749e16d5 Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Mon, 28 Nov 2022 23:58:23 +0100 Subject: [PATCH 06/12] UserStorege removed, RoleStorage removed --- auth/authenticate.py | 31 ++++++++++++++---- orm/user.py | 4 +-- services/auth/roles.py | 35 -------------------- services/auth/users.py | 72 ------------------------------------------ services/main.py | 10 +++--- 5 files changed, 31 insertions(+), 121 deletions(-) delete mode 100644 services/auth/roles.py delete mode 100644 services/auth/users.py diff --git a/auth/authenticate.py b/auth/authenticate.py index 242d2793..bc5509b3 100644 --- a/auth/authenticate.py +++ b/auth/authenticate.py @@ -2,11 +2,14 @@ from functools import wraps from typing import Optional, Tuple from graphql.type import GraphQLResolveInfo +from sqlalchemy.orm import joinedload, exc from starlette.authentication import AuthenticationBackend from starlette.requests import HTTPConnection from auth.credentials import AuthCredentials, AuthUser -from services.auth.users import UserStorage +from base.orm import local_session +from orm import User, Role + from settings import SESSION_TOKEN_HEADER from auth.tokenstorage import SessionToken from base.exceptions import InvalidToken, OperationNotAllowed, Unauthorized @@ -32,10 +35,26 @@ class JWTAuthenticate(AuthenticationBackend): payload = await SessionToken.verify(token) if payload is None: return AuthCredentials(scopes=[]), AuthUser(user_id=None) - user = await UserStorage.get_user(payload.user_id) + + with local_session() as session: + try: + user = ( + session.query(User).options( + joinedload(User.roles), + joinedload(Role.permissions), + joinedload(User.ratings) + ).filter( + User.id == id + ).one() + ) + except exc.NoResultFound: + user = None + if not user: return AuthCredentials(scopes=[]), AuthUser(user_id=None) - scopes = await user.get_permission() + + scopes = user.get_permission() + return ( AuthCredentials( user_id=payload.user_id, @@ -46,10 +65,10 @@ class JWTAuthenticate(AuthenticationBackend): ) else: InvalidToken("please try again") - except Exception as exc: + except Exception as e: print("[auth.authenticate] session token verify error") - print(exc) - return AuthCredentials(scopes=[], error_message=str(exc)), AuthUser(user_id=None) + print(e) + return AuthCredentials(scopes=[], error_message=str(e)), AuthUser(user_id=None) def login_required(func): diff --git a/orm/user.py b/orm/user.py index 75a2d748..11055d29 100644 --- a/orm/user.py +++ b/orm/user.py @@ -103,12 +103,12 @@ class User(Base): async def get_permission(self): scope = {} - for user_role in self.roles: - role: Role = await RoleStorage.get_role(user_role.id) # type: ignore + for role in self.roles: for p in role.permissions: if p.resource_id not in scope: scope[p.resource_id] = set() scope[p.resource_id].add(p.operation_id) + return scope diff --git a/services/auth/roles.py b/services/auth/roles.py deleted file mode 100644 index b5924dc5..00000000 --- a/services/auth/roles.py +++ /dev/null @@ -1,35 +0,0 @@ -import asyncio - -from sqlalchemy.orm import selectinload - -from orm.rbac import Role - - -class RoleStorage: - roles = {} - lock = asyncio.Lock() - - @staticmethod - def init(session): - self = RoleStorage - roles = session.query(Role).options(selectinload(Role.permissions)).all() - self.roles = dict([(role.id, role) for role in roles]) - print("[auth.roles] %d precached" % len(roles)) - - @staticmethod - async def get_role(id): - self = RoleStorage - async with self.lock: - return self.roles.get(id) - - @staticmethod - async def add_role(role): - self = RoleStorage - async with self.lock: - self.roles[id] = role - - @staticmethod - async def del_role(id): - self = RoleStorage - async with self.lock: - del self.roles[id] diff --git a/services/auth/users.py b/services/auth/users.py deleted file mode 100644 index 2ba636bc..00000000 --- a/services/auth/users.py +++ /dev/null @@ -1,72 +0,0 @@ -import asyncio -from sqlalchemy.orm import selectinload, exc -from orm.user import User -from base.orm import local_session - - -class UserStorage: - users = {} - lock = asyncio.Lock() - - @staticmethod - def init(session): - self = UserStorage - users = ( - session.query(User) - .options(selectinload(User.roles), selectinload(User.ratings)) - .all() - ) - self.users = dict([(user.id, user) for user in users]) - print("[auth.users] %d precached" % len(self.users)) - - @staticmethod - async def get_user(id): - with local_session() as session: - try: - user = ( - session.query(User).options( - selectinload(User.roles), - selectinload(User.ratings) - ).filter( - User.id == id - ).one() - ) - return user - except exc.NoResultFound: - return None - - @staticmethod - async def get_all_users(): - self = UserStorage - async with self.lock: - aaa = list(self.users.values()) - aaa.sort(key=lambda user: user.createdAt) - return aaa - - @staticmethod - async def get_top_users(): - self = UserStorage - async with self.lock: - aaa = list(self.users.values()) - aaa.sort(key=lambda user: user.rating) - return aaa - - @staticmethod - async def get_user_by_slug(slug): - self = UserStorage - async with self.lock: - for user in self.users.values(): - if user.slug == slug: - return user - - @staticmethod - async def add_user(user): - self = UserStorage - async with self.lock: - self.users[user.id] = user - - @staticmethod - async def del_user(id): - self = UserStorage - async with self.lock: - del self.users[id] diff --git a/services/main.py b/services/main.py index f87f8afe..10301b86 100644 --- a/services/main.py +++ b/services/main.py @@ -1,5 +1,3 @@ -from services.auth.roles import RoleStorage -from services.auth.users import UserStorage from services.search import SearchService from services.stat.viewed import ViewedStorage from base.orm import local_session @@ -7,9 +5,9 @@ from base.orm import local_session async def storages_init(): with local_session() as session: - print('[main] initialize storages') - RoleStorage.init(session) - UserStorage.init(session) + print('[main] initialize SearchService') await SearchService.init(session) - session.commit() + print('[main] SearchService initialized') + print('[main] initialize storages') await ViewedStorage.init() + print('[main] storages initialized') From 226aeddecd1f082b2dab2173730fef118764a71a Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Tue, 29 Nov 2022 13:36:46 +0100 Subject: [PATCH 07/12] model changes --- .gitignore | 1 + migration/__init__.py | 3 +- migration/tables/comments.py | 67 +++++++++++++++--------- migration/tables/content_items.py | 84 +++++++++++++++++++------------ migration/tables/users.py | 19 ++++--- orm/collab.py | 3 +- orm/collection.py | 4 +- orm/community.py | 4 +- orm/reaction.py | 8 +-- orm/shout.py | 14 +++--- orm/topic.py | 4 +- orm/user.py | 15 +++--- orm/viewed.py | 4 +- resolvers/create/editor.py | 21 +++++--- resolvers/inbox/search.py | 16 ++++-- resolvers/zine/load.py | 10 ++-- resolvers/zine/profile.py | 47 ++++++++++------- resolvers/zine/reactions.py | 38 ++++++++------ resolvers/zine/topics.py | 32 ++++++------ services/stat/viewed.py | 26 ++++++---- 20 files changed, 254 insertions(+), 166 deletions(-) diff --git a/.gitignore b/.gitignore index b8332f4b..10b8c415 100644 --- a/.gitignore +++ b/.gitignore @@ -148,3 +148,4 @@ dump *dump.sql *.csv dev-server-status.txt +/resetdb.sh diff --git a/migration/__init__.py b/migration/__init__.py index 4a25931d..eaded31b 100644 --- a/migration/__init__.py +++ b/migration/__init__.py @@ -119,8 +119,7 @@ async def shouts_handle(storage, args): # print main counter counter += 1 - line = str(counter + 1) + ": " + shout["slug"] + " @" + author["slug"] - print(line) + print('[migration] shouts_handle %d: %s @%s' % ((counter + 1), shout["slug"], author["slug"])) b = bs4.BeautifulSoup(shout["body"], "html.parser") texts = [shout["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")] diff --git a/migration/tables/comments.py b/migration/tables/comments.py index 9fb099f7..09cfb491 100644 --- a/migration/tables/comments.py +++ b/migration/tables/comments.py @@ -5,8 +5,8 @@ from dateutil.parser import parse as date_parse from base.orm import local_session from migration.html2text import html2text from orm.reaction import Reaction, ReactionKind -from orm.shout import ShoutReactionsFollower -from orm.topic import TopicFollower +from orm.shout import ShoutReactionsFollower, Shout +from orm.topic import TopicFollower, Topic from orm.user import User ts = datetime.now(tz=timezone.utc) @@ -69,8 +69,13 @@ async def migrate(entry, storage): author = session.query(User).filter(User.oid == entry["createdBy"]).first() shout_dict = storage["shouts"]["by_oid"][shout_oid] if shout_dict: - reaction_dict["shout"] = shout_dict["slug"] - reaction_dict["createdBy"] = author.slug if author else "discours" + shout = session.query( + Shout + ).where(Shout.slug == shout_dict["slug"]).one() + + + reaction_dict["shout_id"] = shout.id + reaction_dict["createdBy"] = author.id if author else 1 reaction_dict["kind"] = ReactionKind.COMMENT # creating reaction from old comment @@ -80,15 +85,20 @@ async def migrate(entry, storage): # creating shout's reactions following for reaction author following1 = session.query( ShoutReactionsFollower + ).join( + User + ).join( + Shout ).where( - ShoutReactionsFollower.follower == reaction_dict["createdBy"] + User.id == reaction_dict["createdBy"] ).filter( - ShoutReactionsFollower.shout == reaction.shout + ShoutReactionsFollower.shout_id == reaction.shout_id ).first() + if not following1: following1 = ShoutReactionsFollower.create( - follower=reaction_dict["createdBy"], - shout=reaction.shout, + follower_id=reaction_dict["createdBy"], + shout_id=reaction.shout_id, auto=True ) session.add(following1) @@ -97,15 +107,22 @@ async def migrate(entry, storage): for t in shout_dict["topics"]: tf = session.query( TopicFollower + ).join( + Topic ).where( - TopicFollower.follower == reaction_dict["createdBy"] + TopicFollower.follower_id == reaction_dict["createdBy"] ).filter( - TopicFollower.topic == t + Topic.slug == t ).first() + if not tf: + topic = session.query( + Topic + ).where(Topic.slug == t).one() + topic_following = TopicFollower.create( - follower=reaction_dict["createdBy"], - topic=t, + follower_id=reaction_dict["createdBy"], + topic_id=topic.id, auto=True ) session.add(topic_following) @@ -114,16 +131,16 @@ async def migrate(entry, storage): for comment_rating_old in entry.get("ratings", []): rater = ( session.query(User) - .filter(User.oid == comment_rating_old["createdBy"]) - .first() + .filter(User.oid == comment_rating_old["createdBy"]) + .first() ) re_reaction_dict = { - "shout": reaction_dict["shout"], + "shout_id": reaction_dict["shout_id"], "replyTo": reaction.id, "kind": ReactionKind.LIKE if comment_rating_old["value"] > 0 else ReactionKind.DISLIKE, - "createdBy": rater.slug if rater else "discours", + "createdBy": rater.id if rater else 1, } cts = comment_rating_old.get("createdAt") if cts: @@ -134,14 +151,14 @@ async def migrate(entry, storage): following2 = session.query( ShoutReactionsFollower ).where( - ShoutReactionsFollower.follower == re_reaction_dict['createdBy'] + ShoutReactionsFollower.follower_id == re_reaction_dict['createdBy'] ).filter( - ShoutReactionsFollower.shout == rr.shout + ShoutReactionsFollower.shout_id == rr.shout_id ).first() if not following2: following2 = ShoutReactionsFollower.create( - follower=re_reaction_dict['createdBy'], - shout=rr.shout, + follower_id=re_reaction_dict['createdBy'], + shout_id=rr.shout_id, auto=True ) session.add(following2) @@ -163,22 +180,26 @@ def migrate_2stage(rr, old_new_id): reply_oid = rr.get("replyTo") if not reply_oid: return + new_id = old_new_id.get(rr.get("oid")) if not new_id: return + with local_session() as session: comment = session.query(Reaction).filter(Reaction.id == new_id).first() comment.replyTo = old_new_id.get(reply_oid) session.add(comment) srf = session.query(ShoutReactionsFollower).where( - ShoutReactionsFollower.shout == comment.shout + ShoutReactionsFollower.shout_id == comment.shout_id ).filter( - ShoutReactionsFollower.follower == comment.createdBy + ShoutReactionsFollower.follower_id == comment.createdBy ).first() + if not srf: - srf = ShoutReactionsFollower.create(shout=comment.shout, follower=comment.createdBy, auto=True) + srf = ShoutReactionsFollower.create(shout_id=comment.shout_id, follower_id=comment.createdBy, auto=True) session.add(srf) + session.commit() if not rr["body"]: raise Exception(rr) diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index 6d2eadbc..1de70f12 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -8,7 +8,7 @@ from migration.extract import extract_html, extract_media from orm.reaction import Reaction, ReactionKind from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower from orm.user import User -from orm.topic import TopicFollower +from orm.topic import TopicFollower, Topic from services.stat.viewed import ViewedStorage OLD_DATE = "2016-03-05 22:22:00.350000" @@ -41,10 +41,10 @@ def create_author_from_app(app): name = app.get('name') slug = ( translit(name, "ru", reversed=True) - .replace(" ", "-") - .replace("'", "") - .replace(".", "-") - .lower() + .replace(" ", "-") + .replace("'", "") + .replace(".", "-") + .lower() ) # check if nameslug is used user = session.query(User).where(User.slug == slug).first() @@ -84,13 +84,19 @@ def create_author_from_app(app): async def create_shout(shout_dict, userslug): s = Shout.create(**shout_dict) with local_session() as session: - srf = session.query(ShoutReactionsFollower).where( - ShoutReactionsFollower.shout == s.slug + follower = session.query(User).where(User.slug == userslug).one() + + srf = session.query( + ShoutReactionsFollower + ).join( + User + ).where( + ShoutReactionsFollower.shout_id == s.id ).filter( - ShoutReactionsFollower.follower == userslug + User.slug == userslug ).first() if not srf: - srf = ShoutReactionsFollower.create(shout=s.slug, follower=userslug, auto=True) + srf = ShoutReactionsFollower.create(shout_id=s.id, follower_id=follower.id, auto=True) session.add(srf) session.commit() @@ -214,17 +220,21 @@ async def add_topics_follower(entry, storage, userslug): with local_session() as session: for tpc in topics: try: + topic = session.query(Topic).where(Topic.slug == tpc).one() + follower = session.query(User).where(User.slug == userslug).one() + tf = session.query( TopicFollower ).where( - TopicFollower.follower == userslug + TopicFollower.follower_id == follower.id ).filter( - TopicFollower.topic == tpc + TopicFollower.topic_id == topic.id ).first() + if not tf: tf = TopicFollower.create( - topic=tpc, - follower=userslug, + topic_id=topic.id, + follower_id=follower.id, auto=True ) session.add(tf) @@ -300,27 +310,35 @@ async def topics_aftermath(entry, storage): for tpc in filter(lambda x: bool(x), entry["topics"]): oldslug = tpc newslug = storage["replacements"].get(oldslug, oldslug) + if newslug: with local_session() as session: + shout = session.query(Shout).where(Shout.slug == entry["slug"]).one() + new_topic = session.query(Topic).where(Topic.slug == newslug).one() + shout_topic_old = ( session.query(ShoutTopic) - .filter(ShoutTopic.shout == entry["slug"]) - .filter(ShoutTopic.topic == oldslug) - .first() + .join(Shout) + .join(Topic) + .filter(Shout.slug == entry["slug"]) + .filter(Topic.slug == oldslug) + .first() ) if shout_topic_old: - shout_topic_old.update({"slug": newslug}) + shout_topic_old.update({"topic_id": new_topic.id}) else: shout_topic_new = ( session.query(ShoutTopic) - .filter(ShoutTopic.shout == entry["slug"]) - .filter(ShoutTopic.topic == newslug) - .first() + .join(Shout) + .join(Topic) + .filter(Shout.slug == entry["slug"]) + .filter(Topic.slug == newslug) + .first() ) if not shout_topic_new: try: ShoutTopic.create( - **{"shout": entry["slug"], "topic": newslug} + **{"shout_id": shout.id, "topic_id": new_topic.id} ) except Exception: print("[migration] shout topic error: " + newslug) @@ -339,31 +357,35 @@ async def content_ratings_to_reactions(entry, slug): for content_rating in entry.get("ratings", []): rater = ( session.query(User) - .filter(User.oid == content_rating["createdBy"]) - .first() + .filter(User.oid == content_rating["createdBy"]) + .first() ) reactedBy = ( rater if rater - else session.query(User).filter(User.slug == "noname").first() + else session.query(User).filter(User.slug == "anonymous").first() ) if rater: + shout = session.query(Shout).where(Shout.slug == slug).one() + reaction_dict = { "kind": ReactionKind.LIKE if content_rating["value"] > 0 else ReactionKind.DISLIKE, - "createdBy": reactedBy.slug, - "shout": slug, + "createdBy": reactedBy.id, + "shout_id": shout.id, } cts = content_rating.get("createdAt") if cts: reaction_dict["createdAt"] = date_parse(cts) reaction = ( - session.query(Reaction) - .filter(Reaction.shout == reaction_dict["shout"]) - .filter(Reaction.createdBy == reaction_dict["createdBy"]) - .filter(Reaction.kind == reaction_dict["kind"]) - .first() + session.query(Reaction).filter( + Reaction.shout_id == reaction_dict["shout_id"] + ).filter( + Reaction.createdBy == reaction_dict["createdBy"] + ).filter( + Reaction.kind == reaction_dict["kind"] + ).first() ) if reaction: k = ReactionKind.AGREE if content_rating["value"] > 0 else ReactionKind.DISAGREE diff --git a/migration/tables/users.py b/migration/tables/users.py index bca86a76..759a1c3d 100644 --- a/migration/tables/users.py +++ b/migration/tables/users.py @@ -115,18 +115,23 @@ def migrate_2stage(entry, id_map): continue oid = entry["_id"] author_slug = id_map.get(oid) - user_rating_dict = { - "value": rating_entry["value"], - "rater": rater_slug, - "user": author_slug, - } + with local_session() as session: try: + rater = session.query(User).where(User.slug == rater_slug).one() + user = session.query(User).where(User.slug == author_slug).one() + + user_rating_dict = { + "value": rating_entry["value"], + "rater_id": rater.id, + "user_id": user.id, + } + user_rating = UserRating.create(**user_rating_dict) if user_rating_dict['value'] > 0: af = AuthorFollower.create( - author=user_rating_dict['user'], - follower=user_rating_dict['rater'], + author_id=user.id, + follower_id=rater.id, auto=True ) session.add(af) diff --git a/orm/collab.py b/orm/collab.py index 3e3a5839..8cd8affe 100644 --- a/orm/collab.py +++ b/orm/collab.py @@ -11,14 +11,13 @@ class CollabAuthor(Base): id = None # type: ignore collab = Column(ForeignKey("collab.id"), primary_key=True) - author = Column(ForeignKey("user.slug"), primary_key=True) + author_id = Column(ForeignKey("user.id"), primary_key=True) accepted = Column(Boolean, default=False) class Collab(Base): __tablename__ = "collab" - authors = Column() title = Column(String, nullable=True, comment="Title") body = Column(String, nullable=True, comment="Body") pic = Column(String, nullable=True, comment="Picture") diff --git a/orm/collection.py b/orm/collection.py index 8d8b74eb..7651f2bf 100644 --- a/orm/collection.py +++ b/orm/collection.py @@ -9,8 +9,8 @@ class ShoutCollection(Base): __tablename__ = "shout_collection" id = None # type: ignore - shout = Column(ForeignKey("shout.slug"), primary_key=True) - collection = Column(ForeignKey("collection.slug"), primary_key=True) + shout_id = Column(ForeignKey("shout.id"), primary_key=True) + collection_id = Column(ForeignKey("collection.id"), primary_key=True) class Collection(Base): diff --git a/orm/community.py b/orm/community.py index 8c339714..251ac4a4 100644 --- a/orm/community.py +++ b/orm/community.py @@ -8,8 +8,8 @@ class CommunityFollower(Base): __tablename__ = "community_followers" id = None # type: ignore - follower = Column(ForeignKey("user.slug"), primary_key=True) - community = Column(ForeignKey("community.slug"), primary_key=True) + follower_id = Column(ForeignKey("user.id"), primary_key=True) + community_id = Column(ForeignKey("community.id"), primary_key=True) joinedAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" ) diff --git a/orm/reaction.py b/orm/reaction.py index 123cfb22..c3a87385 100644 --- a/orm/reaction.py +++ b/orm/reaction.py @@ -28,12 +28,12 @@ class Reaction(Base): createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" ) - createdBy = Column(ForeignKey("user.slug"), nullable=False, comment="Sender") + createdBy = Column(ForeignKey("user.id"), nullable=False, index=True, comment="Sender") updatedAt = Column(DateTime, nullable=True, comment="Updated at") - updatedBy = Column(ForeignKey("user.slug"), nullable=True, comment="Last Editor") + updatedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Last Editor") deletedAt = Column(DateTime, nullable=True, comment="Deleted at") - deletedBy = Column(ForeignKey("user.slug"), nullable=True, comment="Deleted by") - shout = Column(ForeignKey("shout.slug"), nullable=False) + deletedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Deleted by") + shout_id = Column(ForeignKey("shout.id"), nullable=False, index=True) replyTo = Column( ForeignKey("reaction.id"), nullable=True, comment="Reply to reaction ID" ) diff --git a/orm/shout.py b/orm/shout.py index f65b1268..452d292f 100644 --- a/orm/shout.py +++ b/orm/shout.py @@ -13,16 +13,16 @@ class ShoutTopic(Base): __tablename__ = "shout_topic" id = None # type: ignore - shout = Column(ForeignKey("shout.slug"), primary_key=True) - topic = Column(ForeignKey("topic.slug"), primary_key=True) + shout_id = Column(ForeignKey("shout.id"), primary_key=True, index=True) + topic_id = Column(ForeignKey("topic.id"), primary_key=True, index=True) class ShoutReactionsFollower(Base): __tablename__ = "shout_reactions_followers" id = None # type: ignore - follower = Column(ForeignKey("user.slug"), primary_key=True) - shout = Column(ForeignKey("shout.slug"), primary_key=True) + follower_id = Column(ForeignKey("user.id"), primary_key=True, index=True) + shout_id = Column(ForeignKey("shout.id"), primary_key=True, index=True) auto = Column(Boolean, nullable=False, default=False) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" @@ -34,8 +34,8 @@ class ShoutAuthor(Base): __tablename__ = "shout_author" id = None # type: ignore - shout = Column(ForeignKey("shout.slug"), primary_key=True) - user = Column(ForeignKey("user.slug"), primary_key=True) + shout_id = Column(ForeignKey("shout.id"), primary_key=True, index=True) + user_id = Column(ForeignKey("user.id"), primary_key=True, index=True) caption = Column(String, nullable=True, default="") @@ -55,7 +55,7 @@ class Shout(Base): topics = relationship(lambda: Topic, secondary=ShoutTopic.__tablename__) reactions = relationship(lambda: Reaction) visibility = Column(String, nullable=True) # owner authors community public - versionOf = Column(ForeignKey("shout.slug"), nullable=True) + versionOf = Column(ForeignKey("shout.id"), nullable=True) oid = Column(String, nullable=True) media = Column(JSON, nullable=True) diff --git a/orm/topic.py b/orm/topic.py index f4d094b7..3f99cb03 100644 --- a/orm/topic.py +++ b/orm/topic.py @@ -9,8 +9,8 @@ class TopicFollower(Base): __tablename__ = "topic_followers" id = None # type: ignore - follower = Column(ForeignKey("user.slug"), primary_key=True) - topic = Column(ForeignKey("topic.slug"), primary_key=True) + follower_id = Column(ForeignKey("user.id"), primary_key=True, index=True) + topic_id = Column(ForeignKey("topic.id"), primary_key=True, index=True) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" ) diff --git a/orm/user.py b/orm/user.py index 11055d29..1f283a2b 100644 --- a/orm/user.py +++ b/orm/user.py @@ -6,7 +6,6 @@ from sqlalchemy.orm import relationship from base.orm import Base, local_session from orm.rbac import Role -from services.auth.roles import RoleStorage class UserNotifications(Base): @@ -21,8 +20,8 @@ class UserRating(Base): __tablename__ = "user_rating" id = None # type: ignore - rater = Column(ForeignKey("user.slug"), primary_key=True) - user = Column(ForeignKey("user.slug"), primary_key=True) + rater_id = Column(ForeignKey("user.id"), primary_key=True, index=True) + user_id = Column(ForeignKey("user.id"), primary_key=True, index=True) value = Column(Integer) @staticmethod @@ -34,16 +33,16 @@ class UserRole(Base): __tablename__ = "user_role" id = None # type: ignore - user_id = Column(ForeignKey("user.id"), primary_key=True) - role_id = Column(ForeignKey("role.id"), primary_key=True) + user_id = Column(ForeignKey("user.id"), primary_key=True, index=True) + role_id = Column(ForeignKey("role.id"), primary_key=True, index=True) class AuthorFollower(Base): __tablename__ = "author_follower" id = None # type: ignore - follower = Column(ForeignKey("user.slug"), primary_key=True) - author = Column(ForeignKey("user.slug"), primary_key=True) + follower_id = Column(ForeignKey("user.id"), primary_key=True, index=True) + author_id = Column(ForeignKey("user.id"), primary_key=True, index=True) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" ) @@ -73,7 +72,7 @@ class User(Base): links = Column(JSONType, nullable=True, comment="Links") oauth = Column(String, nullable=True) notifications = relationship(lambda: UserNotifications) - ratings = relationship(UserRating, foreign_keys=UserRating.user) + ratings = relationship(UserRating, foreign_keys=UserRating.user_id) roles = relationship(lambda: Role, secondary=UserRole.__tablename__) oid = Column(String, nullable=True) diff --git a/orm/viewed.py b/orm/viewed.py index 6d089234..b269921a 100644 --- a/orm/viewed.py +++ b/orm/viewed.py @@ -6,8 +6,8 @@ from base.orm import Base, local_session class ViewedEntry(Base): __tablename__ = "viewed" - viewer = Column(ForeignKey("user.slug"), default='anonymous') - shout = Column(ForeignKey("shout.slug"), default="genesis-block") + viewer_id = Column(ForeignKey("user.id"), index=True, default=1) + shout_id = Column(ForeignKey("shout.id"), index=True, default=1) amount = Column(Integer, default=1) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" diff --git a/resolvers/create/editor.py b/resolvers/create/editor.py index 220ff229..499ffb8c 100644 --- a/resolvers/create/editor.py +++ b/resolvers/create/editor.py @@ -1,11 +1,13 @@ from datetime import datetime, timezone +from sqlalchemy import and_ + from auth.authenticate import login_required from base.orm import local_session from base.resolvers import mutation from orm.rbac import Resource from orm.shout import Shout, ShoutAuthor, ShoutTopic -from orm.topic import TopicFollower +from orm.topic import TopicFollower, Topic from orm.user import User from resolvers.zine.reactions import reactions_follow, reactions_unfollow from services.zine.gittask import GitTask @@ -24,7 +26,7 @@ async def create_shout(_, info, inp): new_shout = Shout.create(**inp) # NOTE: shout made by one first author - sa = ShoutAuthor.create(shout=new_shout.slug, user=user.slug) + sa = ShoutAuthor.create(shout_id=new_shout.id, user_id=user.id) session.add(sa) reactions_follow(user, new_shout.slug, True) @@ -33,11 +35,16 @@ async def create_shout(_, info, inp): topic_slugs.append(inp["mainTopic"]) for slug in topic_slugs: - st = ShoutTopic.create(shout=new_shout.slug, topic=slug) + topic = session.query(Topic).where(Topic.slug == slug).one() + + st = ShoutTopic.create(shout_id=new_shout.id, topic_id=topic.id) session.add(st) - tf = session.query(TopicFollower).where(follower=user.slug, topic=slug) + tf = session.query(TopicFollower).where( + and_(TopicFollower.follower_id == user.id, TopicFollower.topic_id == topic.id) + ) + if not tf: - tf = TopicFollower.create(follower=user.slug, topic=slug, auto=True) + tf = TopicFollower.create(follower_id=user.id, topic_id=topic.id, auto=True) session.add(tf) new_shout.topic_slugs = topic_slugs @@ -45,7 +52,7 @@ async def create_shout(_, info, inp): session.commit() - GitTask(inp, user.username, user.email, "new shout %s" % (new_shout.slug)) + GitTask(inp, user.username, user.email, "new shout %s" % new_shout.slug) return {"shout": new_shout} @@ -75,7 +82,7 @@ async def update_shout(_, info, inp): session.add(shout) if inp.get("topics"): # remove old links - links = session.query(ShoutTopic).where(ShoutTopic.shout == slug).all() + links = session.query(ShoutTopic).where(ShoutTopic.shout_id == shout.id).all() for topiclink in links: session.delete(topiclink) # add new topic links diff --git a/resolvers/inbox/search.py b/resolvers/inbox/search.py index 914af4eb..e7132506 100644 --- a/resolvers/inbox/search.py +++ b/resolvers/inbox/search.py @@ -4,7 +4,7 @@ from auth.authenticate import login_required from base.redis import redis from base.resolvers import query from base.orm import local_session -from orm.user import AuthorFollower +from orm.user import AuthorFollower, User @query.field("searchRecipients") @@ -30,13 +30,19 @@ async def search_recipients(_, info, query: str, limit: int = 50, offset: int = with local_session() as session: # followings - result += session.query(AuthorFollower.author).where(AuthorFollower.follower.startswith(query))\ - .offset(offset + len(result)).limit(more_amount) + result += session.query(AuthorFollower.author).join( + User, User.id == AuthorFollower.follower_id + ).where( + User.slug.startswith(query) + ).offset(offset + len(result)).limit(more_amount) more_amount = limit # followers - result += session.query(AuthorFollower.follower).where(AuthorFollower.author.startswith(query))\ - .offset(offset + len(result)).limit(offset + len(result) + limit) + result += session.query(AuthorFollower.follower).join( + User, User.id == AuthorFollower.author_id + ).where( + User.slug.startswith(query) + ).offset(offset + len(result)).limit(offset + len(result) + limit) return { "members": list(result), "error": None diff --git a/resolvers/zine/load.py b/resolvers/zine/load.py index bc76db02..0e0c1ade 100644 --- a/resolvers/zine/load.py +++ b/resolvers/zine/load.py @@ -1,11 +1,11 @@ from datetime import datetime, timedelta, timezone from sqlalchemy.orm import joinedload, aliased -from sqlalchemy.sql.expression import desc, asc, select, case, func +from sqlalchemy.sql.expression import desc, asc, select, func from base.orm import local_session from base.resolvers import query from orm import ViewedEntry from orm.shout import Shout, ShoutAuthor -from orm.reaction import Reaction, ReactionKind +from orm.reaction import Reaction from resolvers.zine._common import add_common_stat_columns @@ -18,7 +18,7 @@ def add_stat_columns(q): def apply_filters(q, filters, user=None): if filters.get("reacted") and user: - q.join(Reaction, Reaction.createdBy == user.slug) + q.join(Reaction, Reaction.createdBy == user.id) v = filters.get("visibility") if v == "public": @@ -66,9 +66,9 @@ async def load_shout(_, info, slug): "rating": rating_stat } - for author_caption in session.query(ShoutAuthor).where(ShoutAuthor.shout == slug): + for author_caption in session.query(ShoutAuthor).join(Shout).where(Shout.slug == slug): for author in shout.authors: - if author.slug == author_caption.user: + if author.id == author_caption.user_id: author.caption = author_caption.caption return shout diff --git a/resolvers/zine/profile.py b/resolvers/zine/profile.py index be38eeff..86368ac5 100644 --- a/resolvers/zine/profile.py +++ b/resolvers/zine/profile.py @@ -19,17 +19,19 @@ from resolvers.zine.topics import followed_by_user def add_author_stat_columns(q): author_followers = aliased(AuthorFollower) author_following = aliased(AuthorFollower) + shout_author_aliased = aliased(ShoutAuthor) + user_rating_aliased = aliased(UserRating) - q = q.outerjoin(ShoutAuthor).add_columns( - func.count(distinct(ShoutAuthor.shout)).label('shouts_stat') - ).outerjoin(author_followers, author_followers.author == User.slug).add_columns( - func.count(distinct(author_followers.follower)).label('followers_stat') - ).outerjoin(author_following, author_following.follower == User.slug).add_columns( - func.count(distinct(author_following.author)).label('followings_stat') - ).outerjoin(UserRating).add_columns( + q = q.outerjoin(shout_author_aliased).add_columns( + func.count(distinct(shout_author_aliased.shout_id)).label('shouts_stat') + ).outerjoin(author_followers, author_followers.author_id == User.id).add_columns( + func.count(distinct(author_followers.follower_id)).label('followers_stat') + ).outerjoin(author_following, author_following.follower_id == User.id).add_columns( + func.count(distinct(author_following.author_id)).label('followings_stat') + ).outerjoin(user_rating_aliased, user_rating_aliased.user_id == User.id).add_columns( # TODO: check - func.sum(UserRating.value).label('rating_stat') - ).outerjoin(Reaction, and_(Reaction.createdBy == User.slug, Reaction.body.is_not(None))).add_columns( + func.sum(user_rating_aliased.value).label('rating_stat') + ).outerjoin(Reaction, and_(Reaction.createdBy == User.id, Reaction.body.is_not(None))).add_columns( func.count(distinct(Reaction.id)).label('commented_stat') ) @@ -83,7 +85,7 @@ async def followed_reactions(slug): return session.query( Reaction.shout ).where( - Reaction.createdBy == slug + Reaction.createdBy == user.id ).filter( Reaction.createdAt > user.lastSeen ).all() @@ -107,7 +109,7 @@ async def get_followed_authors(_, _info, slug) -> List[User]: async def followed_authors(slug) -> List[User]: q = select(User) q = add_author_stat_columns(q) - q = q.join(AuthorFollower).where(AuthorFollower.follower == slug) + q = q.join(AuthorFollower).join(User, User.id == AuthorFollower.follower_id).where(User.slug == slug) return get_authors_from_query(q) @@ -116,7 +118,13 @@ async def followed_authors(slug) -> List[User]: async def user_followers(_, _info, slug) -> List[User]: q = select(User) q = add_author_stat_columns(q) - q = q.join(AuthorFollower).where(AuthorFollower.author == slug) + + aliased_user = aliased(User) + q = q.join(AuthorFollower).join( + aliased_user, aliased_user.id == AuthorFollower.author_id + ).where( + aliased_user.slug == slug + ) return get_authors_from_query(q) @@ -173,7 +181,8 @@ async def rate_user(_, info, rated_userslug, value): # for mutation.field("follow") def author_follow(user, slug): with local_session() as session: - af = AuthorFollower.create(follower=user.slug, author=slug) + author = session.query(User).where(User.slug == slug).one() + af = AuthorFollower.create(follower_id=user.id, author_id=author.id) session.add(af) session.commit() @@ -182,13 +191,13 @@ def author_follow(user, slug): def author_unfollow(user, slug): with local_session() as session: flw = ( - session.query(AuthorFollower) - .filter( + session.query( + AuthorFollower + ).join(User, User.id == AuthorFollower.author_id).filter( and_( - AuthorFollower.follower == user.slug, AuthorFollower.author == slug + AuthorFollower.follower_id == user.id, User.slug == slug ) - ) - .first() + ).first() ) if not flw: raise Exception("[resolvers.profile] follower not exist, cant unfollow") @@ -224,7 +233,7 @@ async def load_authors_by(_, info, by, limit, offset): elif by.get("name"): q = q.filter(User.name.ilike(f"%{by['name']}%")) elif by.get("topic"): - q = q.join(ShoutAuthor).join(ShoutTopic).where(ShoutTopic.topic == by["topic"]) + q = q.join(ShoutAuthor).join(ShoutTopic).join(Topic).where(Topic.slug == by["topic"]) if by.get("lastSeen"): # in days days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["lastSeen"]) q = q.filter(User.lastSeen > days_before) diff --git a/resolvers/zine/reactions.py b/resolvers/zine/reactions.py index 876e4791..de1ed409 100644 --- a/resolvers/zine/reactions.py +++ b/resolvers/zine/reactions.py @@ -16,16 +16,19 @@ def add_reaction_stat_columns(q): def reactions_follow(user: User, slug: str, auto=False): with local_session() as session: + shout = session.query(Shout).where(Shout.slug == slug).one() + following = ( session.query(ShoutReactionsFollower).where(and_( - ShoutReactionsFollower.follower == user.slug, - ShoutReactionsFollower.shout == slug + ShoutReactionsFollower.follower_id == user.id, + ShoutReactionsFollower.shout_id == shout.id, )).first() ) + if not following: following = ShoutReactionsFollower.create( - follower=user.slug, - shout=slug, + follower_id=user.id, + shout_id=shout.id, auto=auto ) session.add(following) @@ -34,12 +37,15 @@ def reactions_follow(user: User, slug: str, auto=False): def reactions_unfollow(user, slug): with local_session() as session: + shout = session.query(Shout).where(Shout.slug == slug).one() + following = ( session.query(ShoutReactionsFollower).where(and_( - ShoutReactionsFollower.follower == user.slug, - ShoutReactionsFollower.shout == slug + ShoutReactionsFollower.follower_id == user.id, + ShoutReactionsFollower.shout_id == shout.id )).first() ) + if following: session.delete(following) session.commit() @@ -68,7 +74,7 @@ def check_to_publish(session, user, reaction): ]: if is_published_author(user): # now count how many approvers are voted already - approvers_reactions = session.query(Reaction).where(Reaction.shout == reaction.shout).all() + approvers_reactions = session.query(Reaction).where(Reaction.shout_id == reaction.shout_id).all() approvers = [user.slug, ] for ar in approvers_reactions: a = ar.createdBy @@ -87,7 +93,7 @@ def check_to_hide(session, user, reaction): ReactionKind.UNPROOF ]: # if is_published_author(user): - approvers_reactions = session.query(Reaction).where(Reaction.shout == reaction.shout).all() + approvers_reactions = session.query(Reaction).where(Reaction.shout_id == reaction.shout_id).all() declines = 0 for r in approvers_reactions: if r.kind in [ @@ -224,22 +230,26 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0): q = select( Reaction, CreatedByUser, ReactedShout ).join( - CreatedByUser, Reaction.createdBy == CreatedByUser.slug + CreatedByUser, Reaction.createdBy == CreatedByUser.id ).join( - ReactedShout, Reaction.shout == ReactedShout.slug + ReactedShout, Reaction.shout_id == ReactedShout.id ) if by.get("shout"): - q = q.filter(Reaction.shout == by["shout"]) + aliased_shout = aliased(Shout) + q = q.join(aliased_shout).filter(aliased_shout.slug == by["shout"]) elif by.get("shouts"): - q = q.filter(Reaction.shout.in_(by["shouts"])) + aliased_shout = aliased(Shout) + q = q.join(aliased_shout).filter(aliased_shout.shout.in_(by["shouts"])) if by.get("createdBy"): - q = q.filter(Reaction.createdBy == by.get("createdBy")) + aliased_user = aliased(User) + q = q.join(aliased_user).filter(aliased_user.slug == by.get("createdBy")) if by.get("topic"): + # TODO: check q = q.filter(Shout.topics.contains(by["topic"])) if by.get("comment"): q = q.filter(func.length(Reaction.body) > 0) - if by.get('search', 0) > 2: + if len(by.get('search', '')) > 2: q = q.filter(Reaction.body.ilike(f'%{by["body"]}%')) if by.get("days"): after = datetime.now(tz=timezone.utc) - timedelta(days=int(by["days"]) or 30) diff --git a/resolvers/zine/topics.py b/resolvers/zine/topics.py index 459d95c8..610dc720 100644 --- a/resolvers/zine/topics.py +++ b/resolvers/zine/topics.py @@ -4,20 +4,20 @@ from base.orm import local_session from base.resolvers import mutation, query from orm.shout import ShoutTopic, ShoutAuthor from orm.topic import Topic, TopicFollower -from orm import Shout +from orm import Shout, User def add_topic_stat_columns(q): - q = q.outerjoin(ShoutTopic, Topic.slug == ShoutTopic.topic).add_columns( - func.count(distinct(ShoutTopic.shout)).label('shouts_stat') - ).outerjoin(ShoutAuthor, ShoutTopic.shout == ShoutAuthor.shout).add_columns( - func.count(distinct(ShoutAuthor.user)).label('authors_stat') + q = q.outerjoin(ShoutTopic, Topic.id == ShoutTopic.topic_id).add_columns( + func.count(distinct(ShoutTopic.shout_id)).label('shouts_stat') + ).outerjoin(ShoutAuthor, ShoutTopic.shout_id == ShoutAuthor.shout_id).add_columns( + func.count(distinct(ShoutAuthor.user_id)).label('authors_stat') ).outerjoin(TopicFollower, and_( - TopicFollower.topic == Topic.slug, - TopicFollower.follower == ShoutAuthor.user + TopicFollower.topic_id == Topic.id, + TopicFollower.follower_id == ShoutAuthor.id )).add_columns( - func.count(distinct(TopicFollower.follower)).label('followers_stat') + func.count(distinct(TopicFollower.follower_id)).label('followers_stat') ) q = q.group_by(Topic.id) @@ -49,7 +49,7 @@ def get_topics_from_query(q): def followed_by_user(user_slug): q = select(Topic) q = add_topic_stat_columns(q) - q = q.where(TopicFollower.follower == user_slug) + q = q.join(User).where(User.slug == user_slug) return get_topics_from_query(q) @@ -74,7 +74,7 @@ async def topics_by_community(_, info, community): async def topics_by_author(_, _info, author): q = select(Topic) q = add_topic_stat_columns(q) - q = q.where(ShoutAuthor.user == author) + q = q.join(User).where(User.slug == author) return get_topics_from_query(q) @@ -117,7 +117,9 @@ async def update_topic(_, _info, inp): async def topic_follow(user, slug): with local_session() as session: - following = TopicFollower.create(topic=slug, follower=user.slug) + topic = session.query(Topic).where(Topic.slug == slug).one() + + following = TopicFollower.create(topic_id=topic.id, follower=user.id) session.add(following) session.commit() @@ -125,10 +127,10 @@ async def topic_follow(user, slug): async def topic_unfollow(user, slug): with local_session() as session: sub = ( - session.query(TopicFollower).filter( + session.query(TopicFollower).join(Topic).filter( and_( - TopicFollower.follower == user.slug, - TopicFollower.topic == slug + TopicFollower.follower_id == user.id, + Topic.slug == slug ) ).first() ) @@ -143,7 +145,7 @@ async def topic_unfollow(user, slug): async def topics_random(_, info, amount=12): q = select(Topic) q = add_topic_stat_columns(q) - q = q.join(Shout, ShoutTopic.shout == Shout.slug).group_by(Topic.id).having(func.count(Shout.id) > 2) + q = q.join(Shout, ShoutTopic.shout_id == Shout.id).group_by(Topic.id).having(func.count(Shout.id) > 2) q = q.order_by(func.random()).limit(amount) return get_topics_from_query(q) diff --git a/services/stat/viewed.py b/services/stat/viewed.py index 77ebe979..32445153 100644 --- a/services/stat/viewed.py +++ b/services/stat/viewed.py @@ -5,7 +5,9 @@ from gql import Client, gql from gql.transport.aiohttp import AIOHTTPTransport from base.orm import local_session from sqlalchemy import func -from orm.shout import ShoutTopic + +from orm import User, Topic +from orm.shout import ShoutTopic, Shout from orm.viewed import ViewedEntry from ssl import create_default_context from os import environ, path @@ -113,6 +115,7 @@ class ViewedStorage: async with self.lock: return self.client.execute_async(load_facts) + # unused yet @staticmethod async def get_shout(shout_slug): """ getting shout views metric by slug """ @@ -123,8 +126,9 @@ class ViewedStorage: shout_views = 0 with local_session() as session: try: + shout = session.query(Shout).where(Shout.slug == shout_slug).one() shout_views = session.query(func.sum(ViewedEntry.amount)).where( - ViewedEntry.shout == shout_slug + ViewedEntry.shout_id == shout.id ).all()[0][0] self.by_shouts[shout_slug] = shout_views self.update_topics(session, shout_slug) @@ -147,11 +151,12 @@ class ViewedStorage: def update_topics(session, shout_slug): """ updates topics counters by shout slug """ self = ViewedStorage - for t in session.query(ShoutTopic).where(ShoutTopic.shout == shout_slug).all(): - tpc = t.topic - if not self.by_topics.get(tpc): - self.by_topics[tpc] = {} - self.by_topics[tpc][shout_slug] = self.by_shouts[shout_slug] + for [shout_topic, topic] in session.query(ShoutTopic, Topic).join(Topic).join(Shout).where( + Shout.slug == shout_slug + ).all(): + if not self.by_topics.get(topic.slug): + self.by_topics[topic.slug] = {} + self.by_topics[topic.slug][shout_slug] = self.by_shouts[shout_slug] @staticmethod async def increment(shout_slug, amount=1, viewer='anonymous'): @@ -159,9 +164,12 @@ class ViewedStorage: self = ViewedStorage async with self.lock: with local_session() as session: + shout = session.query(Shout).where(Shout.slug == shout_slug).one() + viewer = session.query(User).where(User.slug == viewer).one() + viewed = ViewedEntry.create(**{ - "viewer": viewer, - "shout": shout_slug, + "viewer_id": viewer.id, + "shout_id": shout.id, "amount": amount }) session.add(viewed) From 219f0470ca604dc5d7df2d9933b5398a256875e7 Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Tue, 29 Nov 2022 17:37:40 +0100 Subject: [PATCH 08/12] slow queries to fix later --- resolvers/zine/profile.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/resolvers/zine/profile.py b/resolvers/zine/profile.py index 86368ac5..1c60c6cb 100644 --- a/resolvers/zine/profile.py +++ b/resolvers/zine/profile.py @@ -1,6 +1,6 @@ from typing import List from datetime import datetime, timedelta, timezone -from sqlalchemy import and_, func, distinct, select +from sqlalchemy import and_, func, distinct, select, literal from sqlalchemy.orm import aliased, joinedload from auth.authenticate import login_required @@ -24,16 +24,27 @@ def add_author_stat_columns(q): q = q.outerjoin(shout_author_aliased).add_columns( func.count(distinct(shout_author_aliased.shout_id)).label('shouts_stat') - ).outerjoin(author_followers, author_followers.author_id == User.id).add_columns( - func.count(distinct(author_followers.follower_id)).label('followers_stat') - ).outerjoin(author_following, author_following.follower_id == User.id).add_columns( - func.count(distinct(author_following.author_id)).label('followings_stat') - ).outerjoin(user_rating_aliased, user_rating_aliased.user_id == User.id).add_columns( - # TODO: check - func.sum(user_rating_aliased.value).label('rating_stat') - ).outerjoin(Reaction, and_(Reaction.createdBy == User.id, Reaction.body.is_not(None))).add_columns( - func.count(distinct(Reaction.id)).label('commented_stat') ) + q = q.outerjoin(author_followers, author_followers.author_id == User.id).add_columns( + func.count(distinct(author_followers.follower_id)).label('followers_stat') + ) + + q = q.outerjoin(author_following, author_following.follower_id == User.id).add_columns( + func.count(distinct(author_following.author_id)).label('followings_stat') + ) + + q = q.add_columns(literal(0).label('rating_stat')) + # FIXME + # q = q.outerjoin(user_rating_aliased, user_rating_aliased.user_id == User.id).add_columns( + # # TODO: check + # func.sum(user_rating_aliased.value).label('rating_stat') + # ) + + q = q.add_columns(literal(0).label('commented_stat')) + # FIXME + # q = q.outerjoin(Reaction, and_(Reaction.createdBy == User.id, Reaction.body.is_not(None))).add_columns( + # func.count(distinct(Reaction.id)).label('commented_stat') + # ) q = q.group_by(User.id) @@ -210,7 +221,7 @@ def author_unfollow(user, slug): async def get_authors_all(_, _info): q = select(User) q = add_author_stat_columns(q) - q = q.join(ShoutAuthor) + q = q.join(ShoutAuthor, User.id == ShoutAuthor.user_id) return get_authors_from_query(q) From c064170d24cb83af0d88e340cd4a2d4c8311f4e6 Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Tue, 29 Nov 2022 17:44:29 +0100 Subject: [PATCH 09/12] updated CHECKS --- CHECKS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHECKS b/CHECKS index e2d41a77..e1253bd6 100644 --- a/CHECKS +++ b/CHECKS @@ -1,5 +1,5 @@ -WAIT=30 +WAIT=10 TIMEOUT=10 -ATTEMPTS=20 # 20 * 30 = 10 min +ATTEMPTS=30 # 10 * 30 = 5 min / Playground From 8209cc744c6038478a1b06a5c1f425b91e0867ca Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Tue, 29 Nov 2022 20:13:03 +0100 Subject: [PATCH 10/12] consistent naming of database fields --- migration/tables/comments.py | 31 +++++++++++++++---------------- migration/tables/content_items.py | 20 ++++++++++---------- migration/tables/users.py | 8 ++++---- orm/collab.py | 2 +- orm/collection.py | 4 ++-- orm/community.py | 4 ++-- orm/rbac.py | 22 +++++++++++----------- orm/reaction.py | 2 +- orm/shout.py | 12 ++++++------ orm/topic.py | 4 ++-- orm/user.py | 22 +++++++++++----------- orm/viewed.py | 4 ++-- resolvers/create/editor.py | 12 ++++++------ resolvers/inbox/search.py | 4 ++-- resolvers/zine/load.py | 2 +- resolvers/zine/profile.py | 26 +++++++++++++------------- resolvers/zine/reactions.py | 18 +++++++++--------- resolvers/zine/topics.py | 20 ++++++++++---------- schema.graphql | 4 ++-- services/stat/viewed.py | 6 +++--- 20 files changed, 113 insertions(+), 114 deletions(-) diff --git a/migration/tables/comments.py b/migration/tables/comments.py index 09cfb491..160d1dc7 100644 --- a/migration/tables/comments.py +++ b/migration/tables/comments.py @@ -73,8 +73,7 @@ async def migrate(entry, storage): Shout ).where(Shout.slug == shout_dict["slug"]).one() - - reaction_dict["shout_id"] = shout.id + reaction_dict["shoutId"] = shout.id reaction_dict["createdBy"] = author.id if author else 1 reaction_dict["kind"] = ReactionKind.COMMENT @@ -92,13 +91,13 @@ async def migrate(entry, storage): ).where( User.id == reaction_dict["createdBy"] ).filter( - ShoutReactionsFollower.shout_id == reaction.shout_id + ShoutReactionsFollower.shoutId == reaction.shoutId ).first() if not following1: following1 = ShoutReactionsFollower.create( - follower_id=reaction_dict["createdBy"], - shout_id=reaction.shout_id, + followerId=reaction_dict["createdBy"], + shoutId=reaction.shoutId, auto=True ) session.add(following1) @@ -110,7 +109,7 @@ async def migrate(entry, storage): ).join( Topic ).where( - TopicFollower.follower_id == reaction_dict["createdBy"] + TopicFollower.followerId == reaction_dict["createdBy"] ).filter( Topic.slug == t ).first() @@ -121,8 +120,8 @@ async def migrate(entry, storage): ).where(Topic.slug == t).one() topic_following = TopicFollower.create( - follower_id=reaction_dict["createdBy"], - topic_id=topic.id, + followerId=reaction_dict["createdBy"], + topicId=topic.id, auto=True ) session.add(topic_following) @@ -135,7 +134,7 @@ async def migrate(entry, storage): .first() ) re_reaction_dict = { - "shout_id": reaction_dict["shout_id"], + "shoutId": reaction_dict["shoutId"], "replyTo": reaction.id, "kind": ReactionKind.LIKE if comment_rating_old["value"] > 0 @@ -151,14 +150,14 @@ async def migrate(entry, storage): following2 = session.query( ShoutReactionsFollower ).where( - ShoutReactionsFollower.follower_id == re_reaction_dict['createdBy'] + ShoutReactionsFollower.followerId == re_reaction_dict['createdBy'] ).filter( - ShoutReactionsFollower.shout_id == rr.shout_id + ShoutReactionsFollower.shoutId == rr.shoutId ).first() if not following2: following2 = ShoutReactionsFollower.create( - follower_id=re_reaction_dict['createdBy'], - shout_id=rr.shout_id, + followerId=re_reaction_dict['createdBy'], + shoutId=rr.shoutId, auto=True ) session.add(following2) @@ -191,13 +190,13 @@ def migrate_2stage(rr, old_new_id): session.add(comment) srf = session.query(ShoutReactionsFollower).where( - ShoutReactionsFollower.shout_id == comment.shout_id + ShoutReactionsFollower.shoutId == comment.shoutId ).filter( - ShoutReactionsFollower.follower_id == comment.createdBy + ShoutReactionsFollower.followerId == comment.createdBy ).first() if not srf: - srf = ShoutReactionsFollower.create(shout_id=comment.shout_id, follower_id=comment.createdBy, auto=True) + srf = ShoutReactionsFollower.create(shoutId=comment.shoutId, followerId=comment.createdBy, auto=True) session.add(srf) session.commit() diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index 1de70f12..5d1b67e9 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -91,12 +91,12 @@ async def create_shout(shout_dict, userslug): ).join( User ).where( - ShoutReactionsFollower.shout_id == s.id + ShoutReactionsFollower.shoutId == s.id ).filter( User.slug == userslug ).first() if not srf: - srf = ShoutReactionsFollower.create(shout_id=s.id, follower_id=follower.id, auto=True) + srf = ShoutReactionsFollower.create(shoutId=s.id, followerId=follower.id, auto=True) session.add(srf) session.commit() @@ -226,15 +226,15 @@ async def add_topics_follower(entry, storage, userslug): tf = session.query( TopicFollower ).where( - TopicFollower.follower_id == follower.id + TopicFollower.followerId == follower.id ).filter( - TopicFollower.topic_id == topic.id + TopicFollower.topicId == topic.id ).first() if not tf: tf = TopicFollower.create( - topic_id=topic.id, - follower_id=follower.id, + topicId=topic.id, + followerId=follower.id, auto=True ) session.add(tf) @@ -325,7 +325,7 @@ async def topics_aftermath(entry, storage): .first() ) if shout_topic_old: - shout_topic_old.update({"topic_id": new_topic.id}) + shout_topic_old.update({"topicId": new_topic.id}) else: shout_topic_new = ( session.query(ShoutTopic) @@ -338,7 +338,7 @@ async def topics_aftermath(entry, storage): if not shout_topic_new: try: ShoutTopic.create( - **{"shout_id": shout.id, "topic_id": new_topic.id} + **{"shoutId": shout.id, "topicId": new_topic.id} ) except Exception: print("[migration] shout topic error: " + newslug) @@ -373,14 +373,14 @@ async def content_ratings_to_reactions(entry, slug): if content_rating["value"] > 0 else ReactionKind.DISLIKE, "createdBy": reactedBy.id, - "shout_id": shout.id, + "shoutId": shout.id, } cts = content_rating.get("createdAt") if cts: reaction_dict["createdAt"] = date_parse(cts) reaction = ( session.query(Reaction).filter( - Reaction.shout_id == reaction_dict["shout_id"] + Reaction.shoutId == reaction_dict["shoutId"] ).filter( Reaction.createdBy == reaction_dict["createdBy"] ).filter( diff --git a/migration/tables/users.py b/migration/tables/users.py index 759a1c3d..320dd50e 100644 --- a/migration/tables/users.py +++ b/migration/tables/users.py @@ -123,15 +123,15 @@ def migrate_2stage(entry, id_map): user_rating_dict = { "value": rating_entry["value"], - "rater_id": rater.id, - "user_id": user.id, + "raterId": rater.id, + "userId": user.id, } user_rating = UserRating.create(**user_rating_dict) if user_rating_dict['value'] > 0: af = AuthorFollower.create( - author_id=user.id, - follower_id=rater.id, + authorId=user.id, + followerId=rater.id, auto=True ) session.add(af) diff --git a/orm/collab.py b/orm/collab.py index 8cd8affe..accf650c 100644 --- a/orm/collab.py +++ b/orm/collab.py @@ -11,7 +11,7 @@ class CollabAuthor(Base): id = None # type: ignore collab = Column(ForeignKey("collab.id"), primary_key=True) - author_id = Column(ForeignKey("user.id"), primary_key=True) + authorId = Column(ForeignKey("user.id"), primary_key=True) accepted = Column(Boolean, default=False) diff --git a/orm/collection.py b/orm/collection.py index 7651f2bf..3f6eee3d 100644 --- a/orm/collection.py +++ b/orm/collection.py @@ -9,8 +9,8 @@ class ShoutCollection(Base): __tablename__ = "shout_collection" id = None # type: ignore - shout_id = Column(ForeignKey("shout.id"), primary_key=True) - collection_id = Column(ForeignKey("collection.id"), primary_key=True) + shoutId = Column(ForeignKey("shout.id"), primary_key=True) + collectionId = Column(ForeignKey("collection.id"), primary_key=True) class Collection(Base): diff --git a/orm/community.py b/orm/community.py index 251ac4a4..c9d718e3 100644 --- a/orm/community.py +++ b/orm/community.py @@ -8,8 +8,8 @@ class CommunityFollower(Base): __tablename__ = "community_followers" id = None # type: ignore - follower_id = Column(ForeignKey("user.id"), primary_key=True) - community_id = Column(ForeignKey("community.id"), primary_key=True) + followerId = Column(ForeignKey("user.id"), primary_key=True) + communityId = Column(ForeignKey("community.id"), primary_key=True) joinedAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" ) diff --git a/orm/rbac.py b/orm/rbac.py index ad288798..aa49804c 100644 --- a/orm/rbac.py +++ b/orm/rbac.py @@ -122,7 +122,7 @@ class Operation(Base): class Resource(Base): __tablename__ = "resource" - resource_class = Column( + resourceClass = Column( String, nullable=False, unique=True, comment="Resource class" ) name = Column(String, nullable=False, unique=True, comment="Resource name") @@ -134,7 +134,7 @@ class Resource(Base): for res in ["shout", "topic", "reaction", "chat", "message", "invite", "community", "user"]: r = session.query(Resource).filter(Resource.name == res).first() if not r: - r = Resource.create(name=res, resource_class=res) + r = Resource.create(name=res, resourceClass=res) session.add(r) session.commit() @@ -142,19 +142,19 @@ class Resource(Base): class Permission(Base): __tablename__ = "permission" __table_args__ = ( - UniqueConstraint("role_id", "operation_id", "resource_id"), + UniqueConstraint("roleId", "operationId", "resourceId"), {"extend_existing": True}, ) - role_id = Column( + roleId = Column( ForeignKey("role.id", ondelete="CASCADE"), nullable=False, comment="Role" ) - operation_id = Column( + operationId = Column( ForeignKey("operation.id", ondelete="CASCADE"), nullable=False, comment="Operation", ) - resource_id = Column( + resourceId = Column( ForeignKey("resource.id", ondelete="CASCADE"), nullable=False, comment="Resource", @@ -164,11 +164,11 @@ class Permission(Base): if __name__ == "__main__": Base.metadata.create_all(engine) ops = [ - Permission(role_id=1, operation_id=1, resource_id=1), - Permission(role_id=1, operation_id=2, resource_id=1), - Permission(role_id=1, operation_id=3, resource_id=1), - Permission(role_id=1, operation_id=4, resource_id=1), - Permission(role_id=2, operation_id=4, resource_id=1), + Permission(roleId=1, operationId=1, resourceId=1), + Permission(roleId=1, operationId=2, resourceId=1), + Permission(roleId=1, operationId=3, resourceId=1), + Permission(roleId=1, operationId=4, resourceId=1), + Permission(roleId=2, operationId=4, resourceId=1), ] global_session.add_all(ops) global_session.commit() diff --git a/orm/reaction.py b/orm/reaction.py index c3a87385..e7df502f 100644 --- a/orm/reaction.py +++ b/orm/reaction.py @@ -33,7 +33,7 @@ class Reaction(Base): updatedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Last Editor") deletedAt = Column(DateTime, nullable=True, comment="Deleted at") deletedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Deleted by") - shout_id = Column(ForeignKey("shout.id"), nullable=False, index=True) + shoutId = Column(ForeignKey("shout.id"), nullable=False, index=True) replyTo = Column( ForeignKey("reaction.id"), nullable=True, comment="Reply to reaction ID" ) diff --git a/orm/shout.py b/orm/shout.py index 452d292f..72cf9053 100644 --- a/orm/shout.py +++ b/orm/shout.py @@ -13,16 +13,16 @@ class ShoutTopic(Base): __tablename__ = "shout_topic" id = None # type: ignore - shout_id = Column(ForeignKey("shout.id"), primary_key=True, index=True) - topic_id = Column(ForeignKey("topic.id"), primary_key=True, index=True) + shoutId = Column(ForeignKey("shout.id"), primary_key=True, index=True) + topicId = Column(ForeignKey("topic.id"), primary_key=True, index=True) class ShoutReactionsFollower(Base): __tablename__ = "shout_reactions_followers" id = None # type: ignore - follower_id = Column(ForeignKey("user.id"), primary_key=True, index=True) - shout_id = Column(ForeignKey("shout.id"), primary_key=True, index=True) + followerId = Column(ForeignKey("user.id"), primary_key=True, index=True) + shoutId = Column(ForeignKey("shout.id"), primary_key=True, index=True) auto = Column(Boolean, nullable=False, default=False) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" @@ -34,8 +34,8 @@ class ShoutAuthor(Base): __tablename__ = "shout_author" id = None # type: ignore - shout_id = Column(ForeignKey("shout.id"), primary_key=True, index=True) - user_id = Column(ForeignKey("user.id"), primary_key=True, index=True) + shoutId = Column(ForeignKey("shout.id"), primary_key=True, index=True) + userId = Column(ForeignKey("user.id"), primary_key=True, index=True) caption = Column(String, nullable=True, default="") diff --git a/orm/topic.py b/orm/topic.py index 3f99cb03..94ed3968 100644 --- a/orm/topic.py +++ b/orm/topic.py @@ -9,8 +9,8 @@ class TopicFollower(Base): __tablename__ = "topic_followers" id = None # type: ignore - follower_id = Column(ForeignKey("user.id"), primary_key=True, index=True) - topic_id = Column(ForeignKey("topic.id"), primary_key=True, index=True) + followerId = Column(ForeignKey("user.id"), primary_key=True, index=True) + topicId = Column(ForeignKey("topic.id"), primary_key=True, index=True) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" ) diff --git a/orm/user.py b/orm/user.py index 1f283a2b..ab30d861 100644 --- a/orm/user.py +++ b/orm/user.py @@ -11,7 +11,7 @@ from orm.rbac import Role class UserNotifications(Base): __tablename__ = "user_notifications" # id auto - user_id = Column(Integer, ForeignKey("user.id")) + userId = Column(Integer, ForeignKey("user.id")) kind = Column(String, ForeignKey("notification.kind")) values = Column(JSONType, nullable=True) # [ , .. ] @@ -20,8 +20,8 @@ class UserRating(Base): __tablename__ = "user_rating" id = None # type: ignore - rater_id = Column(ForeignKey("user.id"), primary_key=True, index=True) - user_id = Column(ForeignKey("user.id"), primary_key=True, index=True) + raterId = Column(ForeignKey("user.id"), primary_key=True, index=True) + userId = Column(ForeignKey("user.id"), primary_key=True, index=True) value = Column(Integer) @staticmethod @@ -33,16 +33,16 @@ class UserRole(Base): __tablename__ = "user_role" id = None # type: ignore - user_id = Column(ForeignKey("user.id"), primary_key=True, index=True) - role_id = Column(ForeignKey("role.id"), primary_key=True, index=True) + userId = Column(ForeignKey("user.id"), primary_key=True, index=True) + roleId = Column(ForeignKey("role.id"), primary_key=True, index=True) class AuthorFollower(Base): __tablename__ = "author_follower" id = None # type: ignore - follower_id = Column(ForeignKey("user.id"), primary_key=True, index=True) - author_id = Column(ForeignKey("user.id"), primary_key=True, index=True) + followerId = Column(ForeignKey("user.id"), primary_key=True, index=True) + authorId = Column(ForeignKey("user.id"), primary_key=True, index=True) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" ) @@ -72,7 +72,7 @@ class User(Base): links = Column(JSONType, nullable=True, comment="Links") oauth = Column(String, nullable=True) notifications = relationship(lambda: UserNotifications) - ratings = relationship(UserRating, foreign_keys=UserRating.user_id) + ratings = relationship(UserRating, foreign_keys=UserRating.userId) roles = relationship(lambda: Role, secondary=UserRole.__tablename__) oid = Column(String, nullable=True) @@ -104,9 +104,9 @@ class User(Base): scope = {} for role in self.roles: for p in role.permissions: - if p.resource_id not in scope: - scope[p.resource_id] = set() - scope[p.resource_id].add(p.operation_id) + if p.resourceId not in scope: + scope[p.resourceId] = set() + scope[p.resourceId].add(p.operationId) return scope diff --git a/orm/viewed.py b/orm/viewed.py index b269921a..60ec5cf4 100644 --- a/orm/viewed.py +++ b/orm/viewed.py @@ -6,8 +6,8 @@ from base.orm import Base, local_session class ViewedEntry(Base): __tablename__ = "viewed" - viewer_id = Column(ForeignKey("user.id"), index=True, default=1) - shout_id = Column(ForeignKey("shout.id"), index=True, default=1) + viewerId = Column(ForeignKey("user.id"), index=True, default=1) + shoutId = Column(ForeignKey("shout.id"), index=True, default=1) amount = Column(Integer, default=1) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" diff --git a/resolvers/create/editor.py b/resolvers/create/editor.py index 499ffb8c..78f5d292 100644 --- a/resolvers/create/editor.py +++ b/resolvers/create/editor.py @@ -26,7 +26,7 @@ async def create_shout(_, info, inp): new_shout = Shout.create(**inp) # NOTE: shout made by one first author - sa = ShoutAuthor.create(shout_id=new_shout.id, user_id=user.id) + sa = ShoutAuthor.create(shoutId=new_shout.id, userId=user.id) session.add(sa) reactions_follow(user, new_shout.slug, True) @@ -37,14 +37,14 @@ async def create_shout(_, info, inp): for slug in topic_slugs: topic = session.query(Topic).where(Topic.slug == slug).one() - st = ShoutTopic.create(shout_id=new_shout.id, topic_id=topic.id) + st = ShoutTopic.create(shoutId=new_shout.id, topicId=topic.id) session.add(st) tf = session.query(TopicFollower).where( - and_(TopicFollower.follower_id == user.id, TopicFollower.topic_id == topic.id) + and_(TopicFollower.followerId == user.id, TopicFollower.topicId == topic.id) ) if not tf: - tf = TopicFollower.create(follower_id=user.id, topic_id=topic.id, auto=True) + tf = TopicFollower.create(followerId=user.id, topicId=topic.id, auto=True) session.add(tf) new_shout.topic_slugs = topic_slugs @@ -74,7 +74,7 @@ async def update_shout(_, info, inp): if user_id not in authors: scopes = auth.scopes print(scopes) - if Resource.shout_id not in scopes: + if Resource.shoutId not in scopes: return {"error": "access denied"} else: shout.update(inp) @@ -82,7 +82,7 @@ async def update_shout(_, info, inp): session.add(shout) if inp.get("topics"): # remove old links - links = session.query(ShoutTopic).where(ShoutTopic.shout_id == shout.id).all() + links = session.query(ShoutTopic).where(ShoutTopic.shoutId == shout.id).all() for topiclink in links: session.delete(topiclink) # add new topic links diff --git a/resolvers/inbox/search.py b/resolvers/inbox/search.py index e7132506..15aa822f 100644 --- a/resolvers/inbox/search.py +++ b/resolvers/inbox/search.py @@ -31,7 +31,7 @@ async def search_recipients(_, info, query: str, limit: int = 50, offset: int = with local_session() as session: # followings result += session.query(AuthorFollower.author).join( - User, User.id == AuthorFollower.follower_id + User, User.id == AuthorFollower.followerId ).where( User.slug.startswith(query) ).offset(offset + len(result)).limit(more_amount) @@ -39,7 +39,7 @@ async def search_recipients(_, info, query: str, limit: int = 50, offset: int = more_amount = limit # followers result += session.query(AuthorFollower.follower).join( - User, User.id == AuthorFollower.author_id + User, User.id == AuthorFollower.authorId ).where( User.slug.startswith(query) ).offset(offset + len(result)).limit(offset + len(result) + limit) diff --git a/resolvers/zine/load.py b/resolvers/zine/load.py index 0e0c1ade..31b0a21b 100644 --- a/resolvers/zine/load.py +++ b/resolvers/zine/load.py @@ -68,7 +68,7 @@ async def load_shout(_, info, slug): for author_caption in session.query(ShoutAuthor).join(Shout).where(Shout.slug == slug): for author in shout.authors: - if author.id == author_caption.user_id: + if author.id == author_caption.userId: author.caption = author_caption.caption return shout diff --git a/resolvers/zine/profile.py b/resolvers/zine/profile.py index 1c60c6cb..85d87fe1 100644 --- a/resolvers/zine/profile.py +++ b/resolvers/zine/profile.py @@ -23,19 +23,19 @@ def add_author_stat_columns(q): user_rating_aliased = aliased(UserRating) q = q.outerjoin(shout_author_aliased).add_columns( - func.count(distinct(shout_author_aliased.shout_id)).label('shouts_stat') + func.count(distinct(shout_author_aliased.shoutId)).label('shouts_stat') ) - q = q.outerjoin(author_followers, author_followers.author_id == User.id).add_columns( - func.count(distinct(author_followers.follower_id)).label('followers_stat') + q = q.outerjoin(author_followers, author_followers.authorId == User.id).add_columns( + func.count(distinct(author_followers.followerId)).label('followers_stat') ) - q = q.outerjoin(author_following, author_following.follower_id == User.id).add_columns( - func.count(distinct(author_following.author_id)).label('followings_stat') + q = q.outerjoin(author_following, author_following.followerId == User.id).add_columns( + func.count(distinct(author_following.authorId)).label('followings_stat') ) q = q.add_columns(literal(0).label('rating_stat')) # FIXME - # q = q.outerjoin(user_rating_aliased, user_rating_aliased.user_id == User.id).add_columns( + # q = q.outerjoin(user_rating_aliased, user_rating_aliased.userId == User.id).add_columns( # # TODO: check # func.sum(user_rating_aliased.value).label('rating_stat') # ) @@ -120,7 +120,7 @@ async def get_followed_authors(_, _info, slug) -> List[User]: async def followed_authors(slug) -> List[User]: q = select(User) q = add_author_stat_columns(q) - q = q.join(AuthorFollower).join(User, User.id == AuthorFollower.follower_id).where(User.slug == slug) + q = q.join(AuthorFollower).join(User, User.id == AuthorFollower.followerId).where(User.slug == slug) return get_authors_from_query(q) @@ -132,7 +132,7 @@ async def user_followers(_, _info, slug) -> List[User]: aliased_user = aliased(User) q = q.join(AuthorFollower).join( - aliased_user, aliased_user.id == AuthorFollower.author_id + aliased_user, aliased_user.id == AuthorFollower.authorId ).where( aliased_user.slug == slug ) @@ -147,7 +147,7 @@ async def get_user_roles(slug): session.query(Role) .options(joinedload(Role.permissions)) .join(UserRole) - .where(UserRole.user_id == user.id) + .where(UserRole.userId == user.id) .all() ) @@ -193,7 +193,7 @@ async def rate_user(_, info, rated_userslug, value): def author_follow(user, slug): with local_session() as session: author = session.query(User).where(User.slug == slug).one() - af = AuthorFollower.create(follower_id=user.id, author_id=author.id) + af = AuthorFollower.create(followerId=user.id, authorId=author.id) session.add(af) session.commit() @@ -204,9 +204,9 @@ def author_unfollow(user, slug): flw = ( session.query( AuthorFollower - ).join(User, User.id == AuthorFollower.author_id).filter( + ).join(User, User.id == AuthorFollower.authorId).filter( and_( - AuthorFollower.follower_id == user.id, User.slug == slug + AuthorFollower.followerId == user.id, User.slug == slug ) ).first() ) @@ -221,7 +221,7 @@ def author_unfollow(user, slug): async def get_authors_all(_, _info): q = select(User) q = add_author_stat_columns(q) - q = q.join(ShoutAuthor, User.id == ShoutAuthor.user_id) + q = q.join(ShoutAuthor, User.id == ShoutAuthor.userId) return get_authors_from_query(q) diff --git a/resolvers/zine/reactions.py b/resolvers/zine/reactions.py index de1ed409..c85ba3a9 100644 --- a/resolvers/zine/reactions.py +++ b/resolvers/zine/reactions.py @@ -20,15 +20,15 @@ def reactions_follow(user: User, slug: str, auto=False): following = ( session.query(ShoutReactionsFollower).where(and_( - ShoutReactionsFollower.follower_id == user.id, - ShoutReactionsFollower.shout_id == shout.id, + ShoutReactionsFollower.followerId == user.id, + ShoutReactionsFollower.shoutId == shout.id, )).first() ) if not following: following = ShoutReactionsFollower.create( - follower_id=user.id, - shout_id=shout.id, + followerId=user.id, + shoutId=shout.id, auto=auto ) session.add(following) @@ -41,8 +41,8 @@ def reactions_unfollow(user, slug): following = ( session.query(ShoutReactionsFollower).where(and_( - ShoutReactionsFollower.follower_id == user.id, - ShoutReactionsFollower.shout_id == shout.id + ShoutReactionsFollower.followerId == user.id, + ShoutReactionsFollower.shoutId == shout.id )).first() ) @@ -74,7 +74,7 @@ def check_to_publish(session, user, reaction): ]: if is_published_author(user): # now count how many approvers are voted already - approvers_reactions = session.query(Reaction).where(Reaction.shout_id == reaction.shout_id).all() + approvers_reactions = session.query(Reaction).where(Reaction.shoutId == reaction.shoutId).all() approvers = [user.slug, ] for ar in approvers_reactions: a = ar.createdBy @@ -93,7 +93,7 @@ def check_to_hide(session, user, reaction): ReactionKind.UNPROOF ]: # if is_published_author(user): - approvers_reactions = session.query(Reaction).where(Reaction.shout_id == reaction.shout_id).all() + approvers_reactions = session.query(Reaction).where(Reaction.shoutId == reaction.shoutId).all() declines = 0 for r in approvers_reactions: if r.kind in [ @@ -232,7 +232,7 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0): ).join( CreatedByUser, Reaction.createdBy == CreatedByUser.id ).join( - ReactedShout, Reaction.shout_id == ReactedShout.id + ReactedShout, Reaction.shoutId == ReactedShout.id ) if by.get("shout"): diff --git a/resolvers/zine/topics.py b/resolvers/zine/topics.py index 610dc720..0a0aa29f 100644 --- a/resolvers/zine/topics.py +++ b/resolvers/zine/topics.py @@ -8,16 +8,16 @@ from orm import Shout, User def add_topic_stat_columns(q): - q = q.outerjoin(ShoutTopic, Topic.id == ShoutTopic.topic_id).add_columns( - func.count(distinct(ShoutTopic.shout_id)).label('shouts_stat') - ).outerjoin(ShoutAuthor, ShoutTopic.shout_id == ShoutAuthor.shout_id).add_columns( - func.count(distinct(ShoutAuthor.user_id)).label('authors_stat') + q = q.outerjoin(ShoutTopic, Topic.id == ShoutTopic.topicId).add_columns( + func.count(distinct(ShoutTopic.shoutId)).label('shouts_stat') + ).outerjoin(ShoutAuthor, ShoutTopic.shoutId == ShoutAuthor.shoutId).add_columns( + func.count(distinct(ShoutAuthor.userId)).label('authors_stat') ).outerjoin(TopicFollower, and_( - TopicFollower.topic_id == Topic.id, - TopicFollower.follower_id == ShoutAuthor.id + TopicFollower.topicId == Topic.id, + TopicFollower.followerId == ShoutAuthor.id )).add_columns( - func.count(distinct(TopicFollower.follower_id)).label('followers_stat') + func.count(distinct(TopicFollower.followerId)).label('followers_stat') ) q = q.group_by(Topic.id) @@ -119,7 +119,7 @@ async def topic_follow(user, slug): with local_session() as session: topic = session.query(Topic).where(Topic.slug == slug).one() - following = TopicFollower.create(topic_id=topic.id, follower=user.id) + following = TopicFollower.create(topicId=topic.id, followerId=user.id) session.add(following) session.commit() @@ -129,7 +129,7 @@ async def topic_unfollow(user, slug): sub = ( session.query(TopicFollower).join(Topic).filter( and_( - TopicFollower.follower_id == user.id, + TopicFollower.followerId == user.id, Topic.slug == slug ) ).first() @@ -145,7 +145,7 @@ async def topic_unfollow(user, slug): async def topics_random(_, info, amount=12): q = select(Topic) q = add_topic_stat_columns(q) - q = q.join(Shout, ShoutTopic.shout_id == Shout.id).group_by(Topic.id).having(func.count(Shout.id) > 2) + q = q.join(Shout, ShoutTopic.shoutId == Shout.id).group_by(Topic.id).having(func.count(Shout.id) > 2) q = q.order_by(func.random()).limit(amount) return get_topics_from_query(q) diff --git a/schema.graphql b/schema.graphql index 9933cb35..01918c01 100644 --- a/schema.graphql +++ b/schema.graphql @@ -319,8 +319,8 @@ type Operation { } type Permission { - operation_id: Int! - resource_id: Int! + operationId: Int! + resourceId: Int! } type Role { diff --git a/services/stat/viewed.py b/services/stat/viewed.py index 32445153..d749fc92 100644 --- a/services/stat/viewed.py +++ b/services/stat/viewed.py @@ -128,7 +128,7 @@ class ViewedStorage: try: shout = session.query(Shout).where(Shout.slug == shout_slug).one() shout_views = session.query(func.sum(ViewedEntry.amount)).where( - ViewedEntry.shout_id == shout.id + ViewedEntry.shoutId == shout.id ).all()[0][0] self.by_shouts[shout_slug] = shout_views self.update_topics(session, shout_slug) @@ -168,8 +168,8 @@ class ViewedStorage: viewer = session.query(User).where(User.slug == viewer).one() viewed = ViewedEntry.create(**{ - "viewer_id": viewer.id, - "shout_id": shout.id, + "viewerId": viewer.id, + "shoutId": shout.id, "amount": amount }) session.add(viewed) From 44bd4f6edefccf1b7c7362d87715e6c73c9028b1 Mon Sep 17 00:00:00 2001 From: tonyrewin Date: Wed, 30 Nov 2022 09:27:12 +0300 Subject: [PATCH 11/12] old naming --- migration/tables/comments.py | 30 +++++++++++++++--------------- migration/tables/content_items.py | 20 ++++++++++---------- migration/tables/users.py | 6 +++--- orm/collab.py | 2 +- orm/collection.py | 2 +- orm/community.py | 2 +- orm/reaction.py | 2 +- orm/shout.py | 12 ++++++------ orm/topic.py | 4 ++-- orm/user.py | 12 ++++++------ orm/viewed.py | 2 +- resolvers/create/editor.py | 12 ++++++------ resolvers/inbox/search.py | 4 ++-- resolvers/zine/load.py | 2 +- resolvers/zine/profile.py | 26 +++++++++++++------------- resolvers/zine/reactions.py | 18 +++++++++--------- resolvers/zine/topics.py | 20 ++++++++++---------- services/stat/viewed.py | 4 ++-- 18 files changed, 90 insertions(+), 90 deletions(-) diff --git a/migration/tables/comments.py b/migration/tables/comments.py index 160d1dc7..138b3327 100644 --- a/migration/tables/comments.py +++ b/migration/tables/comments.py @@ -73,7 +73,7 @@ async def migrate(entry, storage): Shout ).where(Shout.slug == shout_dict["slug"]).one() - reaction_dict["shoutId"] = shout.id + reaction_dict["shout"] = shout.id reaction_dict["createdBy"] = author.id if author else 1 reaction_dict["kind"] = ReactionKind.COMMENT @@ -91,13 +91,13 @@ async def migrate(entry, storage): ).where( User.id == reaction_dict["createdBy"] ).filter( - ShoutReactionsFollower.shoutId == reaction.shoutId + ShoutReactionsFollower.shout == reaction.shout ).first() if not following1: following1 = ShoutReactionsFollower.create( - followerId=reaction_dict["createdBy"], - shoutId=reaction.shoutId, + follower=reaction_dict["createdBy"], + shout=reaction.shout, auto=True ) session.add(following1) @@ -109,7 +109,7 @@ async def migrate(entry, storage): ).join( Topic ).where( - TopicFollower.followerId == reaction_dict["createdBy"] + TopicFollower.follower == reaction_dict["createdBy"] ).filter( Topic.slug == t ).first() @@ -120,8 +120,8 @@ async def migrate(entry, storage): ).where(Topic.slug == t).one() topic_following = TopicFollower.create( - followerId=reaction_dict["createdBy"], - topicId=topic.id, + follower=reaction_dict["createdBy"], + topic=topic.id, auto=True ) session.add(topic_following) @@ -134,7 +134,7 @@ async def migrate(entry, storage): .first() ) re_reaction_dict = { - "shoutId": reaction_dict["shoutId"], + "shout": reaction_dict["shout"], "replyTo": reaction.id, "kind": ReactionKind.LIKE if comment_rating_old["value"] > 0 @@ -150,14 +150,14 @@ async def migrate(entry, storage): following2 = session.query( ShoutReactionsFollower ).where( - ShoutReactionsFollower.followerId == re_reaction_dict['createdBy'] + ShoutReactionsFollower.follower == re_reaction_dict['createdBy'] ).filter( - ShoutReactionsFollower.shoutId == rr.shoutId + ShoutReactionsFollower.shout == rr.shout ).first() if not following2: following2 = ShoutReactionsFollower.create( - followerId=re_reaction_dict['createdBy'], - shoutId=rr.shoutId, + follower=re_reaction_dict['createdBy'], + shout=rr.shout, auto=True ) session.add(following2) @@ -190,13 +190,13 @@ def migrate_2stage(rr, old_new_id): session.add(comment) srf = session.query(ShoutReactionsFollower).where( - ShoutReactionsFollower.shoutId == comment.shoutId + ShoutReactionsFollower.shout == comment.shout ).filter( - ShoutReactionsFollower.followerId == comment.createdBy + ShoutReactionsFollower.follower == comment.createdBy ).first() if not srf: - srf = ShoutReactionsFollower.create(shoutId=comment.shoutId, followerId=comment.createdBy, auto=True) + srf = ShoutReactionsFollower.create(shout=comment.shout, follower=comment.createdBy, auto=True) session.add(srf) session.commit() diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index 5d1b67e9..628a4a1b 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -91,12 +91,12 @@ async def create_shout(shout_dict, userslug): ).join( User ).where( - ShoutReactionsFollower.shoutId == s.id + ShoutReactionsFollower.shout == s.id ).filter( User.slug == userslug ).first() if not srf: - srf = ShoutReactionsFollower.create(shoutId=s.id, followerId=follower.id, auto=True) + srf = ShoutReactionsFollower.create(shout=s.id, follower=follower.id, auto=True) session.add(srf) session.commit() @@ -226,15 +226,15 @@ async def add_topics_follower(entry, storage, userslug): tf = session.query( TopicFollower ).where( - TopicFollower.followerId == follower.id + TopicFollower.follower == follower.id ).filter( - TopicFollower.topicId == topic.id + TopicFollower.topic == topic.id ).first() if not tf: tf = TopicFollower.create( - topicId=topic.id, - followerId=follower.id, + topic=topic.id, + follower=follower.id, auto=True ) session.add(tf) @@ -325,7 +325,7 @@ async def topics_aftermath(entry, storage): .first() ) if shout_topic_old: - shout_topic_old.update({"topicId": new_topic.id}) + shout_topic_old.update({"topic": new_topic.id}) else: shout_topic_new = ( session.query(ShoutTopic) @@ -338,7 +338,7 @@ async def topics_aftermath(entry, storage): if not shout_topic_new: try: ShoutTopic.create( - **{"shoutId": shout.id, "topicId": new_topic.id} + **{"shout": shout.id, "topic": new_topic.id} ) except Exception: print("[migration] shout topic error: " + newslug) @@ -373,14 +373,14 @@ async def content_ratings_to_reactions(entry, slug): if content_rating["value"] > 0 else ReactionKind.DISLIKE, "createdBy": reactedBy.id, - "shoutId": shout.id, + "shout": shout.id, } cts = content_rating.get("createdAt") if cts: reaction_dict["createdAt"] = date_parse(cts) reaction = ( session.query(Reaction).filter( - Reaction.shoutId == reaction_dict["shoutId"] + Reaction.shout == reaction_dict["shout"] ).filter( Reaction.createdBy == reaction_dict["createdBy"] ).filter( diff --git a/migration/tables/users.py b/migration/tables/users.py index 320dd50e..1917ed47 100644 --- a/migration/tables/users.py +++ b/migration/tables/users.py @@ -124,14 +124,14 @@ def migrate_2stage(entry, id_map): user_rating_dict = { "value": rating_entry["value"], "raterId": rater.id, - "userId": user.id, + "user": user.id, } user_rating = UserRating.create(**user_rating_dict) if user_rating_dict['value'] > 0: af = AuthorFollower.create( - authorId=user.id, - followerId=rater.id, + author=user.id, + follower=rater.id, auto=True ) session.add(af) diff --git a/orm/collab.py b/orm/collab.py index accf650c..094c6391 100644 --- a/orm/collab.py +++ b/orm/collab.py @@ -11,7 +11,7 @@ class CollabAuthor(Base): id = None # type: ignore collab = Column(ForeignKey("collab.id"), primary_key=True) - authorId = Column(ForeignKey("user.id"), primary_key=True) + author = Column(ForeignKey("user.id"), primary_key=True) accepted = Column(Boolean, default=False) diff --git a/orm/collection.py b/orm/collection.py index 3f6eee3d..27dc52e4 100644 --- a/orm/collection.py +++ b/orm/collection.py @@ -9,7 +9,7 @@ class ShoutCollection(Base): __tablename__ = "shout_collection" id = None # type: ignore - shoutId = Column(ForeignKey("shout.id"), primary_key=True) + shout = Column(ForeignKey("shout.id"), primary_key=True) collectionId = Column(ForeignKey("collection.id"), primary_key=True) diff --git a/orm/community.py b/orm/community.py index c9d718e3..66ea5891 100644 --- a/orm/community.py +++ b/orm/community.py @@ -8,7 +8,7 @@ class CommunityFollower(Base): __tablename__ = "community_followers" id = None # type: ignore - followerId = Column(ForeignKey("user.id"), primary_key=True) + follower = Column(ForeignKey("user.id"), primary_key=True) communityId = Column(ForeignKey("community.id"), primary_key=True) joinedAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" diff --git a/orm/reaction.py b/orm/reaction.py index e7df502f..f484808a 100644 --- a/orm/reaction.py +++ b/orm/reaction.py @@ -33,7 +33,7 @@ class Reaction(Base): updatedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Last Editor") deletedAt = Column(DateTime, nullable=True, comment="Deleted at") deletedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Deleted by") - shoutId = Column(ForeignKey("shout.id"), nullable=False, index=True) + shout = Column(ForeignKey("shout.id"), nullable=False, index=True) replyTo = Column( ForeignKey("reaction.id"), nullable=True, comment="Reply to reaction ID" ) diff --git a/orm/shout.py b/orm/shout.py index 72cf9053..55435d5f 100644 --- a/orm/shout.py +++ b/orm/shout.py @@ -13,16 +13,16 @@ class ShoutTopic(Base): __tablename__ = "shout_topic" id = None # type: ignore - shoutId = Column(ForeignKey("shout.id"), primary_key=True, index=True) - topicId = Column(ForeignKey("topic.id"), primary_key=True, index=True) + shout = Column(ForeignKey("shout.id"), primary_key=True, index=True) + topic = Column(ForeignKey("topic.id"), primary_key=True, index=True) class ShoutReactionsFollower(Base): __tablename__ = "shout_reactions_followers" id = None # type: ignore - followerId = Column(ForeignKey("user.id"), primary_key=True, index=True) - shoutId = Column(ForeignKey("shout.id"), primary_key=True, index=True) + follower = Column(ForeignKey("user.id"), primary_key=True, index=True) + shout = Column(ForeignKey("shout.id"), primary_key=True, index=True) auto = Column(Boolean, nullable=False, default=False) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" @@ -34,8 +34,8 @@ class ShoutAuthor(Base): __tablename__ = "shout_author" id = None # type: ignore - shoutId = Column(ForeignKey("shout.id"), primary_key=True, index=True) - userId = Column(ForeignKey("user.id"), primary_key=True, index=True) + shout = Column(ForeignKey("shout.id"), primary_key=True, index=True) + user = Column(ForeignKey("user.id"), primary_key=True, index=True) caption = Column(String, nullable=True, default="") diff --git a/orm/topic.py b/orm/topic.py index 94ed3968..a37dc69a 100644 --- a/orm/topic.py +++ b/orm/topic.py @@ -9,8 +9,8 @@ class TopicFollower(Base): __tablename__ = "topic_followers" id = None # type: ignore - followerId = Column(ForeignKey("user.id"), primary_key=True, index=True) - topicId = Column(ForeignKey("topic.id"), primary_key=True, index=True) + follower = Column(ForeignKey("user.id"), primary_key=True, index=True) + topic = Column(ForeignKey("topic.id"), primary_key=True, index=True) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" ) diff --git a/orm/user.py b/orm/user.py index ab30d861..076c63e7 100644 --- a/orm/user.py +++ b/orm/user.py @@ -11,7 +11,7 @@ from orm.rbac import Role class UserNotifications(Base): __tablename__ = "user_notifications" # id auto - userId = Column(Integer, ForeignKey("user.id")) + user = Column(Integer, ForeignKey("user.id")) kind = Column(String, ForeignKey("notification.kind")) values = Column(JSONType, nullable=True) # [ , .. ] @@ -21,7 +21,7 @@ class UserRating(Base): id = None # type: ignore raterId = Column(ForeignKey("user.id"), primary_key=True, index=True) - userId = Column(ForeignKey("user.id"), primary_key=True, index=True) + user = Column(ForeignKey("user.id"), primary_key=True, index=True) value = Column(Integer) @staticmethod @@ -33,7 +33,7 @@ class UserRole(Base): __tablename__ = "user_role" id = None # type: ignore - userId = Column(ForeignKey("user.id"), primary_key=True, index=True) + user = Column(ForeignKey("user.id"), primary_key=True, index=True) roleId = Column(ForeignKey("role.id"), primary_key=True, index=True) @@ -41,8 +41,8 @@ class AuthorFollower(Base): __tablename__ = "author_follower" id = None # type: ignore - followerId = Column(ForeignKey("user.id"), primary_key=True, index=True) - authorId = Column(ForeignKey("user.id"), primary_key=True, index=True) + follower = Column(ForeignKey("user.id"), primary_key=True, index=True) + author = Column(ForeignKey("user.id"), primary_key=True, index=True) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" ) @@ -72,7 +72,7 @@ class User(Base): links = Column(JSONType, nullable=True, comment="Links") oauth = Column(String, nullable=True) notifications = relationship(lambda: UserNotifications) - ratings = relationship(UserRating, foreign_keys=UserRating.userId) + ratings = relationship(UserRating, foreign_keys=UserRating.user) roles = relationship(lambda: Role, secondary=UserRole.__tablename__) oid = Column(String, nullable=True) diff --git a/orm/viewed.py b/orm/viewed.py index 60ec5cf4..be8a88e8 100644 --- a/orm/viewed.py +++ b/orm/viewed.py @@ -7,7 +7,7 @@ class ViewedEntry(Base): __tablename__ = "viewed" viewerId = Column(ForeignKey("user.id"), index=True, default=1) - shoutId = Column(ForeignKey("shout.id"), index=True, default=1) + shout = Column(ForeignKey("shout.id"), index=True, default=1) amount = Column(Integer, default=1) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" diff --git a/resolvers/create/editor.py b/resolvers/create/editor.py index 78f5d292..ed431375 100644 --- a/resolvers/create/editor.py +++ b/resolvers/create/editor.py @@ -26,7 +26,7 @@ async def create_shout(_, info, inp): new_shout = Shout.create(**inp) # NOTE: shout made by one first author - sa = ShoutAuthor.create(shoutId=new_shout.id, userId=user.id) + sa = ShoutAuthor.create(shout=new_shout.id, user=user.id) session.add(sa) reactions_follow(user, new_shout.slug, True) @@ -37,14 +37,14 @@ async def create_shout(_, info, inp): for slug in topic_slugs: topic = session.query(Topic).where(Topic.slug == slug).one() - st = ShoutTopic.create(shoutId=new_shout.id, topicId=topic.id) + st = ShoutTopic.create(shout=new_shout.id, topic=topic.id) session.add(st) tf = session.query(TopicFollower).where( - and_(TopicFollower.followerId == user.id, TopicFollower.topicId == topic.id) + and_(TopicFollower.follower == user.id, TopicFollower.topic == topic.id) ) if not tf: - tf = TopicFollower.create(followerId=user.id, topicId=topic.id, auto=True) + tf = TopicFollower.create(follower=user.id, topic=topic.id, auto=True) session.add(tf) new_shout.topic_slugs = topic_slugs @@ -74,7 +74,7 @@ async def update_shout(_, info, inp): if user_id not in authors: scopes = auth.scopes print(scopes) - if Resource.shoutId not in scopes: + if Resource.shout not in scopes: return {"error": "access denied"} else: shout.update(inp) @@ -82,7 +82,7 @@ async def update_shout(_, info, inp): session.add(shout) if inp.get("topics"): # remove old links - links = session.query(ShoutTopic).where(ShoutTopic.shoutId == shout.id).all() + links = session.query(ShoutTopic).where(ShoutTopic.shout == shout.id).all() for topiclink in links: session.delete(topiclink) # add new topic links diff --git a/resolvers/inbox/search.py b/resolvers/inbox/search.py index 15aa822f..5a1289ac 100644 --- a/resolvers/inbox/search.py +++ b/resolvers/inbox/search.py @@ -31,7 +31,7 @@ async def search_recipients(_, info, query: str, limit: int = 50, offset: int = with local_session() as session: # followings result += session.query(AuthorFollower.author).join( - User, User.id == AuthorFollower.followerId + User, User.id == AuthorFollower.follower ).where( User.slug.startswith(query) ).offset(offset + len(result)).limit(more_amount) @@ -39,7 +39,7 @@ async def search_recipients(_, info, query: str, limit: int = 50, offset: int = more_amount = limit # followers result += session.query(AuthorFollower.follower).join( - User, User.id == AuthorFollower.authorId + User, User.id == AuthorFollower.author ).where( User.slug.startswith(query) ).offset(offset + len(result)).limit(offset + len(result) + limit) diff --git a/resolvers/zine/load.py b/resolvers/zine/load.py index 31b0a21b..63896348 100644 --- a/resolvers/zine/load.py +++ b/resolvers/zine/load.py @@ -68,7 +68,7 @@ async def load_shout(_, info, slug): for author_caption in session.query(ShoutAuthor).join(Shout).where(Shout.slug == slug): for author in shout.authors: - if author.id == author_caption.userId: + if author.id == author_caption.user: author.caption = author_caption.caption return shout diff --git a/resolvers/zine/profile.py b/resolvers/zine/profile.py index 85d87fe1..652a602a 100644 --- a/resolvers/zine/profile.py +++ b/resolvers/zine/profile.py @@ -23,19 +23,19 @@ def add_author_stat_columns(q): user_rating_aliased = aliased(UserRating) q = q.outerjoin(shout_author_aliased).add_columns( - func.count(distinct(shout_author_aliased.shoutId)).label('shouts_stat') + func.count(distinct(shout_author_aliased.shout)).label('shouts_stat') ) - q = q.outerjoin(author_followers, author_followers.authorId == User.id).add_columns( - func.count(distinct(author_followers.followerId)).label('followers_stat') + q = q.outerjoin(author_followers, author_followers.author == User.id).add_columns( + func.count(distinct(author_followers.follower)).label('followers_stat') ) - q = q.outerjoin(author_following, author_following.followerId == User.id).add_columns( - func.count(distinct(author_following.authorId)).label('followings_stat') + q = q.outerjoin(author_following, author_following.follower == User.id).add_columns( + func.count(distinct(author_following.author)).label('followings_stat') ) q = q.add_columns(literal(0).label('rating_stat')) # FIXME - # q = q.outerjoin(user_rating_aliased, user_rating_aliased.userId == User.id).add_columns( + # q = q.outerjoin(user_rating_aliased, user_rating_aliased.user == User.id).add_columns( # # TODO: check # func.sum(user_rating_aliased.value).label('rating_stat') # ) @@ -120,7 +120,7 @@ async def get_followed_authors(_, _info, slug) -> List[User]: async def followed_authors(slug) -> List[User]: q = select(User) q = add_author_stat_columns(q) - q = q.join(AuthorFollower).join(User, User.id == AuthorFollower.followerId).where(User.slug == slug) + q = q.join(AuthorFollower).join(User, User.id == AuthorFollower.follower).where(User.slug == slug) return get_authors_from_query(q) @@ -132,7 +132,7 @@ async def user_followers(_, _info, slug) -> List[User]: aliased_user = aliased(User) q = q.join(AuthorFollower).join( - aliased_user, aliased_user.id == AuthorFollower.authorId + aliased_user, aliased_user.id == AuthorFollower.author ).where( aliased_user.slug == slug ) @@ -147,7 +147,7 @@ async def get_user_roles(slug): session.query(Role) .options(joinedload(Role.permissions)) .join(UserRole) - .where(UserRole.userId == user.id) + .where(UserRole.user == user.id) .all() ) @@ -193,7 +193,7 @@ async def rate_user(_, info, rated_userslug, value): def author_follow(user, slug): with local_session() as session: author = session.query(User).where(User.slug == slug).one() - af = AuthorFollower.create(followerId=user.id, authorId=author.id) + af = AuthorFollower.create(follower=user.id, author=author.id) session.add(af) session.commit() @@ -204,9 +204,9 @@ def author_unfollow(user, slug): flw = ( session.query( AuthorFollower - ).join(User, User.id == AuthorFollower.authorId).filter( + ).join(User, User.id == AuthorFollower.author).filter( and_( - AuthorFollower.followerId == user.id, User.slug == slug + AuthorFollower.follower == user.id, User.slug == slug ) ).first() ) @@ -221,7 +221,7 @@ def author_unfollow(user, slug): async def get_authors_all(_, _info): q = select(User) q = add_author_stat_columns(q) - q = q.join(ShoutAuthor, User.id == ShoutAuthor.userId) + q = q.join(ShoutAuthor, User.id == ShoutAuthor.user) return get_authors_from_query(q) diff --git a/resolvers/zine/reactions.py b/resolvers/zine/reactions.py index c85ba3a9..c0648fcd 100644 --- a/resolvers/zine/reactions.py +++ b/resolvers/zine/reactions.py @@ -20,15 +20,15 @@ def reactions_follow(user: User, slug: str, auto=False): following = ( session.query(ShoutReactionsFollower).where(and_( - ShoutReactionsFollower.followerId == user.id, - ShoutReactionsFollower.shoutId == shout.id, + ShoutReactionsFollower.follower == user.id, + ShoutReactionsFollower.shout == shout.id, )).first() ) if not following: following = ShoutReactionsFollower.create( - followerId=user.id, - shoutId=shout.id, + follower=user.id, + shout=shout.id, auto=auto ) session.add(following) @@ -41,8 +41,8 @@ def reactions_unfollow(user, slug): following = ( session.query(ShoutReactionsFollower).where(and_( - ShoutReactionsFollower.followerId == user.id, - ShoutReactionsFollower.shoutId == shout.id + ShoutReactionsFollower.follower == user.id, + ShoutReactionsFollower.shout == shout.id )).first() ) @@ -74,7 +74,7 @@ def check_to_publish(session, user, reaction): ]: if is_published_author(user): # now count how many approvers are voted already - approvers_reactions = session.query(Reaction).where(Reaction.shoutId == reaction.shoutId).all() + approvers_reactions = session.query(Reaction).where(Reaction.shout == reaction.shout).all() approvers = [user.slug, ] for ar in approvers_reactions: a = ar.createdBy @@ -93,7 +93,7 @@ def check_to_hide(session, user, reaction): ReactionKind.UNPROOF ]: # if is_published_author(user): - approvers_reactions = session.query(Reaction).where(Reaction.shoutId == reaction.shoutId).all() + approvers_reactions = session.query(Reaction).where(Reaction.shout == reaction.shout).all() declines = 0 for r in approvers_reactions: if r.kind in [ @@ -232,7 +232,7 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0): ).join( CreatedByUser, Reaction.createdBy == CreatedByUser.id ).join( - ReactedShout, Reaction.shoutId == ReactedShout.id + ReactedShout, Reaction.shout == ReactedShout.id ) if by.get("shout"): diff --git a/resolvers/zine/topics.py b/resolvers/zine/topics.py index 0a0aa29f..2ee6809b 100644 --- a/resolvers/zine/topics.py +++ b/resolvers/zine/topics.py @@ -8,16 +8,16 @@ from orm import Shout, User def add_topic_stat_columns(q): - q = q.outerjoin(ShoutTopic, Topic.id == ShoutTopic.topicId).add_columns( - func.count(distinct(ShoutTopic.shoutId)).label('shouts_stat') - ).outerjoin(ShoutAuthor, ShoutTopic.shoutId == ShoutAuthor.shoutId).add_columns( - func.count(distinct(ShoutAuthor.userId)).label('authors_stat') + q = q.outerjoin(ShoutTopic, Topic.id == ShoutTopic.topic).add_columns( + func.count(distinct(ShoutTopic.shout)).label('shouts_stat') + ).outerjoin(ShoutAuthor, ShoutTopic.shout == ShoutAuthor.shout).add_columns( + func.count(distinct(ShoutAuthor.user)).label('authors_stat') ).outerjoin(TopicFollower, and_( - TopicFollower.topicId == Topic.id, - TopicFollower.followerId == ShoutAuthor.id + TopicFollower.topic == Topic.id, + TopicFollower.follower == ShoutAuthor.id )).add_columns( - func.count(distinct(TopicFollower.followerId)).label('followers_stat') + func.count(distinct(TopicFollower.follower)).label('followers_stat') ) q = q.group_by(Topic.id) @@ -119,7 +119,7 @@ async def topic_follow(user, slug): with local_session() as session: topic = session.query(Topic).where(Topic.slug == slug).one() - following = TopicFollower.create(topicId=topic.id, followerId=user.id) + following = TopicFollower.create(topic=topic.id, follower=user.id) session.add(following) session.commit() @@ -129,7 +129,7 @@ async def topic_unfollow(user, slug): sub = ( session.query(TopicFollower).join(Topic).filter( and_( - TopicFollower.followerId == user.id, + TopicFollower.follower == user.id, Topic.slug == slug ) ).first() @@ -145,7 +145,7 @@ async def topic_unfollow(user, slug): async def topics_random(_, info, amount=12): q = select(Topic) q = add_topic_stat_columns(q) - q = q.join(Shout, ShoutTopic.shoutId == Shout.id).group_by(Topic.id).having(func.count(Shout.id) > 2) + q = q.join(Shout, ShoutTopic.shout == Shout.id).group_by(Topic.id).having(func.count(Shout.id) > 2) q = q.order_by(func.random()).limit(amount) return get_topics_from_query(q) diff --git a/services/stat/viewed.py b/services/stat/viewed.py index d749fc92..792552b9 100644 --- a/services/stat/viewed.py +++ b/services/stat/viewed.py @@ -128,7 +128,7 @@ class ViewedStorage: try: shout = session.query(Shout).where(Shout.slug == shout_slug).one() shout_views = session.query(func.sum(ViewedEntry.amount)).where( - ViewedEntry.shoutId == shout.id + ViewedEntry.shout == shout.id ).all()[0][0] self.by_shouts[shout_slug] = shout_views self.update_topics(session, shout_slug) @@ -169,7 +169,7 @@ class ViewedStorage: viewed = ViewedEntry.create(**{ "viewerId": viewer.id, - "shoutId": shout.id, + "shout": shout.id, "amount": amount }) session.add(viewed) From 3ed1857f11e9c7f0fc15942ecfd4ca985f91a4ca Mon Sep 17 00:00:00 2001 From: tonyrewin Date: Wed, 30 Nov 2022 22:47:34 +0300 Subject: [PATCH 12/12] fix-migration-replyto --- migration/__init__.py | 26 +-- migration/tables/comments.py | 286 +++++++++++++++--------------- migration/tables/content_items.py | 273 +++++++++++++--------------- migration/tables/users.py | 8 +- schema.graphql | 6 + 5 files changed, 292 insertions(+), 307 deletions(-) diff --git a/migration/__init__.py b/migration/__init__.py index eaded31b..aadbd8de 100644 --- a/migration/__init__.py +++ b/migration/__init__.py @@ -96,16 +96,16 @@ async def shouts_handle(storage, args): continue # migrate - shout = await migrateShout(entry, storage) - if shout: - storage["shouts"]["by_oid"][entry["_id"]] = shout - storage["shouts"]["by_slug"][shout["slug"]] = shout + shout_dict = await migrateShout(entry, storage) + if shout_dict: + storage["shouts"]["by_oid"][entry["_id"]] = shout_dict + storage["shouts"]["by_slug"][shout_dict["slug"]] = shout_dict # shouts.topics - if not shout["topics"]: + if not shout_dict["topics"]: print("[migration] no topics!") # with author - author: str = shout["authors"][0].dict() + author = shout_dict["authors"][0] if author["slug"] == "discours": discours_author += 1 if author["slug"] == "anonymous": @@ -114,18 +114,20 @@ async def shouts_handle(storage, args): if entry.get("published"): if "mdx" in args: - export_mdx(shout) + export_mdx(shout_dict) pub_counter += 1 # print main counter counter += 1 - print('[migration] shouts_handle %d: %s @%s' % ((counter + 1), shout["slug"], author["slug"])) + print('[migration] shouts_handle %d: %s @%s' % ( + (counter + 1), shout_dict["slug"], author["slug"] + )) - b = bs4.BeautifulSoup(shout["body"], "html.parser") - texts = [shout["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")] + b = bs4.BeautifulSoup(shout_dict["body"], "html.parser") + texts = [shout_dict["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")] texts = texts + b.findAll(text=True) topics_dataset_bodies.append(" ".join([x.strip().lower() for x in texts])) - topics_dataset_tlist.append(shout["topics"]) + topics_dataset_tlist.append(shout_dict["topics"]) else: ignored += 1 @@ -133,9 +135,7 @@ async def shouts_handle(storage, args): # ', fmt='%s') print("[migration] " + str(counter) + " content items were migrated") - print("[migration] " + str(ignored) + " content items were ignored") print("[migration] " + str(pub_counter) + " have been published") - print("[migration] " + str(discours_author) + " authored by @discours") print("[migration] " + str(anonymous_author) + " authored by @anonymous") diff --git a/migration/tables/comments.py b/migration/tables/comments.py index 138b3327..4fde9569 100644 --- a/migration/tables/comments.py +++ b/migration/tables/comments.py @@ -5,13 +5,93 @@ from dateutil.parser import parse as date_parse from base.orm import local_session from migration.html2text import html2text from orm.reaction import Reaction, ReactionKind -from orm.shout import ShoutReactionsFollower, Shout -from orm.topic import TopicFollower, Topic +from orm.shout import ShoutReactionsFollower +from orm.topic import TopicFollower from orm.user import User +from orm.shout import Shout +# from services.stat.reacted import ReactedStorage ts = datetime.now(tz=timezone.utc) +def auto_followers(session, topics, reaction_dict): + # creating shout's reactions following for reaction author + following1 = session.query( + ShoutReactionsFollower + ).where( + ShoutReactionsFollower.follower == reaction_dict["createdBy"] + ).filter( + ShoutReactionsFollower.shout == reaction_dict["shout"] + ).first() + if not following1: + following1 = ShoutReactionsFollower.create( + follower=reaction_dict["createdBy"], + shout=reaction_dict["shout"], + auto=True + ) + session.add(following1) + # creating topics followings for reaction author + for t in topics: + tf = session.query( + TopicFollower + ).where( + TopicFollower.follower == reaction_dict["createdBy"] + ).filter( + TopicFollower.topic == t['id'] + ).first() + if not tf: + topic_following = TopicFollower.create( + follower=reaction_dict["createdBy"], + topic=t['id'], + auto=True + ) + session.add(topic_following) + + +def migrate_ratings(session, entry, reaction_dict): + for comment_rating_old in entry.get("ratings", []): + rater = ( + session.query(User) + .filter(User.oid == comment_rating_old["createdBy"]) + .first() + ) + re_reaction_dict = { + "shout": reaction_dict["shout"], + "replyTo": reaction_dict["id"], + "kind": ReactionKind.LIKE + if comment_rating_old["value"] > 0 + else ReactionKind.DISLIKE, + "createdBy": rater.id if rater else 1, + } + cts = comment_rating_old.get("createdAt") + if cts: + re_reaction_dict["createdAt"] = date_parse(cts) + try: + # creating reaction from old rating + rr = Reaction.create(**re_reaction_dict) + following2 = session.query( + ShoutReactionsFollower + ).where( + ShoutReactionsFollower.follower == re_reaction_dict['createdBy'] + ).filter( + ShoutReactionsFollower.shout == rr.shout + ).first() + if not following2: + following2 = ShoutReactionsFollower.create( + follower=re_reaction_dict['createdBy'], + shout=rr.shout, + auto=True + ) + session.add(following2) + session.add(rr) + # await ReactedStorage.react(rr) + + except Exception as e: + print("[migration] comment rating error: %r" % re_reaction_dict) + raise e + session.commit() + + async def migrate(entry, storage): """ { @@ -29,9 +109,7 @@ async def migrate(entry, storage): "updatedAt": "2020-05-27 19:22:57.091000+00:00", "updatedBy": "0" } - -> - type Reaction { id: Int! shout: Shout! @@ -49,10 +127,9 @@ async def migrate(entry, storage): old_thread: String } """ + old_ts = entry.get("createdAt") reaction_dict = { - "createdAt": ( - ts if not entry.get("createdAt") else date_parse(entry.get("createdAt")) - ), + "createdAt": (ts if not old_ts else date_parse(old_ts)), "body": html2text(entry.get("body", "")), "oid": entry["_id"], } @@ -65,140 +142,71 @@ async def migrate(entry, storage): raise Exception return else: + stage = "started" + reaction = None with local_session() as session: author = session.query(User).filter(User.oid == entry["createdBy"]).first() - shout_dict = storage["shouts"]["by_oid"][shout_oid] - if shout_dict: - shout = session.query( - Shout - ).where(Shout.slug == shout_dict["slug"]).one() - - reaction_dict["shout"] = shout.id - reaction_dict["createdBy"] = author.id if author else 1 - reaction_dict["kind"] = ReactionKind.COMMENT - - # creating reaction from old comment - reaction = Reaction.create(**reaction_dict) - session.add(reaction) - - # creating shout's reactions following for reaction author - following1 = session.query( - ShoutReactionsFollower - ).join( - User - ).join( - Shout - ).where( - User.id == reaction_dict["createdBy"] - ).filter( - ShoutReactionsFollower.shout == reaction.shout - ).first() - - if not following1: - following1 = ShoutReactionsFollower.create( - follower=reaction_dict["createdBy"], - shout=reaction.shout, - auto=True - ) - session.add(following1) - - # creating topics followings for reaction author - for t in shout_dict["topics"]: - tf = session.query( - TopicFollower - ).join( - Topic - ).where( - TopicFollower.follower == reaction_dict["createdBy"] - ).filter( - Topic.slug == t - ).first() - - if not tf: - topic = session.query( - Topic - ).where(Topic.slug == t).one() - - topic_following = TopicFollower.create( - follower=reaction_dict["createdBy"], - topic=topic.id, - auto=True - ) - session.add(topic_following) - - reaction_dict["id"] = reaction.id - for comment_rating_old in entry.get("ratings", []): - rater = ( - session.query(User) - .filter(User.oid == comment_rating_old["createdBy"]) - .first() - ) - re_reaction_dict = { - "shout": reaction_dict["shout"], - "replyTo": reaction.id, - "kind": ReactionKind.LIKE - if comment_rating_old["value"] > 0 - else ReactionKind.DISLIKE, - "createdBy": rater.id if rater else 1, - } - cts = comment_rating_old.get("createdAt") - if cts: - re_reaction_dict["createdAt"] = date_parse(cts) - try: - # creating reaction from old rating - rr = Reaction.create(**re_reaction_dict) - following2 = session.query( - ShoutReactionsFollower - ).where( - ShoutReactionsFollower.follower == re_reaction_dict['createdBy'] - ).filter( - ShoutReactionsFollower.shout == rr.shout - ).first() - if not following2: - following2 = ShoutReactionsFollower.create( - follower=re_reaction_dict['createdBy'], - shout=rr.shout, - auto=True - ) - session.add(following2) - session.add(rr) - - except Exception as e: - print("[migration] comment rating error: %r" % re_reaction_dict) - raise e - session.commit() + old_shout = storage["shouts"]["by_oid"].get(shout_oid) + if not old_shout: + raise Exception("no old shout in storage") else: - print( - "[migration] error: cannot find shout for comment %r" - % reaction_dict - ) - return reaction + stage = "author and old id found" + try: + shout = session.query( + Shout + ).where(Shout.slug == old_shout["slug"]).one() + if shout: + reaction_dict["shout"] = shout.id + reaction_dict["createdBy"] = author.id if author else 1 + reaction_dict["kind"] = ReactionKind.COMMENT + + # creating reaction from old comment + reaction = Reaction.create(**reaction_dict) + session.add(reaction) + # session.commit() + stage = "new reaction commited" + reaction_dict = reaction.dict() + topics = [t.dict() for t in shout.topics] + auto_followers(session, topics, reaction_dict) + + migrate_ratings(session, entry, reaction_dict) + + return reaction + except Exception as e: + print(e) + print(reaction) + raise Exception(stage) + return -def migrate_2stage(rr, old_new_id): - reply_oid = rr.get("replyTo") - if not reply_oid: - return - - new_id = old_new_id.get(rr.get("oid")) - if not new_id: - return - - with local_session() as session: - comment = session.query(Reaction).filter(Reaction.id == new_id).first() - comment.replyTo = old_new_id.get(reply_oid) - session.add(comment) - - srf = session.query(ShoutReactionsFollower).where( - ShoutReactionsFollower.shout == comment.shout - ).filter( - ShoutReactionsFollower.follower == comment.createdBy - ).first() - - if not srf: - srf = ShoutReactionsFollower.create(shout=comment.shout, follower=comment.createdBy, auto=True) - session.add(srf) - - session.commit() - if not rr["body"]: - raise Exception(rr) +def migrate_2stage(old_comment, idmap): + if old_comment.get('body'): + new_id = idmap.get(old_comment.get('oid')) + if new_id: + new_replyto_id = None + old_replyto_id = old_comment.get("replyTo") + if old_replyto_id: + new_replyto_id = int(idmap.get(old_replyto_id, "0")) + with local_session() as session: + comment = session.query(Reaction).where(Reaction.id == new_id).first() + try: + if new_replyto_id: + new_reply = session.query(Reaction).where(Reaction.id == new_replyto_id).first() + if not new_reply: + print(new_replyto_id) + raise Exception("cannot find reply by id!") + comment.replyTo = new_reply.id + session.add(comment) + srf = session.query(ShoutReactionsFollower).where( + ShoutReactionsFollower.shout == comment.shout + ).filter( + ShoutReactionsFollower.follower == comment.createdBy + ).first() + if not srf: + srf = ShoutReactionsFollower.create( + shout=comment.shout, follower=comment.createdBy, auto=True + ) + session.add(srf) + session.commit() + except Exception: + raise Exception("cannot find a comment by oldid") diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index 628a4a1b..158a2253 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -9,7 +9,9 @@ from orm.reaction import Reaction, ReactionKind from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower from orm.user import User from orm.topic import TopicFollower, Topic +# from services.stat.reacted import ReactedStorage from services.stat.viewed import ViewedStorage +import re OLD_DATE = "2016-03-05 22:22:00.350000" ts = datetime.now(tz=timezone.utc) @@ -21,6 +23,8 @@ type2layout = { "Image": "image", } +anondict = {"slug": "anonymous", "id": 1, "name": "Аноним"} + def get_shout_slug(entry): slug = entry.get("slug", "") @@ -29,6 +33,7 @@ def get_shout_slug(entry): slug = friend.get("slug", "") if slug: break + slug = re.sub('[^0-9a-zA-Z]+', '-', slug) return slug @@ -39,13 +44,8 @@ def create_author_from_app(app): user = session.query(User).where(User.email == app['email']).first() if not user: name = app.get('name') - slug = ( - translit(name, "ru", reversed=True) - .replace(" ", "-") - .replace("'", "") - .replace(".", "-") - .lower() - ) + slug = translit(name, "ru", reversed=True).lower() + slug = re.sub('[^0-9a-zA-Z]+', '-', slug) # check if nameslug is used user = session.query(User).where(User.slug == slug).first() # get slug from email @@ -74,122 +74,99 @@ def create_author_from_app(app): session.commit() userdata = user.dict() if not userdata: - userdata = User.default_user.dict() + userdata = User.default_user.dict() # anonymous except Exception as e: print(app) raise e return userdata -async def create_shout(shout_dict, userslug): +async def create_shout(shout_dict, user): s = Shout.create(**shout_dict) with local_session() as session: - follower = session.query(User).where(User.slug == userslug).one() - - srf = session.query( - ShoutReactionsFollower - ).join( - User - ).where( + srf = session.query(ShoutReactionsFollower).where( ShoutReactionsFollower.shout == s.id ).filter( - User.slug == userslug + ShoutReactionsFollower.follower == user.id ).first() if not srf: - srf = ShoutReactionsFollower.create(shout=s.id, follower=follower.id, auto=True) + srf = ShoutReactionsFollower.create(shout=s.id, follower=user.id, auto=True) session.add(srf) session.commit() + return s + + +def get_userdata(entry, storage): + user_oid = entry.get("createdBy", "") + userdata = None + app = entry.get("application") + if app: + userdata = create_author_from_app(app) or anondict + else: + userdata = storage["users"]["by_oid"].get(user_oid) or anondict + slug = userdata.get("slug") + slug = re.sub('[^0-9a-zA-Z]+', '-', slug) + userdata["slug"] = slug + return userdata, user_oid async def migrate(entry, storage): - # init, set title and layout + userdata, user_oid = get_userdata(entry, storage) + user = await get_user(userdata, storage, user_oid) r = { "layout": type2layout[entry["type"]], "title": entry["title"], - "authors": [], - "topics": set([]) + "authors": [userdata["slug"], ], + "slug": get_shout_slug(entry), + "cover": ( + "https://assets.discours.io/unsafe/1600x/" + + entry["thumborId"] if entry.get("thumborId") else entry.get("image", {}).get("url") + ), + "visibility": "public" if entry.get("published") else "authors", + "publishedAt": date_parse(entry.get("publishedAt")) if entry.get("published") else None, + "deletedAt": date_parse(entry.get("deletedAt")) if entry.get("deletedAt") else None, + "createdAt": date_parse(entry.get("createdAt", OLD_DATE)), + "updatedAt": date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts, + "topics": await add_topics_follower(entry, storage, user), + "body": extract_html(entry) } - # author - users_by_oid = storage["users"]["by_oid"] - user_oid = entry.get("createdBy", "") - userdata = users_by_oid.get(user_oid) - user = None - if not userdata: - app = entry.get("application") - if app: - userdata = create_author_from_app(app) - if userdata: - userslug = userdata.get('slug') - else: - userslug = "anonymous" # bad old id slug was found - r["authors"] = [userslug, ] + # main topic patch + r['mainTopic'] = r['topics'][0] - # slug - slug = get_shout_slug(entry) - if slug: - r["slug"] = slug - else: - raise Exception - - # cover - c = "" - if entry.get("thumborId"): - c = "https://assets.discours.io/unsafe/1600x/" + entry["thumborId"] - else: - c = entry.get("image", {}).get("url") - if not c or "cloudinary" in c: - c = "" - r["cover"] = c - - # timestamps - r["createdAt"] = date_parse(entry.get("createdAt", OLD_DATE)) - r["updatedAt"] = date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts - - # visibility + # published author auto-confirm if entry.get("published"): - r["publishedAt"] = date_parse(entry.get("publishedAt", OLD_DATE)) - r["visibility"] = "public" with local_session() as session: # update user.emailConfirmed if published - author = session.query(User).where(User.slug == userslug).first() + author = session.query(User).where(User.slug == userdata["slug"]).first() author.emailConfirmed = True session.add(author) session.commit() - else: - r["visibility"] = "authors" - if "deletedAt" in entry: - r["deletedAt"] = date_parse(entry["deletedAt"]) - - # topics - r['topics'] = await add_topics_follower(entry, storage, userslug) - r['mainTopic'] = r['topics'][0] - - entry["topics"] = r["topics"] - entry["cover"] = r["cover"] - - # body - r["body"] = extract_html(entry) + # media media = extract_media(entry) - if media: - r["media"] = json.dumps(media, ensure_ascii=True) + r["media"] = json.dumps(media, ensure_ascii=True) if media else None + # ----------------------------------- copy shout_dict = r.copy() # user - user = await get_user(userslug, userdata, storage, user_oid) shout_dict["authors"] = [user, ] del shout_dict["topics"] try: # save shout to db - await create_shout(shout_dict, userslug) + shout_dict["oid"] = entry.get("_id", "") + shout = await create_shout(shout_dict, user) except IntegrityError as e: - print(e) - await resolve_create_shout(shout_dict, userslug) + print('[migration] create_shout integrity error', e) + shout = await resolve_create_shout(shout_dict, userdata["slug"]) except Exception as e: raise Exception(e) + # udpate data + shout_dict = shout.dict() + shout_dict["authors"] = [user.dict(), ] + # shout topics aftermath shout_dict["topics"] = await topics_aftermath(r, storage) @@ -200,13 +177,12 @@ async def migrate(entry, storage): await ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1)) # del shout_dict['ratings'] - shout_dict["oid"] = entry.get("_id", "") storage["shouts"]["by_oid"][entry["_id"]] = shout_dict - storage["shouts"]["by_slug"][slug] = shout_dict + storage["shouts"]["by_slug"][shout_dict["slug"]] = shout_dict return shout_dict -async def add_topics_follower(entry, storage, userslug): +async def add_topics_follower(entry, storage, user): topics = set([]) category = entry.get("category") topics_by_oid = storage["topics"]["by_oid"] @@ -218,29 +194,26 @@ async def add_topics_follower(entry, storage, userslug): ttt = list(topics) # add author as TopicFollower with local_session() as session: - for tpc in topics: + for tpcslug in topics: try: - topic = session.query(Topic).where(Topic.slug == tpc).one() - follower = session.query(User).where(User.slug == userslug).one() - + tpc = session.query(Topic).where(Topic.slug == tpcslug).first() tf = session.query( TopicFollower ).where( - TopicFollower.follower == follower.id + TopicFollower.follower == user.id ).filter( - TopicFollower.topic == topic.id + TopicFollower.topic == tpc.id ).first() - if not tf: tf = TopicFollower.create( - topic=topic.id, - follower=follower.id, + topic=tpc.id, + follower=user.id, auto=True ) session.add(tf) session.commit() except IntegrityError: - print('[migration.shout] hidden by topic ' + tpc) + print('[migration.shout] hidden by topic ' + tpc.slug) # main topic maintopic = storage["replacements"].get(topics_by_oid.get(category, {}).get("slug")) if maintopic in ttt: @@ -249,23 +222,28 @@ async def add_topics_follower(entry, storage, userslug): return ttt -async def get_user(userslug, userdata, storage, oid): +async def get_user(userdata, storage, oid): user = None with local_session() as session: - if not user and userslug: - user = session.query(User).filter(User.slug == userslug).first() - if not user and userdata: + uid = userdata.get("id") + if uid: + user = session.query(User).filter(User.id == uid).first() + elif userdata: try: - userdata["slug"] = userdata["slug"].lower().strip().replace(" ", "-") + slug = userdata["slug"].lower().strip() + slug = re.sub('[^0-9a-zA-Z]+', '-', slug) + userdata["slug"] = slug user = User.create(**userdata) session.add(user) session.commit() except IntegrityError: - print("[migration] user error: " + userdata) - userdata["id"] = user.id - userdata["createdAt"] = user.createdAt - storage["users"]["by_slug"][userdata["slug"]] = userdata - storage["users"]["by_oid"][oid] = userdata + print("[migration] user creating with slug %s" % userdata["slug"]) + print("[migration] from userdata: %r" % userdata) + raise Exception("[migration] cannot create user in content_items.get_user()") + userdata["id"] = user.id + userdata["createdAt"] = user.createdAt + storage["users"]["by_slug"][userdata["slug"]] = userdata + storage["users"]["by_oid"][oid] = userdata if not user: raise Exception("could not get a user") return user @@ -303,6 +281,7 @@ async def resolve_create_shout(shout_dict, userslug): print("[migration] something went wrong with shout: \n%r" % shout_dict) raise Exception("") session.commit() + return s async def topics_aftermath(entry, storage): @@ -318,22 +297,22 @@ async def topics_aftermath(entry, storage): shout_topic_old = ( session.query(ShoutTopic) - .join(Shout) - .join(Topic) - .filter(Shout.slug == entry["slug"]) - .filter(Topic.slug == oldslug) - .first() + .join(Shout) + .join(Topic) + .filter(Shout.slug == entry["slug"]) + .filter(Topic.slug == oldslug) + .first() ) if shout_topic_old: shout_topic_old.update({"topic": new_topic.id}) else: shout_topic_new = ( session.query(ShoutTopic) - .join(Shout) - .join(Topic) - .filter(Shout.slug == entry["slug"]) - .filter(Topic.slug == newslug) - .first() + .join(Shout) + .join(Topic) + .filter(Shout.slug == entry["slug"]) + .filter(Topic.slug == newslug) + .first() ) if not shout_topic_new: try: @@ -357,45 +336,37 @@ async def content_ratings_to_reactions(entry, slug): for content_rating in entry.get("ratings", []): rater = ( session.query(User) - .filter(User.oid == content_rating["createdBy"]) - .first() + .filter(User.oid == content_rating["createdBy"]) + .first() + ) or User.default_user + shout = session.query(Shout).where(Shout.slug == slug).first() + cts = content_rating.get("createdAt") + reaction_dict = { + "createdAt": date_parse(cts) if cts else None, + "kind": ReactionKind.LIKE + if content_rating["value"] > 0 + else ReactionKind.DISLIKE, + "createdBy": rater.id, + "shout": shout.id + } + reaction = ( + session.query(Reaction) + .filter(Reaction.shout == reaction_dict["shout"]) + .filter(Reaction.createdBy == reaction_dict["createdBy"]) + .filter(Reaction.kind == reaction_dict["kind"]) + .first() ) - reactedBy = ( - rater - if rater - else session.query(User).filter(User.slug == "anonymous").first() - ) - if rater: - shout = session.query(Shout).where(Shout.slug == slug).one() - - reaction_dict = { - "kind": ReactionKind.LIKE - if content_rating["value"] > 0 - else ReactionKind.DISLIKE, - "createdBy": reactedBy.id, - "shout": shout.id, - } - cts = content_rating.get("createdAt") - if cts: - reaction_dict["createdAt"] = date_parse(cts) - reaction = ( - session.query(Reaction).filter( - Reaction.shout == reaction_dict["shout"] - ).filter( - Reaction.createdBy == reaction_dict["createdBy"] - ).filter( - Reaction.kind == reaction_dict["kind"] - ).first() - ) - if reaction: - k = ReactionKind.AGREE if content_rating["value"] > 0 else ReactionKind.DISAGREE - reaction_dict["kind"] = k - reaction.update(reaction_dict) - else: - rea = Reaction.create(**reaction_dict) - session.add(rea) - # shout_dict['ratings'].append(reaction_dict) + if reaction: + k = ReactionKind.AGREE if content_rating["value"] > 0 else ReactionKind.DISAGREE + reaction_dict["kind"] = k + reaction.update(reaction_dict) + session.add(reaction) + else: + rea = Reaction.create(**reaction_dict) + session.add(rea) + # await ReactedStorage.react(rea) + # shout_dict['ratings'].append(reaction_dict) session.commit() except Exception: - raise Exception("[migration] content_item.ratings error: \n%r" % content_rating) + print("[migration] content_item.ratings error: \n%r" % content_rating) diff --git a/migration/tables/users.py b/migration/tables/users.py index 1917ed47..cb3e87bc 100644 --- a/migration/tables/users.py +++ b/migration/tables/users.py @@ -1,7 +1,7 @@ from dateutil.parser import parse from sqlalchemy.exc import IntegrityError from bs4 import BeautifulSoup - +import re from base.orm import local_session from orm.user import AuthorFollower, User, UserRating @@ -32,9 +32,9 @@ def migrate(entry): user_dict["lastSeen"] = parse(entry["wasOnlineAt"]) if entry.get("profile"): # slug - user_dict["slug"] = ( - entry["profile"].get("path").lower().replace(" ", "-").strip() - ) + slug = entry["profile"].get("path").lower() + slug = re.sub('[^0-9a-zA-Z]+', '-', slug).strip() + user_dict["slug"] = slug bio = BeautifulSoup(entry.get("profile").get("bio") or "", features="lxml").text if bio.startswith('<'): print('[migration] bio! ' + bio) diff --git a/schema.graphql b/schema.graphql index 01918c01..5b84ae54 100644 --- a/schema.graphql +++ b/schema.graphql @@ -371,6 +371,12 @@ type User { oid: String } +type Draft { + title: String + body: String + createdBy: Int +} + type Collab { authors: [String]! invites: [String]