From aba5e11522ac48d444b940d192167120c14d17db Mon Sep 17 00:00:00 2001 From: tonyrewin Date: Sat, 13 Aug 2022 19:19:16 +0300 Subject: [PATCH] some-upgrades --- migration/tables/comments.py | 8 ++++- migration/tables/content_items.py | 6 +++- requirements.txt | 1 + resolvers/zine.py | 6 ++-- schema.graphql | 1 + services/stat/reacted.py | 60 +++++++++++++++++++------------ services/stat/topicstat.py | 2 +- services/stat/viewed.py | 19 ++++++---- services/zine/gittask.py | 2 +- services/zine/shoutscache.py | 2 +- services/zine/topics.py | 2 +- 11 files changed, 70 insertions(+), 39 deletions(-) diff --git a/migration/tables/comments.py b/migration/tables/comments.py index df07da3c..0f426476 100644 --- a/migration/tables/comments.py +++ b/migration/tables/comments.py @@ -5,6 +5,7 @@ from base.orm import local_session from migration.html2text import html2text from orm.reaction import ReactionKind from orm.shout import Shout +from services.stat.reacted import ReactedByDay ts = datetime.now() @@ -70,7 +71,9 @@ def migrate(entry, storage): reaction_dict['kind'] = ReactionKind.COMMENT # creating reaction from old comment + day = (reaction_dict.get('createdAt') or ts).replace(hour=0, minute=0, second=0, microsecond=0) reaction = Reaction.create(**reaction_dict) + ReactedByDay.create(shout=reaction.shout, reaction=reaction.id, kind=reaction.kind, day=day) reaction_dict['id'] = reaction.id for comment_rating_old in entry.get('ratings',[]): @@ -86,7 +89,10 @@ def migrate(entry, storage): if cts: re_reaction_dict['createdAt'] = date_parse(cts) try: # creating reaction from old rating - Reaction.create(**re_reaction_dict) + rr = Reaction.create(**re_reaction_dict) + day = (re_reaction_dict.get('createdAt') or ts).replace(hour=0, minute=0, second=0, microsecond=0) + ReactedByDay.create(shout=rr.shout, reaction=rr.id, kind=rr.kind, day=day) + except Exception as e: print('[migration] comment rating error: %r' % re_reaction_dict) raise e diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index 5db08b34..470ed1b8 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -1,6 +1,7 @@ from dateutil.parser import parse as date_parse import sqlalchemy from orm.shout import Shout, ShoutTopic, User +from services.stat.reacted import ReactedByDay from services.stat.viewed import ViewedByDay from transliterate import translit from datetime import datetime @@ -210,7 +211,10 @@ def migrate(entry, storage): if reaction: reaction_dict['kind'] = ReactionKind.AGREE if content_rating['value'] > 0 else ReactionKind.DISAGREE, reaction.update(reaction_dict) - else: Reaction.create(**reaction_dict) + else: + day = (reaction_dict.get('createdAt') or ts).replace(hour=0, minute=0, second=0, microsecond=0) + rea = Reaction.create(**reaction_dict) + ReactedByDay.create(shout=rea.shout, reaction=rea.id, kind=rea.kind, day=day) # shout_dict['ratings'].append(reaction_dict) except: print('[migration] content_item.ratings error: \n%r' % content_rating) diff --git a/requirements.txt b/requirements.txt index 6ff8641b..1ce909d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +frontmatter aioredis ariadne pyjwt>=2.0.0 diff --git a/resolvers/zine.py b/resolvers/zine.py index 530bc365..5d412616 100644 --- a/resolvers/zine.py +++ b/resolvers/zine.py @@ -54,14 +54,14 @@ async def get_shout_by_slug(_, info, slug): shout = None # FIXME: append captions anyhow with local_session() as session: - shout = session.query(Shout).\ + shout = session.query(Shout, ShoutAuthor.caption.label("author_caption")).\ options([ selectinload(Shout.topics), selectinload(Shout.reactions), - joinedload(Shout.authors, innerjoin=True), + joinedload(Shout.authors), selectinload(ShoutAuthor.caption) ]).\ - join(ShoutAuthor.caption.label('caption'), ShoutAuthor.shout == slug ).\ + join(ShoutAuthor.shout == slug ).\ filter(Shout.slug == slug).first() if not shout: diff --git a/schema.graphql b/schema.graphql index d75c0891..7686df07 100644 --- a/schema.graphql +++ b/schema.graphql @@ -440,6 +440,7 @@ type TopicStat { authors: Int! viewed: Int! reacted: Int! + rating: Int } type Topic { diff --git a/services/stat/reacted.py b/services/stat/reacted.py index 8fc3905a..40a71cca 100644 --- a/services/stat/reacted.py +++ b/services/stat/reacted.py @@ -1,11 +1,10 @@ import asyncio from datetime import datetime -from typing_extensions import Self from sqlalchemy.types import Enum from sqlalchemy import Column, DateTime, ForeignKey, Integer -from sqlalchemy.orm.attributes import flag_modified +# from sqlalchemy.orm.attributes import flag_modified from base.orm import Base, local_session -from orm.reaction import ReactionKind, kind_to_rate +from orm.reaction import Reaction, ReactionKind, kind_to_rate from orm.topic import ShoutTopic class ReactedByDay(Base): @@ -14,7 +13,7 @@ class ReactedByDay(Base): id = None reaction = Column(ForeignKey("reaction.id"), primary_key = True) shout = Column(ForeignKey('shout.slug'), primary_key=True) - reply = Column(ForeignKey('reaction.id'), primary_key=True, nullable=True) + reply = Column(ForeignKey('reaction.id'), nullable=True) kind: int = Column(Enum(ReactionKind), nullable=False, comment="Reaction kind") day = Column(DateTime, primary_key=True, default=datetime.now) @@ -38,39 +37,57 @@ class ReactedStorage: def init(session): self = ReactedStorage all_reactions = session.query(ReactedByDay).all() - day_start = datetime.now().replace(hour=0, minute=0, second=0) - for reaction in all_reactions: - day = reaction.day + all_reactions2 = session.query(Reaction).filter(Reaction.deletedAt == None).all() + print('[stat.reacted] %d reactions total' % len(all_reactions or all_reactions2)) + rrr = (all_reactions or all_reactions2) + create = False + if not all_reactions: create = True + for reaction in rrr: shout = reaction.shout topics = session.query(ShoutTopic.topic).where(ShoutTopic.shout == shout).all() kind = reaction.kind - self.reacted['shouts'][shout] = self.reacted['shouts'].get(shout, 0) + 1 + self.reacted['shouts'][shout] = self.reacted['shouts'].get(shout, []) + self.reacted['shouts'][shout].append(reaction) self.rating['shouts'][shout] = self.rating['shouts'].get(shout, 0) + kind_to_rate(kind) for t in topics: - self.reacted['topics'][t] = self.reacted['topics'].get(t, 0) + 1 # reactions amount + self.reacted['topics'][t] = self.reacted['topics'].get(t, []) + self.reacted['topics'][t].append(reaction) self.rating['topics'][t] = self.rating['topics'].get(t, 0) + kind_to_rate(kind) # rating - if reaction.reply: - self.reacted['reactions'][reaction.reply] = self.reacted['reactions'].get(reaction.reply, 0) + 1 - self.rating['reactions'][reaction.reply] = self.rating['reactions'].get(reaction.reply, 0) + kind_to_rate(reaction.kind) - + if reaction.replyTo: + self.reacted['reactions'][reaction.replyTo] = self.reacted['reactions'].get(reaction.replyTo, []) + self.reacted['reactions'][reaction.replyTo].append(reaction) + self.rating['reactions'][reaction.replyTo] = self.rating['reactions'].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind) + ttt = self.reacted['topics'].values() + print('[stat.reacted] %d topics reacted' % len(ttt)) print('[stat.reacted] %d shouts reacted' % len(self.reacted['shouts'])) print('[stat.reacted] %d reactions reacted' % len(self.reacted['reactions'])) - + + if len(all_reactions) == 0 and len(all_reactions2) != 0: + with local_session() as session: + for r in all_reactions2: + session.add(ReactedByDay(reaction=r.id, shout=r.shout, reply=r.replyTo, kind=r.kind, day=r.createdAt.replace(hour=0, minute=0, second=0))) + session.commit() @staticmethod async def get_shout(shout_slug): self = ReactedStorage async with self.lock: - return self.reacted['shouts'].get(shout_slug, 0) + return self.reacted['shouts'].get(shout_slug, []) @staticmethod async def get_topic(topic_slug): self = ReactedStorage async with self.lock: - return self.reacted['topics'].get(topic_slug, 0) + return self.reacted['topics'].get(topic_slug, []) + + @staticmethod + async def get_reaction(reaction_id): + self = ReactedStorage + async with self.lock: + return self.reacted['reactions'].get(reaction_id, []) @staticmethod async def get_rating(shout_slug): @@ -84,12 +101,6 @@ class ReactedStorage: async with self.lock: return self.rating['topics'].get(topic_slug, 0) - @staticmethod - async def get_reaction(reaction_id): - self = ReactedStorage - async with self.lock: - return self.reacted['reactions'].get(reaction_id, 0) - @staticmethod async def get_reaction_rating(reaction_id): self = ReactedStorage @@ -107,4 +118,7 @@ class ReactedStorage: self.reacted['shouts'][shout_slug].append(reaction) if reply_id: self.reacted['reaction'][reply_id] = self.reacted['reactions'].get(shout_slug, []) - self.reacted['reaction'][reply_id].append(reaction) \ No newline at end of file + self.reacted['reaction'][reply_id].append(reaction) + self.rating['reactions'][reply_id] = self.rating['reactions'].get(reply_id, 0) + kind_to_rate(kind) + else: + self.rating['shouts'][shout_slug] = self.rating['shouts'].get(shout_slug, 0) + kind_to_rate(kind) \ No newline at end of file diff --git a/services/stat/topicstat.py b/services/stat/topicstat.py index 2bb35c16..146f6a81 100644 --- a/services/stat/topicstat.py +++ b/services/stat/topicstat.py @@ -78,7 +78,7 @@ class TopicStat: with local_session() as session: async with self.lock: await self.load_stat(session) - print("[stat.topics] updated") + print("[stat.topics] periodical update") except Exception as err: print("[stat.topics] errror: %s" % (err)) await asyncio.sleep(self.period) diff --git a/services/stat/viewed.py b/services/stat/viewed.py index 4ef3ee9e..b8cbbdd7 100644 --- a/services/stat/viewed.py +++ b/services/stat/viewed.py @@ -18,7 +18,8 @@ class ViewedByDay(Base): class ViewedStorage: viewed = { 'shouts': {}, - 'topics': {} + 'topics': {}, + 'reactions': {} } this_day_views = {} to_flush = [] @@ -46,7 +47,7 @@ class ViewedStorage: if this_day_view.day < view.day: self.this_day_views[shout] = view - print('[stat.viewed] watching %d shouts' % len(views)) + print('[stat.viewed] %d shouts viewed' % len(views)) @staticmethod async def get_shout(shout_slug): @@ -60,6 +61,12 @@ class ViewedStorage: async with self.lock: return self.viewed['topics'].get(topic_slug, 0) + @staticmethod + async def get_reaction(reaction_id): + self = ViewedStorage + async with self.lock: + return self.viewed['reactions'].get(reaction_id, 0) + @staticmethod async def increment(shout_slug): self = ViewedStorage @@ -74,13 +81,11 @@ class ViewedStorage: else: this_day_view.value = this_day_view.value + 1 this_day_view.modified = True - old_value = self.viewed['shouts'].get(shout_slug, 0) - self.viewed['shouts'][shout_slug] = old_value + 1 + self.viewed['shouts'][shout_slug] = self.viewed['shouts'].get(shout_slug, 0) + 1 with local_session() as session: topics = session.query(ShoutTopic.topic).where(ShoutTopic.shout == shout_slug).all() for t in topics: - old_topic_value = self.viewed['topics'].get(t, 0) - self.viewed['topics'][t] = old_topic_value + 1 + self.viewed['topics'][t] = self.viewed['topics'].get(t, 0) + 1 flag_modified(this_day_view, "value") @@ -104,7 +109,7 @@ class ViewedStorage: try: with local_session() as session: await ViewedStorage.flush_changes(session) - print("[stat.viewed] service flushed changes") + print("[stat.viewed] periodical flush") except Exception as err: print("[stat.viewed] errror: %s" % (err)) await asyncio.sleep(ViewedStorage.period) diff --git a/services/zine/gittask.py b/services/zine/gittask.py index 5dd8e541..c4d69b31 100644 --- a/services/zine/gittask.py +++ b/services/zine/gittask.py @@ -53,7 +53,7 @@ class GitTask: @staticmethod async def git_task_worker(): - print("[service.git] worker start") + print("[service.git] starting task worker") while True: task = await GitTask.queue.get() try: diff --git a/services/zine/shoutscache.py b/services/zine/shoutscache.py index 7bae2586..150c3651 100644 --- a/services/zine/shoutscache.py +++ b/services/zine/shoutscache.py @@ -151,7 +151,7 @@ class ShoutsCache: await ShoutsCache.prepare_recent_published() await ShoutsCache.prepare_recent_all() await ShoutsCache.prepare_recent_reacted() - print("[zine.cache] updated") + print("[zine.cache] periodical update") except Exception as err: print("[zine.cache] error: %s" % (err)) raise err diff --git a/services/zine/topics.py b/services/zine/topics.py index a23d557f..bda5e604 100644 --- a/services/zine/topics.py +++ b/services/zine/topics.py @@ -14,7 +14,7 @@ class TopicStorage: for topic in self.topics.values(): self.load_parents(topic) - print('[zine.topics] %d ' % len(self.topics.keys())) + print('[zine.topics] %d precached' % len(self.topics.keys())) @staticmethod def load_parents(topic):