diff --git a/migration/__init__.py b/migration/__init__.py index 74f26c8e..16387196 100644 --- a/migration/__init__.py +++ b/migration/__init__.py @@ -1,5 +1,6 @@ ''' cmd managed migration ''' import csv +import asyncio from datetime import datetime import json import subprocess @@ -74,7 +75,7 @@ def topics_handle(storage): return storage -def shouts_handle(storage, args): +async def shouts_handle(storage, args): ''' migrating content items one by one ''' counter = 0 discours_author = 0 @@ -89,7 +90,7 @@ def shouts_handle(storage, args): if '-' in args and slug not in args: continue # migrate - shout = migrateShout(entry, storage) + shout = await migrateShout(entry, storage) storage['shouts']['by_oid'][entry['_id']] = shout storage['shouts']['by_slug'][shout['slug']] = shout # shouts.topics @@ -123,13 +124,13 @@ def shouts_handle(storage, args): return storage -def comments_handle(storage): +async def comments_handle(storage): id_map = {} ignored_counter = 0 missed_shouts = {} for oldcomment in storage['reactions']['data']: if not oldcomment.get('deleted'): - reaction = migrateComment(oldcomment, storage) + reaction = await migrateComment(oldcomment, storage) if type(reaction) == str: missed_shouts[reaction] = oldcomment elif type(reaction) == Reaction: @@ -166,11 +167,11 @@ def export_one(slug, storage): export_slug(slug, storage) -def all_handle(storage, args): +async def all_handle(storage, args): print('[migration] handle everything') users_handle(storage) topics_handle(storage) - shouts_handle(storage, args) + await shouts_handle(storage, args) comments_handle(storage) # export_email_subscriptions() print('[migration] done!') @@ -213,26 +214,26 @@ def data_load(): content_data = [] try: users_data = json.loads(open('migration/data/users.json').read()) - print('[migration] ' + str(len(users_data)) + ' users ') + print('[migration.load] ' + str(len(users_data)) + ' users ') tags_data = json.loads(open('migration/data/tags.json').read()) storage['topics']['tags'] = tags_data - print('[migration] ' + str(len(tags_data)) + ' tags ') + print('[migration.load] ' + str(len(tags_data)) + ' tags ') cats_data = json.loads( open('migration/data/content_item_categories.json').read()) storage['topics']['cats'] = cats_data - print('[migration] ' + str(len(cats_data)) + ' cats ') + print('[migration.load] ' + str(len(cats_data)) + ' cats ') comments_data = json.loads(open('migration/data/comments.json').read()) storage['reactions']['data'] = comments_data - print('[migration] ' + str(len(comments_data)) + ' comments ') + print('[migration.load] ' + str(len(comments_data)) + ' comments ') content_data = json.loads(open('migration/data/content_items.json').read()) storage['shouts']['data'] = content_data - print('[migration] ' + str(len(content_data)) + ' content items ') + print('[migration.load] ' + str(len(content_data)) + ' content items ') # fill out storage for x in users_data: storage['users']['by_oid'][x['_id']] = x # storage['users']['by_slug'][x['slug']] = x # no user.slug yet - print('[migration] ' + str(len(storage['users'] + print('[migration.load] ' + str(len(storage['users'] ['by_oid'].keys())) + ' users by oid') for x in tags_data: storage['topics']['by_oid'][x['_id']] = x @@ -240,20 +241,20 @@ def data_load(): for x in cats_data: storage['topics']['by_oid'][x['_id']] = x storage['topics']['by_slug'][x['slug']] = x - print('[migration] ' + str(len(storage['topics'] + print('[migration.load] ' + str(len(storage['topics'] ['by_slug'].keys())) + ' topics by slug') for item in content_data: slug = get_shout_slug(item) storage['content_items']['by_slug'][slug] = item storage['content_items']['by_oid'][item['_id']] = item - print('[migration] ' + str(len(content_data)) + ' content items') + print('[migration.load] ' + str(len(content_data)) + ' content items') for x in comments_data: storage['reactions']['by_oid'][x['_id']] = x cid = x['contentItem'] storage['reactions']['by_content'][cid] = x ci = storage['content_items']['by_oid'].get(cid, {}) if 'slug' in ci: storage['reactions']['by_slug'][ci['slug']] = x - print('[migration] ' + str(len(storage['reactions'] + print('[migration.load] ' + str(len(storage['reactions'] ['by_content'].keys())) + ' with comments') except Exception as e: raise e storage['users']['data'] = users_data @@ -288,29 +289,25 @@ def create_pgdump(): ]) -def handle_auto(): +async def handle_auto(): print('[migration] no command given, auto mode') - mongo_download(os.getenv('MONGODB_URL')) + url = os.getenv('MONGODB_URL') + if url: mongo_download(url) bson_handle() - all_handle(data_load(), sys.argv) + await all_handle(data_load(), sys.argv) create_pgdump() -def migrate(): +async def main(): if len(sys.argv) > 1: cmd=sys.argv[1] if type(cmd) == str: print('[migration] command: ' + cmd) - if cmd == 'mongodb': - mongo_download(sys.argv[2]) - elif cmd == 'bson': - bson_handle() - else: - storage=data_load() - if cmd == '-': export_one(sys.argv[2], storage) - else: all_handle(storage, sys.argv) - elif len(sys.argv) == 1: - handle_auto() + await handle_auto() else: - print('[migration] usage: python migrate.py ') - print('[migration] commands: mongodb, bson, all, all mdx, - ') + print('[migration] usage: python server.py migrate') -if __name__ == '__main__': migrate() +def migrate(): + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) + +if __name__ == '__main__': + migrate() \ No newline at end of file diff --git a/migration/tables/comments.py b/migration/tables/comments.py index cbc47b3e..1354b7c5 100644 --- a/migration/tables/comments.py +++ b/migration/tables/comments.py @@ -8,7 +8,7 @@ from services.stat.reacted import ReactedStorage ts = datetime.now() -def migrate(entry, storage): +async def migrate(entry, storage): ''' { "_id": "hdtwS8fSyFLxXCgSC", @@ -72,7 +72,7 @@ def migrate(entry, storage): # creating reaction from old comment day = (reaction_dict.get('createdAt') or ts).replace(hour=0, minute=0, second=0, microsecond=0) reaction = Reaction.create(**reaction_dict) - ReactedStorage.increment(reaction) + await ReactedStorage.increment(reaction) reaction_dict['id'] = reaction.id for comment_rating_old in entry.get('ratings',[]): @@ -89,7 +89,7 @@ def migrate(entry, storage): try: # creating reaction from old rating rr = Reaction.create(**re_reaction_dict) - ReactedStorage.increment(rr) + await ReactedStorage.increment(rr) except Exception as e: print('[migration] comment rating error: %r' % re_reaction_dict) diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index 9471c51d..6aa88664 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -28,7 +28,7 @@ def get_shout_slug(entry): if slug: break return slug -def migrate(entry, storage): +async def migrate(entry, storage): # init, set title and layout r = { 'layout': type2layout[entry['type']], @@ -218,7 +218,7 @@ def migrate(entry, storage): else: day = (reaction_dict.get('createdAt') or ts).replace(hour=0, minute=0, second=0, microsecond=0) rea = Reaction.create(**reaction_dict) - ReactedStorage.increment(rea) + await ReactedStorage.increment(rea) # shout_dict['ratings'].append(reaction_dict) except: print('[migration] content_item.ratings error: \n%r' % content_rating) diff --git a/orm/shout.py b/orm/shout.py index b07922dd..e2e71f9b 100644 --- a/orm/shout.py +++ b/orm/shout.py @@ -67,6 +67,5 @@ class Shout(Base): "viewed": await ViewedStorage.get_shout(self.slug), "reacted": len(await ReactedStorage.get_shout(self.slug)), "commented": len(await ReactedStorage.get_comments(self.slug)), - "rating": await ReactedStorage.get_rating(self.slug), - "bookmarked": len(await ReactedStorage.get_bookmarked(self.slug)) + "rating": await ReactedStorage.get_rating(self.slug) } diff --git a/requirements.txt b/requirements.txt index db3565b9..b1ed70ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ transliterate requests bcrypt websockets +bson \ No newline at end of file diff --git a/resolvers/reactions.py b/resolvers/reactions.py index 5e3d045f..0550e8d9 100644 --- a/resolvers/reactions.py +++ b/resolvers/reactions.py @@ -1,4 +1,5 @@ from sqlalchemy import desc +from sqlalchemy.orm import joinedload, selectinload from orm.reaction import Reaction from base.orm import local_session from orm.shout import ShoutReactionsFollower @@ -112,31 +113,31 @@ async def get_shout_reactions(_, info, slug, page, size): filter(Reaction.shout == slug).\ limit(size).offset(offset).all() for r in reactions: - r.createdBy = await UserStorage.get_user(r.createdBy) + r.createdBy = await UserStorage.get_user(r.createdBy or 'discours') return reactions @query.field("reactionsAll") -def get_all_reactions(_, info, page=1, size=10): +async def get_all_reactions(_, info, page=1, size=10): offset = page * size reactions = [] with local_session() as session: - # raw sql: statement = text(open('queries/reactions-all.sql', 'r').read())) - statement = session.query(Reaction).\ + reactions = session.query(Reaction).\ filter(Reaction.deletedAt == None).\ order_by(desc("createdAt")).\ offset(offset).limit(size) - reactions = [] - for row in session.execute(statement): - reaction = row.Reaction - reactions.append(reaction) + for r in reactions: + r.createdBy = await UserStorage.get_user(r.createdBy or 'discours') + reactions = list(reactions) reactions.sort(key=lambda x: x.createdAt, reverse=True) return reactions @query.field("reactionsByAuthor") -def get_reactions_by_author(_, info, slug, page=1, size=50): +async def get_reactions_by_author(_, info, slug, page=1, size=50): offset = page * size reactions = [] with local_session() as session: - reactions = session.query(Reaction).filter(Reaction.createdBy == slug).limit(size).offset(offset).all() + reactions = session.query(Reaction).filter(Reaction.createdBy == slug).limit(size).offset(offset) + for r in reactions: + r.createdBy = await UserStorage.get_user(r.createdBy or 'discours') return reactions \ No newline at end of file diff --git a/schema.graphql b/schema.graphql index 6a4f4cbd..95876be5 100644 --- a/schema.graphql +++ b/schema.graphql @@ -414,7 +414,6 @@ type Stat { reacted: Int rating: Int commented: Int - bookmarked: Int } type Community { diff --git a/services/stat/reacted.py b/services/stat/reacted.py index d17e35e5..8c50e0aa 100644 --- a/services/stat/reacted.py +++ b/services/stat/reacted.py @@ -1,7 +1,7 @@ import asyncio from datetime import datetime from sqlalchemy.types import Enum -from sqlalchemy import Column, DateTime, ForeignKey +from sqlalchemy import Column, DateTime, ForeignKey, Boolean # from sqlalchemy.orm.attributes import flag_modified from sqlalchemy import Enum import enum @@ -37,6 +37,7 @@ def kind_to_rate(kind) -> int: ReactionKind.REJECT ]: return -1 else: return 0 + class ReactedByDay(Base): __tablename__ = "reacted_by_day" @@ -46,6 +47,8 @@ class ReactedByDay(Base): replyTo = Column(ForeignKey('reaction.id'), nullable=True) kind: int = Column(Enum(ReactionKind), nullable=False, comment="Reaction kind") day = Column(DateTime, primary_key=True, default=datetime.now) + comment = Column(Boolean, nullable=True) + class ReactedStorage: reacted = { @@ -63,34 +66,6 @@ class ReactedStorage: period = 30*60 # sec lock = asyncio.Lock() - @staticmethod - def init(session): - self = ReactedStorage - all_reactions = session.query(ReactedByDay).all() - print('[stat.reacted] %d reactions total' % len(all_reactions)) - for reaction in all_reactions: - shout = reaction.shout - topics = session.query(ShoutTopic.topic).where(ShoutTopic.shout == shout).all() - kind = reaction.kind - - self.reacted['shouts'][shout] = self.reacted['shouts'].get(shout, []) - self.reacted['shouts'][shout].append(reaction) - self.rating['shouts'][shout] = self.rating['shouts'].get(shout, 0) + kind_to_rate(kind) - - for t in topics: - self.reacted['topics'][t] = self.reacted['topics'].get(t, []) - self.reacted['topics'][t].append(reaction) - self.rating['topics'][t] = self.rating['topics'].get(t, 0) + kind_to_rate(kind) # rating - - if reaction.replyTo: - self.reacted['reactions'][reaction.replyTo] = self.reacted['reactions'].get(reaction.replyTo, []) - self.reacted['reactions'][reaction.replyTo].append(reaction) - self.rating['reactions'][reaction.replyTo] = self.rating['reactions'].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind) - ttt = self.reacted['topics'].values() - print('[stat.reacted] %d topics reacted' % len(ttt)) - print('[stat.reacted] %d shouts reacted' % len(self.reacted['shouts'])) - print('[stat.reacted] %d reactions reacted' % len(self.reacted['reactions'])) - @staticmethod async def get_shout(shout_slug): self = ReactedStorage @@ -103,6 +78,24 @@ class ReactedStorage: async with self.lock: return self.reacted['topics'].get(topic_slug, []) + @staticmethod + async def get_comments(shout_slug): + self = ReactedStorage + async with self.lock: + return list(filter(lambda r: r.comment, self.reacted['shouts'].get(shout_slug, []))) + + @staticmethod + async def get_topic_comments(topic_slug): + self = ReactedStorage + async with self.lock: + return list(filter(lambda r: r.comment, self.reacted['topics'].get(topic_slug, []))) + + @staticmethod + async def get_reaction_comments(reaction_id): + self = ReactedStorage + async with self.lock: + return list(filter(lambda r: r.comment, self.reacted['reactions'].get(reaction_id))) + @staticmethod async def get_reaction(reaction_id): self = ReactedStorage @@ -137,17 +130,52 @@ class ReactedStorage: return rating @staticmethod - async def increment(shout_slug, kind, reply_id = None): + async def increment(reaction): self = ReactedStorage - reaction: ReactedByDay = None async with self.lock: with local_session() as session: - reaction = ReactedByDay.create(shout=shout_slug, kind=kind, reply=reply_id) - self.reacted['shouts'][shout_slug] = self.reacted['shouts'].get(shout_slug, []) - self.reacted['shouts'][shout_slug].append(reaction) - if reply_id: - self.reacted['reaction'][reply_id] = self.reacted['reactions'].get(shout_slug, []) - self.reacted['reaction'][reply_id].append(reaction) - self.rating['reactions'][reply_id] = self.rating['reactions'].get(reply_id, 0) + kind_to_rate(kind) + r = { + "day": datetime.now().replace(hour=0, minute=0, second=0, microsecond=0), + "reaction": reaction.id, + "kind": reaction.kind, + "shout": reaction.shout + } + if reaction.replyTo: r['replyTo'] = reaction.replyTo + if reaction.body: r['comment'] = True + reaction = ReactedByDay.create(**r) + self.reacted['shouts'][reaction.shout] = self.reacted['shouts'].get(reaction.shout, []) + self.reacted['shouts'][reaction.shout].append(reaction) + if reaction.replyTo: + self.reacted['reaction'][reaction.replyTo] = self.reacted['reactions'].get(reaction.shout, []) + self.reacted['reaction'][reaction.replyTo].append(reaction) + self.rating['reactions'][reaction.replyTo] = self.rating['reactions'].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind) else: - self.rating['shouts'][shout_slug] = self.rating['shouts'].get(shout_slug, 0) + kind_to_rate(kind) \ No newline at end of file + self.rating['shouts'][reaction.replyTo] = self.rating['shouts'].get(reaction.shout, 0) + kind_to_rate(reaction.kind) + + @staticmethod + def init(session): + self = ReactedStorage + all_reactions = session.query(ReactedByDay).all() + print('[stat.reacted] %d reactions total' % len(all_reactions)) + for reaction in all_reactions: + shout = reaction.shout + topics = session.query(ShoutTopic.topic).where(ShoutTopic.shout == shout).all() + kind = reaction.kind + + self.reacted['shouts'][shout] = self.reacted['shouts'].get(shout, []) + self.reacted['shouts'][shout].append(reaction) + self.rating['shouts'][shout] = self.rating['shouts'].get(shout, 0) + kind_to_rate(kind) + + for t in topics: + self.reacted['topics'][t] = self.reacted['topics'].get(t, []) + self.reacted['topics'][t].append(reaction) + self.rating['topics'][t] = self.rating['topics'].get(t, 0) + kind_to_rate(kind) # rating + + if reaction.replyTo: + self.reacted['reactions'][reaction.replyTo] = self.reacted['reactions'].get(reaction.replyTo, []) + self.reacted['reactions'][reaction.replyTo].append(reaction) + self.rating['reactions'][reaction.replyTo] = self.rating['reactions'].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind) + ttt = self.reacted['topics'].values() + print('[stat.reacted] %d topics reacted' % len(ttt)) + print('[stat.reacted] %d shouts reacted' % len(self.reacted['shouts'])) + print('[stat.reacted] %d reactions reacted' % len(self.reacted['reactions'])) \ No newline at end of file diff --git a/services/stat/topicstat.py b/services/stat/topicstat.py index 544c63c4..d2201761 100644 --- a/services/stat/topicstat.py +++ b/services/stat/topicstat.py @@ -67,7 +67,6 @@ class TopicStat: "viewed": await ViewedStorage.get_topic(topic), "reacted" : len(await ReactedStorage.get_topic(topic)), "commented": len(await ReactedStorage.get_topic_comments(topic)), - "bookmarked": len(await ReactedStorage.get_topic_bookmarked(topic)), "rating" : await ReactedStorage.get_topic_rating(topic), }