From a985e800a220323200dbd5a8506ca2ee85a0a9be Mon Sep 17 00:00:00 2001 From: knst-kotov Date: Mon, 13 Dec 2021 10:50:33 +0300 Subject: [PATCH] remove shout id --- migrate.py | 11 ++---- migration/tables/comments.py | 16 ++++++--- migration/tables/content_items.py | 9 +++-- orm/comment.py | 2 +- orm/shout.py | 49 +++++++++++++------------- resolvers/zine.py | 58 ++++++++++++++++--------------- schema.graphql | 14 ++++---- 7 files changed, 79 insertions(+), 80 deletions(-) diff --git a/migrate.py b/migrate.py index 485fbf95..e5e47a2a 100644 --- a/migrate.py +++ b/migrate.py @@ -123,13 +123,6 @@ def shouts(content_data, shouts_by_slug, shouts_by_oid): try: limit = int(sys.argv[2]) if len(sys.argv) > 2 else len(content_data) except ValueError: limit = len(content_data) - if not topics_by_cat: - with local_session() as session: - topics = session.query(Topic).all() - print("loaded %s topics" % len(topics)) - for topic in topics: - topics_by_cat[topic.cat_id] = topic - for entry in content_data[:limit]: try: shout = migrateShout(entry, users_by_oid, topics_by_cat) @@ -202,7 +195,9 @@ def export_slug(slug, export_articles, export_authors, content_dict): def comments(comments_data): id_map = {} for comment in comments_data: - comment = migrateComment(comment) + comment = migrateComment(comment, shouts_by_oid) + if not comment: + continue id = comment.get('id') old_id = comment.get('old_id') id_map[old_id] = id diff --git a/migration/tables/comments.py b/migration/tables/comments.py index d38ed714..4ae00422 100644 --- a/migration/tables/comments.py +++ b/migration/tables/comments.py @@ -6,7 +6,7 @@ from orm import Shout, Comment, CommentRating, User from orm.base import local_session from migration.html2text import html2text -def migrate(entry): +def migrate(entry, shouts_by_oid): ''' { "_id": "hdtwS8fSyFLxXCgSC", @@ -40,15 +40,19 @@ def migrate(entry): views: Int } ''' + + shout_old_id = entry['contentItem'] + if not shout_old_id in shouts_by_oid: + return + shout = shouts_by_oid[shout_old_id] + with local_session() as session: - shout = session.query(Shout).filter(Shout.old_id == entry['contentItem']).first() - if not shout: shout = session.query(Shout).first() author = session.query(User).filter(User.old_id == entry['createdBy']).first() comment_dict = { 'author': author.id if author else 0, 'createdAt': date_parse(entry['createdAt']), 'body': html2text(entry['body']), - 'shout': shout.id + 'shout': shout["slug"] } if entry.get('deleted'): comment_dict['deletedAt'] = date_parse(entry['updatedAt']) @@ -86,7 +90,9 @@ def migrate_2stage(entry, id_map): if not old_reply_to: return old_id = entry['_id'] - id = id_map.get(old_id) + if not old_id in id_map: + return + id = id_map[old_id] with local_session() as session: comment = session.query(Comment).filter(Comment.id == id).first() reply_to = id_map.get(old_reply_to) diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index 11090040..d9ff1155 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -133,7 +133,6 @@ def migrate(entry, users_by_oid, topics_by_oid): body_html = str(BeautifulSoup(body_orig, features="html.parser")) r['body'] = body_html # html2text(body_html) body = r.get('body', '') - r['old_id'] = entry.get('_id') # get author data userdata = {} @@ -212,8 +211,8 @@ def migrate(entry, users_by_oid, topics_by_oid): if rater: shout_rating_dict = { 'value': shout_rating_old['value'], - 'rater_id': rater.id, - 'shout_id': s.id + 'rater': rater.id, + 'shout': s.slug } cts = shout_rating_old.get('createdAt') if cts: shout_rating_dict['rater_id'] = date_parse(cts) @@ -221,14 +220,14 @@ def migrate(entry, users_by_oid, topics_by_oid): except sqlalchemy.exc.IntegrityError: pass shout_dict['ratings'].append(shout_rating_dict) # shout topics - shout_dict['id'] = s.id shout_dict['topics'] = [] for topic in r['topics']: - ShoutTopic.create(**{ 'shout': s.id, 'topic': topic['slug'] }) + ShoutTopic.create(**{ 'shout': s.slug, 'topic': topic['slug'] }) shout_dict['topics'].append(topic['slug']) except Exception as e: raise e except Exception as e: if not shout_dict['body']: r['body'] = 'body moved' raise e + shout_dict['old_id'] = entry.get('_id') return shout_dict # for json diff --git a/orm/comment.py b/orm/comment.py index f497e23b..56a7b4a1 100644 --- a/orm/comment.py +++ b/orm/comment.py @@ -25,7 +25,7 @@ class Comment(Base): updatedBy = Column(ForeignKey("user.id"), nullable=True, comment="Last Editor") deletedAt = Column(DateTime, nullable=True, comment="Deleted at") deletedBy = Column(ForeignKey("user.id"), nullable=True, comment="Deleted by") - shout: int = Column(ForeignKey("shout.id"), nullable=False, comment="Shout ID") + shout: int = Column(ForeignKey("shout.slug"), nullable=False) replyTo: int = Column(ForeignKey("comment.id"), nullable=True, comment="comment ID") ratings = relationship(CommentRating, foreign_keys=CommentRating.comment_id) diff --git a/orm/shout.py b/orm/shout.py index 7839f45f..82d1b27e 100644 --- a/orm/shout.py +++ b/orm/shout.py @@ -15,29 +15,29 @@ class ShoutAuthor(Base): __tablename__ = "shout_author" id = None - shout = Column(ForeignKey('shout.id'), primary_key = True) + shout = Column(ForeignKey('shout.slug'), primary_key = True) user = Column(ForeignKey('user.id'), primary_key = True) class ShoutViewer(Base): __tablename__ = "shout_viewer" id = None - shout = Column(ForeignKey('shout.id'), primary_key = True) + shout = Column(ForeignKey('shout.slug'), primary_key = True) user = Column(ForeignKey('user.id'), primary_key = True) class ShoutTopic(Base): __tablename__ = 'shout_topic' id = None - shout = Column(ForeignKey('shout.id'), primary_key = True) + shout = Column(ForeignKey('shout.slug'), primary_key = True) topic = Column(ForeignKey('topic.slug'), primary_key = True) class ShoutRating(Base): __tablename__ = "shout_rating" id = None - rater_id = Column(ForeignKey('user.id'), primary_key = True) - shout_id = Column(ForeignKey('shout.id'), primary_key = True) + rater = Column(ForeignKey('user.id'), primary_key = True) + shout = Column(ForeignKey('shout.slug'), primary_key = True) ts = Column(DateTime, nullable=False, default = datetime.now, comment="Timestamp") value = Column(Integer) @@ -52,16 +52,16 @@ class ShoutRatingStorage: ShoutRatingStorage.ratings = session.query(ShoutRating).all() @staticmethod - async def get_rating(shout_id): + async def get_rating(shout_slug): async with ShoutRatingStorage.lock: - shout_ratings = list(filter(lambda x: x.shout_id == shout_id, ShoutRatingStorage.ratings)) + shout_ratings = list(filter(lambda x: x.shout == shout_slug, ShoutRatingStorage.ratings)) return reduce((lambda x, y: x + y.value), shout_ratings, 0) @staticmethod async def update_rating(new_rating): async with ShoutRatingStorage.lock: rating = next((x for x in ShoutRatingStorage.ratings \ - if x.rater_id == new_rating.rater_id and x.shout_id == new_rating.shout_id), None) + if x.rater == new_rating.rater and x.shout == new_rating.shout), None) if rating: rating.value = new_rating.value rating.ts = new_rating.ts @@ -73,7 +73,7 @@ class ShoutViewByDay(Base): __tablename__ = "shout_view_by_day" id = None - shout_id = Column(ForeignKey('shout.id'), primary_key = True) + shout = Column(ForeignKey('shout.slug'), primary_key = True) day = Column(DateTime, primary_key = True, default = datetime.now) value = Column(Integer) @@ -91,28 +91,28 @@ class ShoutViewStorage: self = ShoutViewStorage self.views = session.query(ShoutViewByDay).all() for view in self.views: - shout_id = view.shout_id - if not shout_id in self.this_day_views: - self.this_day_views[shout_id] = view - this_day_view = self.this_day_views[shout_id] + shout_slug = view.shout + if not shout_slug in self.this_day_views: + self.this_day_views[shout_slug] = view + this_day_view = self.this_day_views[shout_slug] if this_day_view.day < view.day: - self.this_day_views[shout_id] = view + self.this_day_views[shout_slug] = view @staticmethod - async def get_view(shout_id): + async def get_view(shout_slug): async with ShoutViewStorage.lock: - shout_views = list(filter(lambda x: x.shout_id == shout_id, ShoutViewStorage.views)) + shout_views = list(filter(lambda x: x.shout == shout_slug, ShoutViewStorage.views)) return reduce((lambda x, y: x + y.value), shout_views, 0) @staticmethod - async def inc_view(shout_id): + async def inc_view(shout_slug): self = ShoutViewStorage async with ShoutViewStorage.lock: - this_day_view = self.this_day_views.get(shout_id) + this_day_view = self.this_day_views.get(shout_slug) day_start = datetime.now().replace(hour = 0, minute = 0, second = 0) if not this_day_view or this_day_view.day < day_start: - this_day_view = ShoutViewByDay.create(shout_id = shout_id, value = 1) - self.this_day_views[shout_id] = this_day_view + this_day_view = ShoutViewByDay.create(shout = shout_slug, value = 1) + self.this_day_views[shout_slug] = this_day_view self.views.append(this_day_view) else: this_day_view.value = this_day_view.value + 1 @@ -144,15 +144,15 @@ class ShoutViewStorage: class Shout(Base): __tablename__ = 'shout' - # NOTE: automatic ID here + id = None - slug: str = Column(String, nullable=False, unique=True) + slug: str = Column(String, primary_key=True) community: int = Column(Integer, ForeignKey("community.id"), nullable=True, comment="Community") body: str = Column(String, nullable=False, comment="Body") createdAt: str = Column(DateTime, nullable=False, default = datetime.now, comment="Created at") updatedAt: str = Column(DateTime, nullable=True, comment="Updated at") - replyTo: int = Column(ForeignKey("shout.id"), nullable=True) - versionOf: int = Column(ForeignKey("shout.id"), nullable=True) + replyTo: int = Column(ForeignKey("shout.slug"), nullable=True) + versionOf: int = Column(ForeignKey("shout.slug"), nullable=True) tags: str = Column(String, nullable=True) publishedBy: bool = Column(ForeignKey("user.id"), nullable=True) publishedAt: str = Column(DateTime, nullable=True) @@ -164,4 +164,3 @@ class Shout(Base): authors = relationship(lambda: User, secondary=ShoutAuthor.__tablename__) # NOTE: multiple authors topics = relationship(lambda: Topic, secondary=ShoutTopic.__tablename__) visibleFor = relationship(lambda: User, secondary=ShoutViewer.__tablename__) - old_id: str = Column(String, nullable = True) diff --git a/resolvers/zine.py b/resolvers/zine.py index ba571834..6f597b39 100644 --- a/resolvers/zine.py +++ b/resolvers/zine.py @@ -87,8 +87,8 @@ class ShoutsCache: shouts = [] for row in session.execute(stmt): shout = row.Shout - shout.rating = await ShoutRatingStorage.get_rating(shout.id) - shout.views = await ShoutViewStorage.get_view(shout.id) + shout.rating = await ShoutRatingStorage.get_rating(shout.slug) + shout.views = await ShoutViewStorage.get_view(shout.slug) shouts.append(shout) async with ShoutsCache.lock: ShoutsCache.recent_shouts = shouts @@ -101,14 +101,14 @@ class ShoutsCache: options(selectinload(Shout.authors), selectinload(Shout.topics)).\ join(ShoutRating).\ where(Shout.publishedAt != None).\ - group_by(Shout.id).\ + group_by(Shout.slug).\ order_by(desc("rating")).\ limit(ShoutsCache.limit) shouts = [] for row in session.execute(stmt): shout = row.Shout shout.rating = row.rating - shout.views = await ShoutViewStorage.get_view(shout.id) + shout.views = await ShoutViewStorage.get_view(shout.slug) shouts.append(shout) async with ShoutsCache.lock: ShoutsCache.top_overall = shouts @@ -121,14 +121,14 @@ class ShoutsCache: options(selectinload(Shout.authors), selectinload(Shout.topics)).\ join(ShoutRating).\ where(and_(Shout.createdAt > month_ago, Shout.publishedAt != None)).\ - group_by(Shout.id).\ + group_by(Shout.slug).\ order_by(desc("rating")).\ limit(ShoutsCache.limit) shouts = [] for row in session.execute(stmt): shout = row.Shout shout.rating = row.rating - shout.views = await ShoutViewStorage.get_view(shout.id) + shout.views = await ShoutViewStorage.get_view(shout.slug) shouts.append(shout) async with ShoutsCache.lock: ShoutsCache.top_month = shouts @@ -141,13 +141,13 @@ class ShoutsCache: options(selectinload(Shout.authors), selectinload(Shout.topics)).\ join(ShoutViewByDay).\ where(and_(ShoutViewByDay.day > month_ago, Shout.publishedAt != None)).\ - group_by(Shout.id).\ + group_by(Shout.slug).\ order_by(desc("views")).\ limit(ShoutsCache.limit) shouts = [] for row in session.execute(stmt): shout = row.Shout - shout.rating = await ShoutRatingStorage.get_rating(shout.id) + shout.rating = await ShoutRatingStorage.get_rating(shout.slug) shout.views = row.views shouts.append(shout) async with ShoutsCache.lock: @@ -157,13 +157,13 @@ class ShoutsCache: async def prepare_top_authors(): month_ago = datetime.now() - timedelta(days = 30) with local_session() as session: - shout_with_view = select(Shout.id, func.sum(ShoutViewByDay.value).label("view")).\ + shout_with_view = select(Shout.slug, func.sum(ShoutViewByDay.value).label("view")).\ join(ShoutViewByDay).\ where(and_(ShoutViewByDay.day > month_ago, Shout.publishedAt != None)).\ - group_by(Shout.id).\ + group_by(Shout.slug).\ order_by(desc("view")).cte() stmt = select(ShoutAuthor.user, func.sum(shout_with_view.c.view).label("view")).\ - join(shout_with_view, ShoutAuthor.shout == shout_with_view.c.id).\ + join(shout_with_view, ShoutAuthor.shout == shout_with_view.c.slug).\ group_by(ShoutAuthor.user).\ order_by(desc("view")).\ limit(ShoutsCache.limit) @@ -254,12 +254,12 @@ async def create_shout(_, info, input): new_shout = Shout.create(**input) ShoutAuthor.create( - shout = new_shout.id, + shout = new_shout.slug, user = user_id) for slug in topic_slugs: topic = ShoutTopic.create( - shout = new_shout.id, + shout = new_shout.slug, topic = slug) new_shout.topic_slugs = topic_slugs @@ -278,13 +278,15 @@ async def create_shout(_, info, input): @mutation.field("updateShout") @login_required -async def update_shout(_, info, id, input): +async def update_shout(_, info, input): auth = info.context["request"].auth user_id = auth.user_id + slug = input["slug"] + session = local_session() user = session.query(User).filter(User.id == user_id).first() - shout = session.query(Shout).filter(Shout.id == id).first() + shout = session.query(Shout).filter(Shout.slug == slug).first() if not shout: return { @@ -305,16 +307,16 @@ async def update_shout(_, info, id, input): session.commit() session.close() - for topic in input.get("topic_slugs"): + for topic in input.get("topic_slugs", []): ShoutTopic.create( - shout = shout.id, + shout = slug, topic = topic) task = GitTask( input, user.username, user.email, - "update shout %s" % (shout.slug) + "update shout %s" % (slug) ) return { @@ -323,21 +325,21 @@ async def update_shout(_, info, id, input): @mutation.field("rateShout") @login_required -async def rate_shout(_, info, shout_id, value): +async def rate_shout(_, info, slug, value): auth = info.context["request"].auth user_id = auth.user_id with local_session() as session: rating = session.query(ShoutRating).\ - filter(and_(ShoutRating.rater_id == user_id, ShoutRating.shout_id == shout_id)).first() + filter(and_(ShoutRating.rater == user_id, ShoutRating.shout == slug)).first() if rating: rating.value = value; rating.ts = datetime.now() session.commit() else: rating = ShoutRating.create( - rater_id = user_id, - shout_id = shout_id, + rater = user_id, + shout = slug, value = value ) @@ -346,8 +348,8 @@ async def rate_shout(_, info, shout_id, value): return {"error" : ""} @mutation.field("viewShout") -async def view_shout(_, info, shout_id): - await ShoutViewStorage.inc_view(shout_id) +async def view_shout(_, info, slug): + await ShoutViewStorage.inc_view(slug) return {"error" : ""} @query.field("getShoutBySlug") @@ -360,16 +362,16 @@ async def get_shout_by_slug(_, info, slug): shout = session.query(Shout).\ options(select_options).\ filter(Shout.slug == slug).first() - shout.rating = await ShoutRatingStorage.get_rating(shout.id) - shout.views = await ShoutViewStorage.get_view(shout.id) + shout.rating = await ShoutRatingStorage.get_rating(slug) + shout.views = await ShoutViewStorage.get_view(slug) return shout @query.field("getShoutComments") -async def get_shout_comments(_, info, shout_id): +async def get_shout_comments(_, info, slug): with local_session() as session: comments = session.query(Comment).\ options(selectinload(Comment.ratings)).\ - filter(Comment.shout == shout_id).\ + filter(Comment.shout == slug).\ group_by(Comment.id).all() for comment in comments: comment.author = await UserStorage.get_user(comment.author) diff --git a/schema.graphql b/schema.graphql index b48a18dd..e1a1a149 100644 --- a/schema.graphql +++ b/schema.graphql @@ -109,10 +109,10 @@ type Mutation { # shout createShout(input: ShoutInput!): ShoutResult! - updateShout(id: Int!, input: ShoutInput!): ShoutResult! - deleteShout(id: Int!): Result! - rateShout(shout_id: Int!, value: Int!): Result! - viewShout(shout_id: Int!): Result! + updateShout(input: ShoutInput!): ShoutResult! + deleteShout(slug: String!): Result! + rateShout(slug: String!, value: Int!): Result! + viewShout(slug: String!): Result! # user profile # rateUser(value: Int!): Result! @@ -125,7 +125,7 @@ type Mutation { topicSubscribe(slug: String!): Result! topicUnsubscribe(slug: String!): Result! - createComment(body: String!, shout: Int!, replyTo: Int): CommentResult! + createComment(body: String!, shout: String!, replyTo: Int): CommentResult! updateComment(id: Int!, body: String!): CommentResult! deleteComment(id: Int!): Result! rateComment(id: Int!, value: Int!): Result! @@ -157,7 +157,7 @@ type Query { shoutsByTopic(topic: String!, limit: Int!): [Shout]! shoutsByAuthor(author: String!, limit: Int!): [Shout]! shoutsByCommunity(community: String!, limit: Int!): [Shout]! - getShoutComments(shout_id: Int!): [Comment]! + getShoutComments(slug: String!): [Comment]! # mainpage topViewed(limit: Int): [Shout]! @@ -299,7 +299,6 @@ type CommentRating { # is publication type Shout { - id: Int! slug: String! body: String! createdAt: DateTime! @@ -323,7 +322,6 @@ type Shout { deletedBy: Int publishedBy: Int # if there is no published field - it is not published publishedAt: DateTime - old_id: String } type Community {