From f9bc1d67aec780a3e14e1bd54f8d94bbbdfcb3fb Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Wed, 13 Dec 2023 23:56:01 +0100 Subject: [PATCH 1/6] random top articles query (#109) * loadRandomTopShouts * minor fixes --------- Co-authored-by: Igor Lobanov --- resolvers/zine/load.py | 150 ++++++++++++++++++++++++++--------------- schema.graphql | 12 +++- 2 files changed, 106 insertions(+), 56 deletions(-) diff --git a/resolvers/zine/load.py b/resolvers/zine/load.py index 4db893a9..db1ee0e2 100644 --- a/resolvers/zine/load.py +++ b/resolvers/zine/load.py @@ -1,5 +1,5 @@ import json -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from sqlalchemy.orm import aliased, joinedload from sqlalchemy.sql.expression import and_, asc, case, desc, func, nulls_last, select @@ -15,6 +15,41 @@ from orm.shout import Shout, ShoutAuthor, ShoutTopic from orm.user import AuthorFollower +def get_shouts_from_query(q): + shouts = [] + with local_session() as session: + for [shout, reacted_stat, commented_stat, rating_stat, last_comment] in session.execute( + q + ).unique(): + shouts.append(shout) + shout.stat = { + "viewed": shout.views, + "reacted": reacted_stat, + "commented": commented_stat, + "rating": rating_stat, + } + + return shouts + + +def get_rating_func(aliased_reaction): + return func.sum( + case( + # do not count comments' reactions + (aliased_reaction.replyTo.is_not(None), 0), + (aliased_reaction.kind == ReactionKind.AGREE, 1), + (aliased_reaction.kind == ReactionKind.DISAGREE, -1), + (aliased_reaction.kind == ReactionKind.PROOF, 1), + (aliased_reaction.kind == ReactionKind.DISPROOF, -1), + (aliased_reaction.kind == ReactionKind.ACCEPT, 1), + (aliased_reaction.kind == ReactionKind.REJECT, -1), + (aliased_reaction.kind == ReactionKind.LIKE, 1), + (aliased_reaction.kind == ReactionKind.DISLIKE, -1), + else_=0, + ) + ) + + def add_stat_columns(q): aliased_reaction = aliased(Reaction) @@ -23,21 +58,7 @@ def add_stat_columns(q): func.sum(case((aliased_reaction.kind == ReactionKind.COMMENT, 1), else_=0)).label( "commented_stat" ), - func.sum( - case( - # do not count comments' reactions - (aliased_reaction.replyTo.is_not(None), 0), - (aliased_reaction.kind == ReactionKind.AGREE, 1), - (aliased_reaction.kind == ReactionKind.DISAGREE, -1), - (aliased_reaction.kind == ReactionKind.PROOF, 1), - (aliased_reaction.kind == ReactionKind.DISPROOF, -1), - (aliased_reaction.kind == ReactionKind.ACCEPT, 1), - (aliased_reaction.kind == ReactionKind.REJECT, -1), - (aliased_reaction.kind == ReactionKind.LIKE, 1), - (aliased_reaction.kind == ReactionKind.DISLIKE, -1), - else_=0, - ) - ).label("rating_stat"), + get_rating_func(aliased_reaction).label("rating_stat"), func.max( case( (aliased_reaction.kind != ReactionKind.COMMENT, None), @@ -67,13 +88,14 @@ def apply_filters(q, filters, user_id=None): # noqa: C901 q = q.filter(Shout.authors.any(slug=filters.get("author"))) if filters.get("topic"): q = q.filter(Shout.topics.any(slug=filters.get("topic"))) - if filters.get("title"): - q = q.filter(Shout.title.ilike(f'%{filters.get("title")}%')) - if filters.get("body"): - q = q.filter(Shout.body.ilike(f'%{filters.get("body")}%s')) - if filters.get("days"): - before = datetime.now(tz=timezone.utc) - timedelta(days=int(filters.get("days")) or 30) - q = q.filter(Shout.createdAt > before) + if filters.get("fromDate"): + # fromDate: '2022-12-31 + date_from = datetime.strptime(filters.get("fromDate"), "%Y-%m-%d") + q = q.filter(Shout.createdAt >= date_from) + if filters.get("toDate"): + # toDate: '2023-12-31' + date_to = datetime.strptime(filters.get("toDate"), "%Y-%m-%d") + q = q.filter(Shout.createdAt < (date_to + timedelta(days=1))) return q @@ -136,7 +158,8 @@ async def load_shouts_by(_, info, options): topic: 'culture', title: 'something', body: 'something else', - days: 30 + fromDate: '2022-12-31', + toDate: '2023-12-31' } offset: 0 limit: 50 @@ -169,23 +192,57 @@ async def load_shouts_by(_, info, options): q = q.group_by(Shout.id).order_by(nulls_last(query_order_by)).limit(limit).offset(offset) - shouts = [] - with local_session() as session: - shouts_map = {} + return get_shouts_from_query(q) - for [shout, reacted_stat, commented_stat, rating_stat, last_comment] in session.execute( - q - ).unique(): - shouts.append(shout) - shout.stat = { - "viewed": shout.views, - "reacted": reacted_stat, - "commented": commented_stat, - "rating": rating_stat, - } - shouts_map[shout.id] = shout - return shouts +@query.field("loadRandomTopShouts") +async def load_random_top_shouts(_, info, params): + """ + :param params: { + filters: { + layout: 'music', + excludeLayout: 'article', + fromDate: '2022-12-31' + toDate: '2023-12-31' + } + fromRandomCount: 100, + limit: 50 + } + :return: Shout[] + """ + + aliased_reaction = aliased(Reaction) + + subquery = ( + select(Shout.id) + .outerjoin(aliased_reaction) + .where(and_(Shout.deletedAt.is_(None), Shout.layout.is_not(None))) + ) + + subquery = apply_filters(subquery, params.get("filters", {})) + subquery = subquery.group_by(Shout.id).order_by(desc(get_rating_func(aliased_reaction))) + + from_random_count = params.get("fromRandomCount") + if from_random_count: + subquery = subquery.limit(from_random_count) + + q = ( + select(Shout) + .options( + joinedload(Shout.authors), + joinedload(Shout.topics), + ) + .where(Shout.id.in_(subquery)) + ) + + q = add_stat_columns(q) + + limit = params.get("limit", 10) + q = q.group_by(Shout.id).order_by(func.random()).limit(limit) + + # print(q.compile(compile_kwargs={"literal_binds": True})) + + return get_shouts_from_query(q) @query.field("loadDrafts") @@ -256,17 +313,4 @@ async def get_my_feed(_, info, options): # print(q.compile(compile_kwargs={"literal_binds": True})) - shouts = [] - with local_session() as session: - for [shout, reacted_stat, commented_stat, rating_stat, last_comment] in session.execute( - q - ).unique(): - shouts.append(shout) - shout.stat = { - "viewed": shout.views, - "reacted": reacted_stat, - "commented": commented_stat, - "rating": rating_stat, - } - - return shouts + return get_shouts_from_query(q) diff --git a/schema.graphql b/schema.graphql index 79b26c0b..91aecb55 100644 --- a/schema.graphql +++ b/schema.graphql @@ -212,14 +212,13 @@ input AuthorsBy { } input LoadShoutsFilters { - title: String - body: String topic: String author: String layout: String excludeLayout: String visibility: String - days: Int + fromDate: String + toDate: String reacted: Boolean } @@ -232,6 +231,12 @@ input LoadShoutsOptions { order_by_desc: Boolean } +input LoadRandomTopShoutsParams { + filters: LoadShoutsFilters + limit: Int! + fromRandomCount: Int +} + input ReactionBy { shout: String # slug shouts: [String] @@ -276,6 +281,7 @@ type Query { loadAuthorsBy(by: AuthorsBy, limit: Int, offset: Int): [Author]! loadShout(slug: String, shout_id: Int): Shout loadShouts(options: LoadShoutsOptions): [Shout]! + loadRandomTopShouts(params: LoadRandomTopShoutsParams): [Shout]! loadDrafts: [Shout]! loadReactionsBy(by: ReactionBy!, limit: Int, offset: Int): [Reaction]! userFollowers(slug: String!): [Author]! From f5a3e273a61d98e7bcf4e335257b62e60def4273 Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Thu, 14 Dec 2023 19:40:12 +0100 Subject: [PATCH 2/6] unrated shouts query (#110) Co-authored-by: Igor Lobanov --- resolvers/zine/load.py | 28 ++++++++++++++++++++++++++++ schema.graphql | 1 + 2 files changed, 29 insertions(+) diff --git a/resolvers/zine/load.py b/resolvers/zine/load.py index db1ee0e2..aecda7e1 100644 --- a/resolvers/zine/load.py +++ b/resolvers/zine/load.py @@ -245,6 +245,34 @@ async def load_random_top_shouts(_, info, params): return get_shouts_from_query(q) +@query.field("loadUnratedShouts") +async def load_unrated_shouts(_, info, limit): + q = ( + select(Shout) + .options( + joinedload(Shout.authors), + joinedload(Shout.topics), + ) + .outerjoin( + Reaction, + and_( + Reaction.shout == Shout.id, + Reaction.replyTo.is_(None), + Reaction.kind.in_([ReactionKind.LIKE, ReactionKind.DISLIKE]), + ), + ) + .where(and_(Shout.deletedAt.is_(None), Shout.layout.is_not(None), Reaction.id.is_(None))) + ) + + q = add_stat_columns(q) + + q = q.group_by(Shout.id).order_by(desc(Shout.createdAt)).limit(limit) + + # print(q.compile(compile_kwargs={"literal_binds": True})) + + return get_shouts_from_query(q) + + @query.field("loadDrafts") @login_required async def get_drafts(_, info): diff --git a/schema.graphql b/schema.graphql index 91aecb55..56517191 100644 --- a/schema.graphql +++ b/schema.graphql @@ -282,6 +282,7 @@ type Query { loadShout(slug: String, shout_id: Int): Shout loadShouts(options: LoadShoutsOptions): [Shout]! loadRandomTopShouts(params: LoadRandomTopShoutsParams): [Shout]! + loadUnratedShouts(limit: Int!): [Shout]! loadDrafts: [Shout]! loadReactionsBy(by: ReactionBy!, limit: Int, offset: Int): [Reaction]! userFollowers(slug: String!): [Author]! From e23e3791020cc4ac9cef958a1c6f98eff81c8739 Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Sat, 16 Dec 2023 14:47:58 +0100 Subject: [PATCH 3/6] unrated shouts query update (#111) Co-authored-by: Igor Lobanov --- resolvers/zine/load.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/resolvers/zine/load.py b/resolvers/zine/load.py index aecda7e1..bda57f6d 100644 --- a/resolvers/zine/load.py +++ b/resolvers/zine/load.py @@ -2,7 +2,16 @@ import json from datetime import datetime, timedelta from sqlalchemy.orm import aliased, joinedload -from sqlalchemy.sql.expression import and_, asc, case, desc, func, nulls_last, select +from sqlalchemy.sql.expression import ( + and_, + asc, + case, + desc, + distinct, + func, + nulls_last, + select, +) from auth.authenticate import login_required from auth.credentials import AuthCredentials @@ -247,6 +256,8 @@ async def load_random_top_shouts(_, info, params): @query.field("loadUnratedShouts") async def load_unrated_shouts(_, info, limit): + auth: AuthCredentials = info.context["request"].auth + q = ( select(Shout) .options( @@ -261,12 +272,25 @@ async def load_unrated_shouts(_, info, limit): Reaction.kind.in_([ReactionKind.LIKE, ReactionKind.DISLIKE]), ), ) - .where(and_(Shout.deletedAt.is_(None), Shout.layout.is_not(None), Reaction.id.is_(None))) + .where( + and_( + Shout.deletedAt.is_(None), + Shout.layout.is_not(None), + Shout.createdAt >= (datetime.now() - timedelta(days=14)).date(), + ) + ) ) + user_id = auth.user_id + if user_id: + q = q.where(Reaction.createdBy != user_id) + + # 3 or fewer votes is 0, 1, 2 or 3 votes (null, reaction id1, reaction id2, reaction id3) + q = q.having(func.count(distinct(Reaction.id)) <= 4) + q = add_stat_columns(q) - q = q.group_by(Shout.id).order_by(desc(Shout.createdAt)).limit(limit) + q = q.group_by(Shout.id).order_by(func.random()).limit(limit) # print(q.compile(compile_kwargs={"literal_binds": True})) From ff834987d410c5c15ea2e31f5cc801c5d023e1aa Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Mon, 18 Dec 2023 14:38:45 +0100 Subject: [PATCH 4/6] unrated shouts query fix (#112) Co-authored-by: Igor Lobanov --- resolvers/zine/load.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/resolvers/zine/load.py b/resolvers/zine/load.py index bda57f6d..c5f04d6e 100644 --- a/resolvers/zine/load.py +++ b/resolvers/zine/load.py @@ -257,6 +257,9 @@ async def load_random_top_shouts(_, info, params): @query.field("loadUnratedShouts") async def load_unrated_shouts(_, info, limit): auth: AuthCredentials = info.context["request"].auth + user_id = auth.user_id + + aliased_reaction = aliased(Reaction) q = ( select(Shout) @@ -272,22 +275,36 @@ async def load_unrated_shouts(_, info, limit): Reaction.kind.in_([ReactionKind.LIKE, ReactionKind.DISLIKE]), ), ) - .where( + ) + + if user_id: + q = q.outerjoin( + aliased_reaction, and_( - Shout.deletedAt.is_(None), - Shout.layout.is_not(None), - Shout.createdAt >= (datetime.now() - timedelta(days=14)).date(), - ) + aliased_reaction.shout == Shout.id, + aliased_reaction.replyTo.is_(None), + aliased_reaction.kind.in_([ReactionKind.LIKE, ReactionKind.DISLIKE]), + aliased_reaction.createdBy == user_id, + ), + ) + + q = q.where( + and_( + Shout.deletedAt.is_(None), + Shout.layout.is_not(None), + Shout.createdAt >= (datetime.now() - timedelta(days=14)).date(), ) ) - user_id = auth.user_id if user_id: - q = q.where(Reaction.createdBy != user_id) + q = q.where(Shout.createdBy != user_id) # 3 or fewer votes is 0, 1, 2 or 3 votes (null, reaction id1, reaction id2, reaction id3) q = q.having(func.count(distinct(Reaction.id)) <= 4) + if user_id: + q = q.having(func.count(distinct(aliased_reaction.id)) == 0) + q = add_stat_columns(q) q = q.group_by(Shout.id).order_by(func.random()).limit(limit) From f395832d32b4e0a9ce241ff3776ca18576e557b3 Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Thu, 21 Dec 2023 00:53:53 +0100 Subject: [PATCH 5/6] random topic shouts query, published date filter in random tops (#113) Co-authored-by: Igor Lobanov --- resolvers/zine/load.py | 41 +++++++++++++++++++++++++++++++++++----- resolvers/zine/topics.py | 19 ++++++++++++++++--- schema.graphql | 6 ++++++ 3 files changed, 58 insertions(+), 8 deletions(-) diff --git a/resolvers/zine/load.py b/resolvers/zine/load.py index c5f04d6e..2cbe20d6 100644 --- a/resolvers/zine/load.py +++ b/resolvers/zine/load.py @@ -22,6 +22,7 @@ from orm import TopicFollower from orm.reaction import Reaction, ReactionKind from orm.shout import Shout, ShoutAuthor, ShoutTopic from orm.user import AuthorFollower +from resolvers.zine.topics import get_random_topic def get_shouts_from_query(q): @@ -79,7 +80,8 @@ def add_stat_columns(q): return q -def apply_filters(q, filters, user_id=None): # noqa: C901 +# use_published_date is a quick fix, will be reworked as a part of tech debt +def apply_filters(q, filters, user_id=None, use_published_date=False): # noqa: C901 if filters.get("reacted") and user_id: q.join(Reaction, Reaction.createdBy == user_id) @@ -100,12 +102,17 @@ def apply_filters(q, filters, user_id=None): # noqa: C901 if filters.get("fromDate"): # fromDate: '2022-12-31 date_from = datetime.strptime(filters.get("fromDate"), "%Y-%m-%d") - q = q.filter(Shout.createdAt >= date_from) + if use_published_date: + q = q.filter(Shout.publishedAt >= date_from) + else: + q = q.filter(Shout.createdAt >= date_from) if filters.get("toDate"): # toDate: '2023-12-31' date_to = datetime.strptime(filters.get("toDate"), "%Y-%m-%d") - q = q.filter(Shout.createdAt < (date_to + timedelta(days=1))) - + if use_published_date: + q = q.filter(Shout.publishedAt < (date_to + timedelta(days=1))) + else: + q = q.filter(Shout.createdAt < (date_to + timedelta(days=1))) return q @@ -228,7 +235,8 @@ async def load_random_top_shouts(_, info, params): .where(and_(Shout.deletedAt.is_(None), Shout.layout.is_not(None))) ) - subquery = apply_filters(subquery, params.get("filters", {})) + subquery = apply_filters(subquery, params.get("filters", {}), use_published_date=True) + subquery = subquery.group_by(Shout.id).order_by(desc(get_rating_func(aliased_reaction))) from_random_count = params.get("fromRandomCount") @@ -254,6 +262,29 @@ async def load_random_top_shouts(_, info, params): return get_shouts_from_query(q) +@query.field("loadRandomTopicShouts") +async def load_random_topic_shouts(_, info, limit): + topic = get_random_topic() + + q = ( + select(Shout) + .options( + joinedload(Shout.authors), + joinedload(Shout.topics), + ) + .join(ShoutTopic, and_(Shout.id == ShoutTopic.shout, ShoutTopic.topic == topic.id)) + .where(and_(Shout.deletedAt.is_(None), Shout.layout.is_not(None))) + ) + + q = add_stat_columns(q) + + q = q.group_by(Shout.id).order_by(desc(Shout.createdAt)).limit(limit) + + shouts = get_shouts_from_query(q) + + return {"topic": topic, "shouts": shouts} + + @query.field("loadUnratedShouts") async def load_unrated_shouts(_, info, limit): auth: AuthCredentials = info.context["request"].auth diff --git a/resolvers/zine/topics.py b/resolvers/zine/topics.py index ad4f59fc..c9c9aae0 100644 --- a/resolvers/zine/topics.py +++ b/resolvers/zine/topics.py @@ -12,11 +12,12 @@ from orm.topic import Topic, TopicFollower def add_topic_stat_columns(q): aliased_shout_author = aliased(ShoutAuthor) aliased_topic_follower = aliased(TopicFollower) + aliased_shout_topic = aliased(ShoutTopic) q = ( - q.outerjoin(ShoutTopic, Topic.id == ShoutTopic.topic) - .add_columns(func.count(distinct(ShoutTopic.shout)).label("shouts_stat")) - .outerjoin(aliased_shout_author, ShoutTopic.shout == aliased_shout_author.shout) + q.outerjoin(aliased_shout_topic, Topic.id == aliased_shout_topic.topic) + .add_columns(func.count(distinct(aliased_shout_topic.shout)).label("shouts_stat")) + .outerjoin(aliased_shout_author, aliased_shout_topic.shout == aliased_shout_author.shout) .add_columns(func.count(distinct(aliased_shout_author.user)).label("authors_stat")) .outerjoin(aliased_topic_follower) .add_columns(func.count(distinct(aliased_topic_follower.follower)).label("followers_stat")) @@ -146,6 +147,18 @@ def topic_unfollow(user_id, slug): return False +def get_random_topic(): + q = select(Topic) + q = q.join(ShoutTopic) + q = q.group_by(Topic.id) + q = q.having(func.count(distinct(ShoutTopic.shout)) > 10) + q = q.order_by(func.random()).limit(1) + + with local_session() as session: + [topic] = session.execute(q).first() + return topic + + @query.field("topicsRandom") async def topics_random(_, info, amount=12): q = select(Topic) diff --git a/schema.graphql b/schema.graphql index 56517191..92560991 100644 --- a/schema.graphql +++ b/schema.graphql @@ -264,6 +264,11 @@ type MySubscriptionsQueryResult { authors: [Author]! } +type RandomTopicShoutsQueryResult { + topic: Topic! + shouts: [Shout]! +} + type Query { # inbox loadChats( limit: Int, offset: Int): Result! # your chats @@ -282,6 +287,7 @@ type Query { loadShout(slug: String, shout_id: Int): Shout loadShouts(options: LoadShoutsOptions): [Shout]! loadRandomTopShouts(params: LoadRandomTopShoutsParams): [Shout]! + loadRandomTopicShouts(limit: Int!): RandomTopicShoutsQueryResult! loadUnratedShouts(limit: Int!): [Shout]! loadDrafts: [Shout]! loadReactionsBy(by: ReactionBy!, limit: Int, offset: Int): [Reaction]! From 67576d0a5bb9760eb2e20a4476fa0bbe20be8620 Mon Sep 17 00:00:00 2001 From: Igor Lobanov Date: Thu, 21 Dec 2023 11:49:28 +0100 Subject: [PATCH 6/6] only published in random topic shouts (#114) --- resolvers/zine/load.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/resolvers/zine/load.py b/resolvers/zine/load.py index 2cbe20d6..58b4ad68 100644 --- a/resolvers/zine/load.py +++ b/resolvers/zine/load.py @@ -273,7 +273,9 @@ async def load_random_topic_shouts(_, info, limit): joinedload(Shout.topics), ) .join(ShoutTopic, and_(Shout.id == ShoutTopic.shout, ShoutTopic.topic == topic.id)) - .where(and_(Shout.deletedAt.is_(None), Shout.layout.is_not(None))) + .where( + and_(Shout.deletedAt.is_(None), Shout.layout.is_not(None), Shout.visibility == "public") + ) ) q = add_stat_columns(q)