diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 525a9fc5..42162bc2 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,7 @@ - feat: sentry integration enabled with glitchtip - fix: reindex on update shout - packages upgrade, isort +- separated stats queries for author and topic [0.3.2] - redis cache for what author follows diff --git a/resolvers/__init__.py b/resolvers/__init__.py index dd22ecb7..41e40d03 100644 --- a/resolvers/__init__.py +++ b/resolvers/__init__.py @@ -1,24 +1,52 @@ -from resolvers.author import (get_author, get_author_followers, - get_author_follows, get_author_follows_authors, - get_author_follows_topics, get_author_id, - get_authors_all, load_authors_by, search_authors, - update_author) +from resolvers.author import ( + get_author, + get_author_followers, + get_author_follows, + get_author_follows_authors, + get_author_follows_topics, + get_author_id, + get_authors_all, + load_authors_by, + search_authors, + update_author, +) from resolvers.community import get_communities_all, get_community from resolvers.editor import create_shout, delete_shout, update_shout -from resolvers.follower import (follow, get_shout_followers, - get_topic_followers, unfollow) -from resolvers.notifier import (load_notifications, notification_mark_seen, - notifications_seen_after, - notifications_seen_thread) +from resolvers.follower import ( + follow, + get_shout_followers, + get_topic_followers, + unfollow, +) +from resolvers.notifier import ( + load_notifications, + notification_mark_seen, + notifications_seen_after, + notifications_seen_thread, +) from resolvers.rating import rate_author -from resolvers.reaction import (create_reaction, delete_reaction, - load_reactions_by, load_shouts_followed, - update_reaction) -from resolvers.reader import (get_shout, load_shouts_by, load_shouts_feed, - load_shouts_random_top, load_shouts_random_topic, - load_shouts_search, load_shouts_unrated) -from resolvers.topic import (get_topic, get_topics_all, get_topics_by_author, - get_topics_by_community) +from resolvers.reaction import ( + create_reaction, + delete_reaction, + load_reactions_by, + load_shouts_followed, + update_reaction, +) +from resolvers.reader import ( + get_shout, + load_shouts_by, + load_shouts_feed, + load_shouts_random_top, + load_shouts_random_topic, + load_shouts_search, + load_shouts_unrated, +) +from resolvers.topic import ( + get_topic, + get_topics_all, + get_topics_by_author, + get_topics_by_community, +) from services.triggers import events_register events_register() diff --git a/resolvers/author.py b/resolvers/author.py index fdadb4a7..04993b62 100644 --- a/resolvers/author.py +++ b/resolvers/author.py @@ -8,8 +8,7 @@ from sqlalchemy_searchable import search from orm.author import Author, AuthorFollower from orm.shout import ShoutAuthor, ShoutTopic from orm.topic import Topic -from resolvers.stat import (author_follows_authors, author_follows_topics, - get_with_stat) +from resolvers.stat import author_follows_authors, author_follows_topics, get_with_stat from services.auth import login_required from services.cache import cache_author, cache_follower from services.db import local_session diff --git a/resolvers/collab.py b/resolvers/collab.py index e2fd073f..981f244c 100644 --- a/resolvers/collab.py +++ b/resolvers/collab.py @@ -139,10 +139,9 @@ async def remove_invite(_, info, invite_id: int): author_dict = info.context["author"] author_id = author_dict.get("id") - if author_id: + if isinstance(author_id, int): # Check if the user exists with local_session() as session: - author_id == int(author_id) # Check if the invite exists invite = session.query(Invite).filter(Invite.id == invite_id).first() if isinstance(invite, Invite): diff --git a/resolvers/follower.py b/resolvers/follower.py index eb0d4e09..b29f75ab 100644 --- a/resolvers/follower.py +++ b/resolvers/follower.py @@ -11,8 +11,7 @@ from orm.community import Community from orm.reaction import Reaction from orm.shout import Shout, ShoutReactionsFollower from orm.topic import Topic, TopicFollower -from resolvers.stat import (author_follows_authors, author_follows_topics, - get_with_stat) +from resolvers.stat import author_follows_authors, author_follows_topics, get_with_stat from services.auth import login_required from services.cache import DEFAULT_FOLLOWS, cache_follower from services.db import local_session diff --git a/resolvers/notifier.py b/resolvers/notifier.py index 18e3fc22..8ce824e3 100644 --- a/resolvers/notifier.py +++ b/resolvers/notifier.py @@ -8,8 +8,12 @@ from sqlalchemy.orm import aliased from sqlalchemy.sql import not_ from orm.author import Author -from orm.notification import (Notification, NotificationAction, - NotificationEntity, NotificationSeen) +from orm.notification import ( + Notification, + NotificationAction, + NotificationEntity, + NotificationSeen, +) from orm.shout import Shout from services.auth import login_required from services.db import local_session @@ -142,10 +146,10 @@ def get_notifications_grouped( elif str(notification.entity) == NotificationEntity.REACTION.value: reaction = payload - if not isinstance(shout, dict): + if not isinstance(reaction, dict): raise ValueError("reaction data is not consistent") - shout_id = shout.get("shout") - author_id = shout.get("created_by", 0) + shout_id = reaction.get("shout") + author_id = reaction.get("created_by", 0) if shout_id and author_id: with local_session() as session: author = ( diff --git a/resolvers/reaction.py b/resolvers/reaction.py index 31dd51b7..5abc152c 100644 --- a/resolvers/reaction.py +++ b/resolvers/reaction.py @@ -6,8 +6,7 @@ from sqlalchemy.orm import aliased, joinedload from sqlalchemy.sql import union from orm.author import Author -from orm.rating import (PROPOSAL_REACTIONS, RATING_REACTIONS, is_negative, - is_positive) +from orm.rating import PROPOSAL_REACTIONS, RATING_REACTIONS, is_negative, is_positive from orm.reaction import Reaction, ReactionKind from orm.shout import Shout from resolvers.editor import handle_proposing diff --git a/resolvers/reader.py b/resolvers/reader.py index 8090cd15..fa130bd2 100644 --- a/resolvers/reader.py +++ b/resolvers/reader.py @@ -1,7 +1,6 @@ from sqlalchemy import bindparam, distinct, or_, text from sqlalchemy.orm import aliased, joinedload -from sqlalchemy.sql.expression import (and_, asc, case, desc, func, nulls_last, - select) +from sqlalchemy.sql.expression import and_, asc, case, desc, func, nulls_last, select from orm.author import Author, AuthorFollower from orm.reaction import Reaction, ReactionKind diff --git a/resolvers/stat.py b/resolvers/stat.py index 8d2a2b1a..e50818f0 100644 --- a/resolvers/stat.py +++ b/resolvers/stat.py @@ -10,39 +10,51 @@ from services.db import local_session from services.logger import root_logger as logger -def add_topic_stat_columns(q): - aliased_shout_topic = aliased(ShoutTopic) - aliased_authors = aliased(ShoutAuthor) - aliased_followers = aliased(TopicFollower) - aliased_shout = aliased(Shout) - - # shouts - q = q.outerjoin(aliased_shout_topic, aliased_shout_topic.topic == Topic.id) - q = q.add_columns( - func.count(distinct(aliased_shout_topic.shout)).label("shouts_stat") +def get_topic_shouts_stat(topic_id: int): + q = ( + select(func.count(distinct(ShoutTopic.shout))) + .select_from(join(ShoutTopic, Shout, ShoutTopic.shout == Shout.id)) + .filter( + and_( + ShoutTopic.topic == topic_id, + Shout.published_at.is_not(None), + Shout.deleted_at.is_(None), + ) + ) ) + [shouts_stat] = local_session().execute(q) + return shouts_stat or 0 + +def get_topic_authors_stat(topic_id: int): # authors - q = q.outerjoin( - aliased_shout, - and_( - aliased_shout.id == aliased_shout_topic.shout, - aliased_shout.published_at.is_not(None), - aliased_shout.deleted_at.is_(None), - ), - ) - q = q.outerjoin(aliased_authors, aliased_shout.authors.any(id=aliased_authors.id)) - q = q.add_columns( - func.count(distinct(aliased_authors.author)).label("authors_stat") + q = ( + select(func.count(distinct(ShoutAuthor.author))) + .select_from(join(ShoutTopic, Shout, ShoutTopic.shout == Shout.id)) + .join(ShoutAuthor, ShoutAuthor.shout == Shout.id) + .filter( + and_( + ShoutTopic.topic == topic_id, + Shout.published_at.is_not(None), + Shout.deleted_at.is_(None), + ) + ) ) + [authors_stat] = local_session().execute(q) + return authors_stat or 0 - # followers - q = q.outerjoin(aliased_followers, aliased_followers.topic == Topic.id) - q = q.add_columns( - func.count(distinct(aliased_followers.follower)).label("followers_stat") - ) - # comments +def get_topic_followers_stat(topic_id: int): + aliased_followers = aliased(TopicFollower) + q = select(func.count(distinct(aliased_followers.follower))).filter( + aliased_followers.topic == topic_id + ) + with local_session() as session: + [followers_stat] = session.execute(q) + return followers_stat or 0 + + +def get_topic_comments_stat(topic_id: int): sub_comments = ( select( Shout.id.label("shout_id"), @@ -61,39 +73,48 @@ def add_topic_stat_columns(q): .group_by(Shout.id) .subquery() ) - q = q.outerjoin(sub_comments, aliased_shout_topic.shout == sub_comments.c.shout_id) - q = q.add_columns( - func.coalesce(sub_comments.c.comments_count, 0).label("comments_stat") + q = select(func.coalesce(func.sum(sub_comments.c.comments_count), 0)).filter( + ShoutTopic.topic == topic_id ) - - group_list = [Topic.id, sub_comments.c.comments_count] - - q = q.group_by(*group_list) - logger.debug(q) - return q + q = q.outerjoin(sub_comments, ShoutTopic.shout == sub_comments.c.shout_id) + [comments_stat] = local_session().execute(q) + return comments_stat or 0 -def add_author_stat_columns(q): +def get_author_shouts_stat(author_id: int): aliased_shout_author = aliased(ShoutAuthor) + q = select(func.count(distinct(aliased_shout_author.shout))).filter( + aliased_shout_author.author == author_id + ) + with local_session() as session: + [shouts_stat] = session.execute(q) + return shouts_stat or 0 + + +def get_author_authors_stat(author_id: int): aliased_authors = aliased(AuthorFollower) + q = select(func.count(distinct(aliased_authors.author))).filter( + and_( + aliased_authors.follower == author_id, + aliased_authors.author != author_id, + ) + ) + with local_session() as session: + [authors_stat] = session.execute(q) + return authors_stat or 0 + + +def get_author_followers_stat(author_id: int): aliased_followers = aliased(AuthorFollower) - - q = q.outerjoin(aliased_shout_author, aliased_shout_author.author == Author.id) - q = q.add_columns( - func.count(distinct(aliased_shout_author.shout)).label("shouts_stat") + q = select(func.count(distinct(aliased_followers.follower))).filter( + aliased_followers.author == author_id ) + with local_session() as session: + [followers_stat] = session.execute(q) + return followers_stat or 0 - q = q.outerjoin(aliased_authors, aliased_authors.follower == Author.id) - q = q.add_columns( - func.count(distinct(aliased_authors.author)).label("authors_stat") - ) - q = q.outerjoin(aliased_followers, aliased_followers.author == Author.id) - q = q.add_columns( - func.count(distinct(aliased_followers.follower)).label("followers_stat") - ) - - # Create a subquery for comments count +def get_author_comments_stat(author_id: int): sub_comments = ( select( Author.id, func.coalesce(func.count(Reaction.id)).label("comments_count") @@ -109,37 +130,43 @@ def add_author_stat_columns(q): .group_by(Author.id) .subquery() ) - - q = q.outerjoin(sub_comments, Author.id == sub_comments.c.id) - q = q.add_columns(sub_comments.c.comments_count) - group_list = [Author.id, sub_comments.c.comments_count] - - q = q.group_by(*group_list) - - return q + q = select(sub_comments.c.comments_count).filter(sub_comments.c.id == author_id) + with local_session() as session: + [comments_stat] = session.execute(q) + return comments_stat or 0 def get_with_stat(q): records = [] try: is_author = f"{q}".lower().startswith("select author") - is_topic = f"{q}".lower().startswith("select topic") - if is_author: - q = add_author_stat_columns(q) - elif is_topic: - q = add_topic_stat_columns(q) + f"{q}".lower().startswith("select topic") + result = [] with local_session() as session: result = session.execute(q) - for cols in result: - entity = cols[0] - stat = dict() - stat["shouts"] = cols[1] - stat["authors"] = cols[2] - stat["followers"] = cols[3] - if is_author: - stat["comments"] = cols[4] - entity.stat = stat - records.append(entity) + + for cols in result: + entity = cols[0] + stat = dict() + stat["shouts"] = ( + get_author_shouts_stat(entity.id) + if is_author + else get_topic_shouts_stat(entity.id) + ) + stat["authors"] = ( + get_author_authors_stat(entity.id) + if is_author + else get_topic_authors_stat(entity.id) + ) + stat["followers"] = ( + get_author_followers_stat(entity.id) + if is_author + else get_topic_followers_stat(entity.id) + ) + if is_author: + stat["comments"] = get_author_comments_stat(entity.id) + entity.stat = stat + records.append(entity) except Exception as exc: import traceback diff --git a/services/auth.py b/services/auth.py index 8580ada6..dfc41821 100644 --- a/services/auth.py +++ b/services/auth.py @@ -35,6 +35,7 @@ async def request_data(gql, headers=None): except Exception as e: # Handling and logging exceptions during authentication check import traceback + logger.error(f"request_data error: {e}") logger.error(traceback.format_exc()) return None diff --git a/services/db.py b/services/db.py index 048333b7..7030f3bb 100644 --- a/services/db.py +++ b/services/db.py @@ -5,8 +5,7 @@ import traceback import warnings from typing import Any, Callable, Dict, TypeVar -from sqlalchemy import (JSON, Column, Engine, Integer, create_engine, event, - exc, inspect) +from sqlalchemy import JSON, Column, Engine, Integer, create_engine, event, exc, inspect from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import Session, configure_mappers from sqlalchemy.sql.schema import Table diff --git a/services/viewed.py b/services/viewed.py index c6abfe0f..abf0f22e 100644 --- a/services/viewed.py +++ b/services/viewed.py @@ -7,8 +7,12 @@ from typing import Dict # ga from google.analytics.data_v1beta import BetaAnalyticsDataClient -from google.analytics.data_v1beta.types import (DateRange, Dimension, Metric, - RunReportRequest) +from google.analytics.data_v1beta.types import ( + DateRange, + Dimension, + Metric, + RunReportRequest, +) from orm.author import Author from orm.shout import Shout, ShoutAuthor, ShoutTopic