From a6c5243c065186883710da9d249b402b84b4a6d5 Mon Sep 17 00:00:00 2001 From: Untone Date: Sun, 17 Dec 2023 23:30:20 +0300 Subject: [PATCH] viewed-service-fixes --- main.py | 5 +++-- orm/invite.py | 8 +++++--- pyproject.toml | 15 +++++++-------- resolvers/__init__.py | 26 +++++++++---------------- resolvers/author.py | 17 +++++++++-------- resolvers/collab.py | 6 +++--- resolvers/community.py | 9 +++++---- resolvers/editor.py | 7 ++++--- resolvers/follower.py | 10 +++++----- resolvers/reaction.py | 13 +++++++------ resolvers/reader.py | 14 ++++++++------ resolvers/topic.py | 8 ++++---- server.py | 3 ++- services/auth.py | 1 + services/notify.py | 1 + services/rediscache.py | 11 +---------- services/schema.py | 3 +-- services/search.py | 8 ++++++-- services/unread.py | 11 ++++++----- services/viewed.py | 43 ++++++++++++++++++++++-------------------- 20 files changed, 110 insertions(+), 109 deletions(-) diff --git a/main.py b/main.py index 2cacbb24..0b69da53 100644 --- a/main.py +++ b/main.py @@ -1,20 +1,21 @@ import os from importlib import import_module from os.path import exists + from ariadne import load_schema_from_path, make_executable_schema from ariadne.asgi import GraphQL +from sentry_sdk.integrations.aiohttp import AioHttpIntegration from sentry_sdk.integrations.ariadne import AriadneIntegration from sentry_sdk.integrations.redis import RedisIntegration from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration from sentry_sdk.integrations.starlette import StarletteIntegration -from sentry_sdk.integrations.aiohttp import AioHttpIntegration from starlette.applications import Starlette from starlette.routing import Route from resolvers.webhook import WebhookEndpoint from services.rediscache import redis from services.schema import resolvers -from settings import DEV_SERVER_PID_FILE_NAME, SENTRY_DSN, MODE +from settings import DEV_SERVER_PID_FILE_NAME, MODE, SENTRY_DSN import_module("resolvers") schema = make_executable_schema(load_schema_from_path("schemas/core.graphql"), resolvers) # type: ignore diff --git a/orm/invite.py b/orm/invite.py index 2b46b8fd..c03c0f2c 100644 --- a/orm/invite.py +++ b/orm/invite.py @@ -1,9 +1,11 @@ -from sqlalchemy import Column, ForeignKey, Enum, String +from enum import Enum as Enumeration + +from sqlalchemy import Column, ForeignKey, String from sqlalchemy.orm import relationship -from services.db import Base + from orm.author import Author from orm.shout import Shout -from enum import Enum as Enumeration +from services.db import Base class InviteStatus(Enumeration): diff --git a/pyproject.toml b/pyproject.toml index 29cab600..6abef300 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,25 +12,24 @@ SQLAlchemy = "^2.0.22" psycopg2-binary = "^2.9.9" redis = {extras = ["hiredis"], version = "^5.0.1"} uvicorn = "^0.24" -sentry-sdk = "^1.38.0" -starlette = "^0.32.0.post1" +sentry-sdk = "^1.39.1" +starlette = "^0.34.0" gql = "^3.4.1" ariadne = "^0.21" aiohttp = "^3.9.1" -requests = "^2.31.0" [tool.poetry.group.dev.dependencies] setuptools = "^69.0.2" +pyright = "^1.1.341" +pytest = "^7.4.2" +black = { version = "^23.12.0", python = ">=3.12" } +ruff = { version = "^0.1.8", python = ">=3.12" } +isort = "^5.13.2" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" -[tool.poetry.dev-dependencies] -pytest = "^7.4.2" -black = { version = "^23.12.0", python = ">=3.12" } -ruff = { version = "^0.1.8", python = ">=3.12" } - [tool.black] line-length = 120 target-version = ['py312'] diff --git a/resolvers/__init__.py b/resolvers/__init__.py index bf8eec1a..e8f31440 100644 --- a/resolvers/__init__.py +++ b/resolvers/__init__.py @@ -1,40 +1,32 @@ -from resolvers.editor import create_shout, delete_shout, update_shout - from resolvers.author import ( get_author, + get_author_followed, + get_author_followers, get_author_id, load_authors_all, - get_author_followers, - get_author_followed, load_authors_by, - update_profile, rate_author, + update_profile, ) - +from resolvers.community import get_communities_all, get_community +from resolvers.editor import create_shout, delete_shout, update_shout +from resolvers.follower import follow, get_my_followed, unfollow from resolvers.reaction import ( create_reaction, - update_reaction, delete_reaction, load_reactions_by, load_shouts_followed, + update_reaction, ) -from resolvers.topic import ( - get_topics_by_author, - get_topics_by_community, - get_topics_all, - get_topic, -) - -from resolvers.follower import follow, unfollow, get_my_followed from resolvers.reader import ( get_shout, load_shouts_by, load_shouts_feed, + load_shouts_random_top, load_shouts_search, load_shouts_unrated, - load_shouts_random_top, ) -from resolvers.community import get_community, get_communities_all +from resolvers.topic import get_topic, get_topics_all, get_topics_by_author, get_topics_by_community __all__ = [ # author diff --git a/resolvers/author.py b/resolvers/author.py index 13d50bce..ee7d39f4 100644 --- a/resolvers/author.py +++ b/resolvers/author.py @@ -1,20 +1,21 @@ import time from typing import List -from sqlalchemy import and_, func, distinct, select, literal, case + +from sqlalchemy import and_, case, distinct, func, literal, select from sqlalchemy.orm import aliased -from orm.reaction import Reaction, ReactionKind -from services.auth import login_required -from services.db import local_session -from services.unread import get_total_unread_counter -from services.schema import mutation, query +from orm.author import Author, AuthorFollower, AuthorRating from orm.community import Community +from orm.reaction import Reaction, ReactionKind from orm.shout import ShoutAuthor, ShoutTopic from orm.topic import Topic -from orm.author import AuthorFollower, Author, AuthorRating from resolvers.community import followed_communities -from resolvers.topic import followed_topics from resolvers.reaction import reacted_shouts_updates as followed_reactions +from resolvers.topic import followed_topics +from services.auth import login_required +from services.db import local_session +from services.schema import mutation, query +from services.unread import get_total_unread_counter def add_author_stat_columns(q): diff --git a/resolvers/collab.py b/resolvers/collab.py index 90aec563..add690fc 100644 --- a/resolvers/collab.py +++ b/resolvers/collab.py @@ -1,9 +1,9 @@ +from orm.author import Author +from orm.invite import Invite, InviteStatus +from orm.shout import Shout from services.auth import login_required from services.db import local_session from services.schema import mutation -from orm.invite import Invite, InviteStatus -from orm.author import Author -from orm.shout import Shout @mutation.field("accept_invite") diff --git a/resolvers/community.py b/resolvers/community.py index fac11ff6..366f92e1 100644 --- a/resolvers/community.py +++ b/resolvers/community.py @@ -1,10 +1,11 @@ -from services.db import local_session -from services.schema import query +from sqlalchemy import and_, distinct, func, literal, select +from sqlalchemy.orm import aliased + from orm.author import Author from orm.community import Community, CommunityAuthor from orm.shout import ShoutCommunity -from sqlalchemy import select, distinct, func, literal, and_ -from sqlalchemy.orm import aliased +from services.db import local_session +from services.schema import query def add_community_stat_columns(q): diff --git a/resolvers/editor.py b/resolvers/editor.py index 9f6dbb0d..4937e21a 100644 --- a/resolvers/editor.py +++ b/resolvers/editor.py @@ -1,15 +1,16 @@ import time # For Unix timestamps + from sqlalchemy import and_, select from sqlalchemy.orm import joinedload from orm.author import Author -from services.auth import login_required -from services.db import local_session -from services.schema import mutation, query from orm.shout import Shout, ShoutAuthor, ShoutTopic, ShoutVisibility from orm.topic import Topic from resolvers.reaction import reactions_follow, reactions_unfollow +from services.auth import login_required +from services.db import local_session from services.notify import notify_shout +from services.schema import mutation, query @query.field("get_shouts_drafts") diff --git a/resolvers/follower.py b/resolvers/follower.py index f74358b4..53aeb142 100644 --- a/resolvers/follower.py +++ b/resolvers/follower.py @@ -2,18 +2,18 @@ from typing import List from sqlalchemy import select -from orm.community import Community, CommunityAuthor +from orm.author import Author, AuthorFollower +from orm.community import Community from orm.reaction import Reaction from orm.shout import Shout from orm.topic import Topic, TopicFollower -from services.auth import login_required from resolvers.author import author_follow, author_unfollow +from resolvers.community import community_follow, community_unfollow from resolvers.reaction import reactions_follow, reactions_unfollow from resolvers.topic import topic_follow, topic_unfollow -from resolvers.community import community_follow, community_unfollow -from services.following import FollowingManager, FollowingResult +from services.auth import login_required from services.db import local_session -from orm.author import Author, AuthorFollower +from services.following import FollowingManager, FollowingResult from services.notify import notify_follower from services.schema import mutation, query diff --git a/resolvers/reaction.py b/resolvers/reaction.py index 65440adb..6438fa68 100644 --- a/resolvers/reaction.py +++ b/resolvers/reaction.py @@ -1,15 +1,16 @@ import time from typing import List -from sqlalchemy import and_, asc, desc, select, text, func, case +from sqlalchemy import and_, asc, case, desc, func, select, text from sqlalchemy.orm import aliased, joinedload -from services.notify import notify_reaction -from services.auth import login_required -from services.db import local_session -from services.schema import mutation, query + +from orm.author import Author from orm.reaction import Reaction, ReactionKind from orm.shout import Shout, ShoutReactionsFollower -from orm.author import Author +from services.auth import login_required +from services.db import local_session +from services.notify import notify_reaction +from services.schema import mutation, query def add_reaction_stat_columns(q): diff --git a/resolvers/reader.py b/resolvers/reader.py index a8cae0fe..3d5a2ea0 100644 --- a/resolvers/reader.py +++ b/resolvers/reader.py @@ -1,15 +1,15 @@ -from sqlalchemy import distinct, bindparam, or_ +from sqlalchemy import bindparam, distinct, or_ from sqlalchemy.orm import aliased, joinedload from sqlalchemy.sql.expression import and_, asc, case, desc, func, nulls_last, select from starlette.exceptions import HTTPException +from orm.author import Author, AuthorFollower +from orm.reaction import Reaction, ReactionKind +from orm.shout import Shout, ShoutAuthor, ShoutTopic, ShoutVisibility +from orm.topic import Topic, TopicFollower from services.auth import login_required from services.db import local_session from services.schema import query -from orm.topic import TopicFollower, Topic -from orm.reaction import Reaction, ReactionKind -from orm.shout import Shout, ShoutAuthor, ShoutTopic, ShoutVisibility -from orm.author import AuthorFollower, Author from services.search import SearchService from services.viewed import ViewedStorage @@ -161,7 +161,9 @@ async def load_shouts_by(_, _info, options): session.query(Topic.slug) .join( ShoutTopic, - and_(ShoutTopic.topic == Topic.id, ShoutTopic.shout == shout.id, ShoutTopic.main == True), + and_( + ShoutTopic.topic == Topic.id, ShoutTopic.shout == shout.id, ShoutTopic.main == True + ), # noqa: E712 ) .first() ) diff --git a/resolvers/topic.py b/resolvers/topic.py index 8adc40a4..c7128b91 100644 --- a/resolvers/topic.py +++ b/resolvers/topic.py @@ -1,12 +1,12 @@ -from sqlalchemy import and_, select, distinct, func +from sqlalchemy import and_, distinct, func, select from sqlalchemy.orm import aliased +from orm.author import Author +from orm.shout import ShoutAuthor, ShoutTopic +from orm.topic import Topic, TopicFollower from services.auth import login_required from services.db import local_session from services.schema import mutation, query -from orm.shout import ShoutTopic, ShoutAuthor -from orm.topic import Topic, TopicFollower -from orm.author import Author async def followed_topics(follower_id): diff --git a/server.py b/server.py index 12dbcfd7..da1a4927 100644 --- a/server.py +++ b/server.py @@ -1,4 +1,5 @@ import sys + import uvicorn from uvicorn.main import logger @@ -58,5 +59,5 @@ if __name__ == "__main__": if "dev" in sys.argv: import os - os.environ.set("MODE", "development") + os.environ["MODE"] = "development" uvicorn.run("main:app", host="0.0.0.0", port=PORT, proxy_headers=True, server_header=True) diff --git a/services/auth.py b/services/auth.py index 25f39b12..305384be 100644 --- a/services/auth.py +++ b/services/auth.py @@ -1,4 +1,5 @@ from functools import wraps + import aiohttp from aiohttp.web import HTTPUnauthorized diff --git a/services/notify.py b/services/notify.py index 8e721699..fdd238cb 100644 --- a/services/notify.py +++ b/services/notify.py @@ -1,4 +1,5 @@ import json + from services.rediscache import redis diff --git a/services/rediscache.py b/services/rediscache.py index 52824713..582de65b 100644 --- a/services/rediscache.py +++ b/services/rediscache.py @@ -1,4 +1,5 @@ import redis.asyncio as aredis + from settings import REDIS_URL @@ -45,16 +46,6 @@ class RedisCache: return await self._client.publish(channel, data) - async def lrange(self, key, start, stop): - if self._client: - print(f"[redis] LRANGE {key} {start} {stop}") - return await self._client.lrange(key, start, stop) - - async def mget(self, key, *keys): - if self._client: - print(f"[redis] MGET {key} {keys}") - return await self._client.mget(key, *keys) - redis = RedisCache() diff --git a/services/schema.py b/services/schema.py index 4a84610b..84364baa 100644 --- a/services/schema.py +++ b/services/schema.py @@ -1,5 +1,4 @@ -from ariadne import QueryType, MutationType # , ScalarType - +from ariadne import MutationType, QueryType # , ScalarType # datetime_scalar = ScalarType("DateTime") query = QueryType() diff --git a/services/search.py b/services/search.py index 665a1f9b..967c15fc 100644 --- a/services/search.py +++ b/services/search.py @@ -1,8 +1,11 @@ import asyncio import json +from typing import List + import aiohttp -from services.rediscache import redis + from orm.shout import Shout +from services.rediscache import redis class SearchService: @@ -16,11 +19,12 @@ class SearchService: SearchService.cache = {} @staticmethod - async def search(text, limit, offset) -> [Shout]: + async def search(text, limit: int = 50, offset: int = 0) -> List[Shout]: cached = await redis.execute("GET", text) if not cached: async with SearchService.lock: # Use aiohttp to send a request to ElasticSearch + # TODO: add limit offset usage async with aiohttp.ClientSession() as session: search_url = f"https://search.discours.io/search?q={text}" async with session.get(search_url) as response: diff --git a/services/unread.py b/services/unread.py index 87c4efa0..fc600a47 100644 --- a/services/unread.py +++ b/services/unread.py @@ -2,14 +2,15 @@ from services.rediscache import redis async def get_unread_counter(chat_id: str, author_id: int) -> int: - unread = await redis.execute("LLEN", f"chats/{chat_id}/unread/{author_id}") - return unread or 0 + unread: int = await redis.execute("LLEN", f"chats/{chat_id}/unread/{author_id}") or 0 + return unread async def get_total_unread_counter(author_id: int) -> int: chats_set = await redis.execute("SMEMBERS", f"chats_by_author/{author_id}") unread = 0 - for chat_id in list(chats_set): - n = await get_unread_counter(chat_id, author_id) - unread += n + if chats_set: + for chat_id in list(chats_set): + n = await get_unread_counter(chat_id, author_id) + unread += n return unread diff --git a/services/viewed.py b/services/viewed.py index d8954c6f..9aeeaf05 100644 --- a/services/viewed.py +++ b/services/viewed.py @@ -1,14 +1,14 @@ import asyncio import time -from datetime import timedelta, timezone, datetime +from datetime import datetime, timedelta, timezone from os import environ from gql import Client, gql from gql.transport.aiohttp import AIOHTTPTransport -from services.db import local_session +from orm.shout import Shout, ShoutTopic from orm.topic import Topic -from orm.shout import ShoutTopic, Shout +from services.db import local_session load_facts = gql( """ query getDomains { @@ -60,7 +60,7 @@ class ViewedStorage: pages = None domains = None period = 60 * 60 # every hour - client = None + client: Client | None = None auth_result = None disabled = False @@ -70,7 +70,7 @@ class ViewedStorage: self = ViewedStorage async with self.lock: if token: - self.client = create_client({"Authorization": "Bearer %s" % str(token)}, schema=schema_str) + self.client = create_client({"Authorization": f"Bearer {token}"}, schema=schema_str) print("[services.viewed] * authorized permanently by ackee.discours.io: %s" % token) else: print("[services.viewed] * please set ACKEE_TOKEN") @@ -83,22 +83,19 @@ class ViewedStorage: start = time.time() self = ViewedStorage try: - async with self.client as session: - self.pages = await session.execute(load_pages) + if self.client: + self.pages = self.client.execute(load_pages) self.pages = self.pages["domains"][0]["statistics"]["pages"] shouts = {} - try: - for page in self.pages: - p = page["value"].split("?")[0] - slug = p.split("discours.io/")[-1] - shouts[slug] = page["count"] - for slug in shouts.keys(): - await ViewedStorage.increment(slug, shouts[slug]) - except Exception: - pass + for page in self.pages: + p = page["value"].split("?")[0] + slug = p.split("discours.io/")[-1] + shouts[slug] = page["count"] + for slug in shouts.keys(): + await ViewedStorage.increment(slug, shouts[slug]) print("[services.viewed] ⎪ %d pages collected " % len(shouts.keys())) except Exception as e: - raise e + raise Exception(e) end = time.time() print("[services.viewed] ⎪ update_pages took %fs " % (end - start)) @@ -106,8 +103,14 @@ class ViewedStorage: @staticmethod async def get_facts(): self = ViewedStorage - async with self.lock: - return await self.client.execute(load_facts) + facts = [] + try: + if self.client: + async with self.lock: + facts = self.client.execute(load_facts) + except Exception as er: + print(f"[services.viewed] get_facts error: {er}") + return facts or [] @staticmethod async def get_shout(shout_slug): @@ -138,7 +141,7 @@ class ViewedStorage: """updates topics counters by shout slug""" self = ViewedStorage with local_session() as session: - for [shout_topic, topic] in ( + for [_shout_topic, topic] in ( session.query(ShoutTopic, Topic).join(Topic).join(Shout).where(Shout.slug == shout_slug).all() ): if not self.by_topics.get(topic.slug):