From c5ee827230842e278b3ab624a8827af24ec8ff62 Mon Sep 17 00:00:00 2001 From: Untone Date: Mon, 12 Aug 2024 11:00:01 +0300 Subject: [PATCH] merged --- cache/cache.py | 8 +++--- cache/precache.py | 10 +++++--- main.py | 3 +-- resolvers/author.py | 12 ++++----- resolvers/reaction.py | 5 ++-- resolvers/reader.py | 56 +++++++++++++++++++++------------------- resolvers/stat.py | 2 +- resolvers/topic.py | 10 +++++--- server.py | 4 ++- services/db.py | 59 ++++++++++++++++++++++++++++--------------- utils/logger.py | 5 ++-- 11 files changed, 103 insertions(+), 71 deletions(-) diff --git a/cache/cache.py b/cache/cache.py index 49895051..08783f65 100644 --- a/cache/cache.py +++ b/cache/cache.py @@ -1,13 +1,15 @@ import asyncio import json from typing import List -from sqlalchemy import select, join, and_ + +from sqlalchemy import and_, join, select + from orm.author import Author, AuthorFollower -from orm.topic import Topic, TopicFollower from orm.shout import Shout, ShoutAuthor, ShoutTopic +from orm.topic import Topic, TopicFollower from services.db import local_session -from utils.encoders import CustomJSONEncoder from services.redis import redis +from utils.encoders import CustomJSONEncoder from utils.logger import root_logger as logger DEFAULT_FOLLOWS = { diff --git a/cache/precache.py b/cache/precache.py index 2c76454d..01da6445 100644 --- a/cache/precache.py +++ b/cache/precache.py @@ -1,15 +1,17 @@ -import json import asyncio +import json + from sqlalchemy import and_, join, select + +from cache.cache import cache_author, cache_topic from orm.author import Author, AuthorFollower -from orm.shout import Shout, ShoutAuthor, ShoutTopic, ShoutReactionsFollower +from orm.shout import Shout, ShoutAuthor, ShoutReactionsFollower, ShoutTopic from orm.topic import Topic, TopicFollower from resolvers.stat import get_with_stat -from cache.cache import cache_author, cache_topic from services.db import local_session +from services.redis import redis from utils.encoders import CustomJSONEncoder from utils.logger import root_logger as logger -from services.redis import redis # Предварительное кеширование подписчиков автора diff --git a/main.py b/main.py index b4d61f20..2dc63466 100644 --- a/main.py +++ b/main.py @@ -13,7 +13,6 @@ from services.exception import ExceptionHandlerMiddleware from services.redis import redis from services.schema import resolvers from services.search import search_service -from services.sentry import start_sentry from services.viewed import ViewedStorage from services.webhook import WebhookEndpoint from settings import DEV_SERVER_PID_FILE_NAME, MODE @@ -42,7 +41,7 @@ app = Starlette( precache_data, ViewedStorage.init, search_service.info, - start_sentry, + # start_sentry, start, revalidation_manager.start, ], diff --git a/resolvers/author.py b/resolvers/author.py index 34f2a0bd..7709c5f1 100644 --- a/resolvers/author.py +++ b/resolvers/author.py @@ -3,11 +3,6 @@ import time from sqlalchemy import desc, select, text -from orm.author import Author -from orm.shout import ShoutAuthor, ShoutTopic -from orm.topic import Topic -from resolvers.stat import get_with_stat -from services.auth import login_required from cache.cache import ( cache_author, get_cached_author, @@ -16,9 +11,14 @@ from cache.cache import ( get_cached_follower_authors, get_cached_follower_topics, ) +from orm.author import Author +from orm.shout import ShoutAuthor, ShoutTopic +from orm.topic import Topic +from resolvers.stat import get_with_stat +from services.auth import login_required from services.db import local_session -from utils.logger import root_logger as logger from services.schema import mutation, query +from utils.logger import root_logger as logger @mutation.field("update_author") diff --git a/resolvers/reaction.py b/resolvers/reaction.py index b7001c03..42c5c997 100644 --- a/resolvers/reaction.py +++ b/resolvers/reaction.py @@ -1,5 +1,6 @@ import time -from sqlalchemy import and_, case, desc, func, select, asc + +from sqlalchemy import and_, asc, case, desc, func, select from sqlalchemy.orm import aliased from orm.author import Author @@ -11,9 +12,9 @@ from resolvers.follower import follow from resolvers.stat import update_author_stat from services.auth import add_user_role, login_required from services.db import local_session -from utils.logger import root_logger as logger from services.notify import notify_reaction from services.schema import mutation, query +from utils.logger import root_logger as logger def query_reactions(): diff --git a/resolvers/reader.py b/resolvers/reader.py index bab76042..f720556b 100644 --- a/resolvers/reader.py +++ b/resolvers/reader.py @@ -280,35 +280,39 @@ async def get_shout(_, info, slug: str): """ try: with local_session() as session: - q, aliased_reaction = query_shouts(slug) - results = session.execute(q).first() - if results: - [ - shout, - commented_stat, - followers_stat, - rating_stat, - last_reaction_at, - authors, - topics, - main_topic_slug, - ] = results + # Отключение автосохранения + with session.no_autoflush: + q, aliased_reaction = query_shouts(slug) + results = session.execute(q).first() + if results: + [ + shout, + commented_stat, + followers_stat, + rating_stat, + last_reaction_at, + authors, + topics, + main_topic_slug, + ] = results - shout.stat = { - "viewed": ViewedStorage.get_shout(shout.id), - "commented": commented_stat, - "rating": rating_stat, - "last_reacted_at": last_reaction_at, - } - # Используем класс модели Author для преобразования строк в объекты - shout.authors = parse_aggregated_string(authors, Author) - # Используем класс модели Topic для преобразования строк в объекты - shout.topics = parse_aggregated_string(topics, Topic) + shout.stat = { + "viewed": ViewedStorage.get_shout(shout.id), + "commented": commented_stat, + "rating": rating_stat, + "last_reacted_at": last_reaction_at, + } - # Добавляем основной топик, если он существует - shout.main_topic = main_topic_slug + # Преобразование строк в объекты Author без их создания + shout.authors = parse_aggregated_string(authors, Author) - return shout + # Преобразование строк в объекты Topic без их создания + shout.topics = parse_aggregated_string(topics, Topic) + + # Добавляем основной топик, если он существует + shout.main_topic = main_topic_slug + + return shout except Exception as _exc: import traceback diff --git a/resolvers/stat.py b/resolvers/stat.py index 2f829313..424c4164 100644 --- a/resolvers/stat.py +++ b/resolvers/stat.py @@ -3,11 +3,11 @@ import asyncio from sqlalchemy import and_, distinct, func, join, select from sqlalchemy.orm import aliased +from cache.cache import cache_author from orm.author import Author, AuthorFollower from orm.reaction import Reaction, ReactionKind from orm.shout import Shout, ShoutAuthor, ShoutTopic from orm.topic import Topic, TopicFollower -from cache.cache import cache_author from services.db import local_session from utils.logger import root_logger as logger diff --git a/resolvers/topic.py b/resolvers/topic.py index 45239576..2d0f66bd 100644 --- a/resolvers/topic.py +++ b/resolvers/topic.py @@ -1,15 +1,19 @@ from sqlalchemy import distinct, func, select +from cache.cache import ( + get_cached_topic_authors, + get_cached_topic_by_slug, + get_cached_topic_followers, +) +from cache.memorycache import cache_region from orm.author import Author from orm.shout import ShoutTopic from orm.topic import Topic from resolvers.stat import get_with_stat from services.auth import login_required -from cache.cache import get_cached_topic_authors, get_cached_topic_by_slug, get_cached_topic_followers from services.db import local_session -from utils.logger import root_logger as logger -from cache.memorycache import cache_region from services.schema import mutation, query +from utils.logger import root_logger as logger # Запрос на получение всех тем diff --git a/server.py b/server.py index 3140d441..8cc35392 100644 --- a/server.py +++ b/server.py @@ -1,6 +1,7 @@ import subprocess from granian.constants import Interfaces +from granian.log import LogLevels from granian.server import Granian from settings import PORT @@ -21,8 +22,9 @@ if __name__ == "__main__": "main:app", address="0.0.0.0", # noqa S104 port=PORT, + interface=Interfaces.ASGI, threads=4, websockets=False, - interface=Interfaces.ASGI, + log_level=LogLevels.debug, ) granian_instance.serve() diff --git a/services/db.py b/services/db.py index 9e32ac9b..0f2d6865 100644 --- a/services/db.py +++ b/services/db.py @@ -1,19 +1,15 @@ import json -import math import time import traceback import warnings +import math from typing import Any, Callable, Dict, TypeVar - from sqlalchemy import JSON, Column, Engine, Integer, create_engine, event, exc, inspect from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import Session, configure_mappers from sqlalchemy.sql.schema import Table - -from settings import DB_URL from utils.logger import root_logger as logger - -# from sqlalchemy_searchable import make_searchable +from settings import DB_URL # Подключение к базе данных SQLAlchemy @@ -96,23 +92,44 @@ warnings.showwarning = warning_with_traceback warnings.simplefilter("always", exc.SAWarning) -@event.listens_for(Engine, "before_cursor_execute") -def before_cursor_execute(conn, cursor, statement, parameters, context, executemany): - conn.query_start_time = time.time() - conn.last_statement = None - - -@event.listens_for(Engine, "after_cursor_execute") -def after_cursor_execute(conn, cursor, statement, parameters, context, executemany): +# Функция для извлечения SQL-запроса из контекста +def get_statement_from_context(context): + query = None compiled_statement = context.compiled.string compiled_parameters = context.compiled.params if compiled_statement: - elapsed = time.time() - conn.query_start_time - if compiled_parameters is not None: - query = compiled_statement.format(*compiled_parameters) + if compiled_parameters: + try: + # Безопасное форматирование параметров + query = compiled_statement % compiled_parameters + except Exception as e: + logger.error(f"Error formatting query: {e}") else: - query = compiled_statement # or handle this case in a way that makes sense for your application + query = compiled_statement + if query: + query = query.replace("\n", " ").replace(" ", " ").strip() + return query - if elapsed > 1 and conn.last_statement != query: - conn.last_statement = query - logger.debug(f"\n{query}\n{'*' * math.floor(elapsed)} {elapsed:.3f} s\n") + +# Обработчик события перед выполнением запроса +@event.listens_for(Engine, "before_cursor_execute") +def before_cursor_execute(conn, cursor, statement, parameters, context, executemany): + conn.query_start_time = time.time() + conn.cursor_id = id(cursor) # Отслеживание конкретного курсора + + +# Обработчик события после выполнения запроса +@event.listens_for(Engine, "after_cursor_execute") +def after_cursor_execute(conn, cursor, statement, parameters, context, executemany): + if hasattr(conn, "cursor_id") and conn.cursor_id == id(cursor): + query = get_statement_from_context(context) + if query: + elapsed = time.time() - conn.query_start_time + if elapsed > 1: + query_end = query[-16:] + query = query.split(query_end)[0] + query_end + logger.debug(query) + elapsed_n = math.floor(elapsed) + logger.debug('*' * elapsed_n) + logger.debug(f"{elapsed_n:.3f} s") + del conn.cursor_id # Удаление идентификатора курсора после выполнения diff --git a/utils/logger.py b/utils/logger.py index d45606fc..27dc4bf6 100644 --- a/utils/logger.py +++ b/utils/logger.py @@ -63,8 +63,9 @@ stream.setFormatter(formatter) # Set up the root logger with the same formatting root_logger = logging.getLogger() -root_logger.setLevel(logging.DEBUG) -root_logger.addHandler(stream) +if not root_logger.hasHandlers(): + root_logger.setLevel(logging.DEBUG) + root_logger.addHandler(stream) ignore_logs = ["_trace", "httpx", "_client", "_trace.atrace", "aiohttp", "_client", "base"] for lgr in ignore_logs: