searchable
All checks were successful
Deploy on push / deploy (push) Successful in 4m21s

This commit is contained in:
Untone 2024-02-25 16:43:04 +03:00
parent 4b83f5d0f5
commit 2222f6fc19
5 changed files with 40 additions and 65 deletions

View File

@ -1,6 +1,8 @@
import time import time
from sqlalchemy import JSON, Boolean, Column, ForeignKey, Integer, String from sqlalchemy import JSON, Boolean, Column, ForeignKey, Integer, String
from sqlalchemy_utils import TSVectorType
from services.db import Base from services.db import Base
@ -38,3 +40,5 @@ class Author(Base):
last_seen = Column(Integer, nullable=False, default=lambda: int(time.time())) last_seen = Column(Integer, nullable=False, default=lambda: int(time.time()))
updated_at = Column(Integer, nullable=False, default=lambda: int(time.time())) updated_at = Column(Integer, nullable=False, default=lambda: int(time.time()))
deleted_at = Column(Integer, nullable=True, comment='Deleted at') deleted_at = Column(Integer, nullable=True, comment='Deleted at')
search_vector = Column(TSVectorType("name", "slug", "bio", "about"))

View File

@ -22,6 +22,7 @@ opensearch-py = "^2.4.2"
httpx = "^0.26.0" httpx = "^0.26.0"
dogpile-cache = "^1.3.1" dogpile-cache = "^1.3.1"
colorlog = "^6.8.2" colorlog = "^6.8.2"
sqlalchemy-searchable = "^2.1.0"
[tool.poetry.group.dev.dependencies] [tool.poetry.group.dev.dependencies]
ruff = "^0.2.1" ruff = "^0.2.1"

View File

@ -1,8 +1,9 @@
import json import json
import time import time
from sqlalchemy import desc, select, or_, and_, func from sqlalchemy import desc, select, or_, and_
from sqlalchemy.orm import aliased from sqlalchemy.orm import aliased
from sqlalchemy_searchable import search
from orm.author import Author, AuthorFollower from orm.author import Author, AuthorFollower
from orm.shout import ShoutAuthor, ShoutTopic from orm.shout import ShoutAuthor, ShoutTopic
@ -212,9 +213,5 @@ def get_author_followers(_, _info, slug: str):
@query.field('search_authors') @query.field('search_authors')
def search_authors(_, info, text: str): def search_authors(_, info, text: str):
v1 = func.to_tsquery('russian', text) q = search(select(Author), text)
v2 = func.to_tsvector(
'russian', Author.name or ' ' or Author.bio or ' ' or Author.about
)
q = select(Author).filter(v2.match(v1))
return get_with_stat(q) return get_with_stat(q)

26
services/cache.py Normal file
View File

@ -0,0 +1,26 @@
from functools import wraps
from dogpile.cache import make_region
# Создание региона кэша с TTL 300 секунд
cache_region = make_region().configure('dogpile.cache.memory', expiration_time=300)
# Декоратор для кэширования методов
def cache_method(cache_key: str):
def decorator(f):
@wraps(f)
def decorated_function(*args, **kwargs):
# Генерация ключа для кэширования
key = cache_key.format(*args, **kwargs)
# Получение значения из кэша
result = cache_region.get(key)
if result is None:
# Если значение отсутствует в кэше, вызываем функцию и кэшируем результат
result = f(*args, **kwargs)
cache_region.set(key, result)
return result
return decorated_function
return decorator

View File

@ -1,14 +1,14 @@
import math import math
import time import time
from functools import wraps
from sqlalchemy import event, Engine, inspect, text
from typing import Any, Callable, Dict, TypeVar from typing import Any, Callable, Dict, TypeVar
from dogpile.cache import make_region from sqlalchemy import exc, event, Engine, inspect, Column, Integer, create_engine
from sqlalchemy import exc, Column, Integer, create_engine
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from sqlalchemy.sql.schema import Table from sqlalchemy.sql.schema import Table
from sqlalchemy_searchable import make_searchable
from services.logger import root_logger as logger from services.logger import root_logger as logger
from settings import DB_URL from settings import DB_URL
import warnings import warnings
@ -31,17 +31,14 @@ warnings.simplefilter('always', exc.SAWarning)
warnings.showwarning = warning_with_traceback warnings.showwarning = warning_with_traceback
warnings.simplefilter('always', exc.SAWarning) warnings.simplefilter('always', exc.SAWarning)
# Создание региона кэша с TTL 300 секунд
cache_region = make_region().configure('dogpile.cache.memory', expiration_time=300)
# Подключение к базе данных SQLAlchemy # Подключение к базе данных SQLAlchemy
engine = create_engine(DB_URL, echo=False, pool_size=10, max_overflow=20) engine = create_engine(DB_URL, echo=False, pool_size=10, max_overflow=20)
inspector = inspect(engine)
T = TypeVar('T') T = TypeVar('T')
REGISTRY: Dict[str, type] = {} REGISTRY: Dict[str, type] = {}
# Перехватчики для журнала запросов SQLAlchemy # Перехватчики для журнала запросов SQLAlchemy
# noinspection PyUnusedLocal
@event.listens_for(Engine, 'before_cursor_execute') @event.listens_for(Engine, 'before_cursor_execute')
def before_cursor_execute(conn, cursor, statement, parameters, context, executemany): def before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
conn.query_start_time = time.time() conn.query_start_time = time.time()
@ -92,54 +89,4 @@ class Base(declarative_base()):
setattr(self, key, value) setattr(self, key, value)
# Декоратор для кэширования методов make_searchable(Base.metadata)
def cache_method(cache_key: str):
def decorator(f):
@wraps(f)
def decorated_function(*args, **kwargs):
# Генерация ключа для кэширования
key = cache_key.format(*args, **kwargs)
# Получение значения из кэша
result = cache_region.get(key)
if result is None:
# Если значение отсутствует в кэше, вызываем функцию и кэшируем результат
result = f(*args, **kwargs)
cache_region.set(key, result)
return result
return decorated_function
return decorator
inspector = inspect(engine)
def add_pg_trgm_extension_if_not_exists():
with local_session() as session:
result = session.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_trgm';"))
if not result.scalar():
session.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm;"))
print("pg_trgm extension added successfully.")
else:
print("pg_trgm extension already exists.")
def create_fts_index(table_name, fts_index_name):
add_pg_trgm_extension_if_not_exists()
logger.info(f'Full text index for {table_name}...')
authors_indexes = inspector.get_indexes(table_name)
author_fts_index_exists = any(
index['name'] == fts_index_name for index in authors_indexes
)
if not author_fts_index_exists:
with local_session() as session:
q = text("""
CREATE INDEX {index_name} ON {author_table_name}
USING gin(to_tsvector('russian', COALESCE(name,'') || ' ' || COALESCE(bio,'') || ' ' || COALESCE(about,'')));
""".format(index_name=fts_index_name, author_table_name=table_name))
session.execute(q)
logger.info('Full text index created successfully.')
create_fts_index('author', 'author_fts_idx')