diff --git a/CHECKS b/disabled-CHECKS similarity index 100% rename from CHECKS rename to disabled-CHECKS diff --git a/main.py b/main.py index b504cc04..7b29f901 100644 --- a/main.py +++ b/main.py @@ -8,6 +8,7 @@ from starlette.middleware import Middleware from starlette.middleware.authentication import AuthenticationMiddleware from starlette.middleware.sessions import SessionMiddleware from starlette.routing import Route +from orm import init_tables from auth.authenticate import JWTAuthenticate from auth.oauth import oauth_login, oauth_authorize @@ -30,6 +31,7 @@ middleware = [ async def start_up(): + init_tables() await redis.connect() await storages_init() views_stat_task = asyncio.create_task(ViewedStorage().worker()) diff --git a/migration/__init__.py b/migration/__init__.py index 6794bf38..a6aaa0ff 100644 --- a/migration/__init__.py +++ b/migration/__init__.py @@ -7,7 +7,6 @@ import sys from datetime import datetime import bs4 -from base.redis import redis from migration.tables.comments import migrate as migrateComment from migration.tables.comments import migrate_2stage as migrateComment_2stage from migration.tables.content_items import get_shout_slug @@ -17,6 +16,7 @@ from migration.tables.users import migrate as migrateUser from migration.tables.users import migrate_2stage as migrateUser_2stage from orm.reaction import Reaction from settings import DB_URL +from orm import init_tables # from export import export_email_subscriptions from .export import export_mdx, export_slug @@ -84,6 +84,7 @@ async def shouts_handle(storage, args): discours_author = 0 anonymous_author = 0 pub_counter = 0 + ignored = 0 topics_dataset_bodies = [] topics_dataset_tlist = [] for entry in storage["shouts"]["data"]: @@ -96,40 +97,44 @@ async def shouts_handle(storage, args): # migrate shout = await migrateShout(entry, storage) - storage["shouts"]["by_oid"][entry["_id"]] = shout - storage["shouts"]["by_slug"][shout["slug"]] = shout - # shouts.topics - if not shout["topics"]: - print("[migration] no topics!") + if shout: + storage["shouts"]["by_oid"][entry["_id"]] = shout + storage["shouts"]["by_slug"][shout["slug"]] = shout + # shouts.topics + if not shout["topics"]: + print("[migration] no topics!") - # with author - author: str = shout["authors"][0].dict() - if author["slug"] == "discours": - discours_author += 1 - if author["slug"] == "anonymous": - anonymous_author += 1 - # print('[migration] ' + shout['slug'] + ' with author ' + author) + # with author + author: str = shout["authors"][0].dict() + if author["slug"] == "discours": + discours_author += 1 + if author["slug"] == "anonymous": + anonymous_author += 1 + # print('[migration] ' + shout['slug'] + ' with author ' + author) - if entry.get("published"): - if "mdx" in args: - export_mdx(shout) - pub_counter += 1 + if entry.get("published"): + if "mdx" in args: + export_mdx(shout) + pub_counter += 1 - # print main counter - counter += 1 - line = str(counter + 1) + ": " + shout["slug"] + " @" + author["slug"] - print(line) + # print main counter + counter += 1 + line = str(counter + 1) + ": " + shout["slug"] + " @" + author["slug"] + print(line) - b = bs4.BeautifulSoup(shout["body"], "html.parser") - texts = [shout["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")] - texts = texts + b.findAll(text=True) - topics_dataset_bodies.append(" ".join([x.strip().lower() for x in texts])) - topics_dataset_tlist.append(shout["topics"]) + b = bs4.BeautifulSoup(shout["body"], "html.parser") + texts = [shout["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")] + texts = texts + b.findAll(text=True) + topics_dataset_bodies.append(" ".join([x.strip().lower() for x in texts])) + topics_dataset_tlist.append(shout["topics"]) + else: + ignored += 1 # np.savetxt('topics_dataset.csv', (topics_dataset_bodies, topics_dataset_tlist), delimiter=', # ', fmt='%s') print("[migration] " + str(counter) + " content items were migrated") + print("[migration] " + str(ignored) + " content items were ignored") print("[migration] " + str(pub_counter) + " have been published") print("[migration] " + str(discours_author) + " authored by @discours") print("[migration] " + str(anonymous_author) + " authored by @anonymous") @@ -182,8 +187,6 @@ async def all_handle(storage, args): await users_handle(storage) await topics_handle(storage) print("[migration] users and topics are migrated") - await redis.connect() - print("[migration] redis connected") await shouts_handle(storage, args) print("[migration] migrating comments") await comments_handle(storage) @@ -314,6 +317,7 @@ async def main(): cmd = sys.argv[1] if type(cmd) == str: print("[migration] command: " + cmd) + init_tables() await handle_auto() else: print("[migration] usage: python server.py migrate") diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index f6f7db7d..4ff8fec5 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -3,10 +3,8 @@ import json from dateutil.parser import parse as date_parse from sqlalchemy.exc import IntegrityError from transliterate import translit - from base.orm import local_session from migration.extract import prepare_html_body -from orm.community import Community from orm.reaction import Reaction, ReactionKind from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower from orm.user import User @@ -103,12 +101,8 @@ async def migrate(entry, storage): r = { "layout": type2layout[entry["type"]], "title": entry["title"], - "community": Community.default_community.id, "authors": [], - "topics": set([]), - # 'rating': 0, - # 'ratings': [], - "createdAt": [], + "topics": set([]) } topics_by_oid = storage["topics"]["by_oid"] users_by_oid = storage["users"]["by_oid"] @@ -177,20 +171,24 @@ async def migrate(entry, storage): # add author as TopicFollower with local_session() as session: for tpc in r['topics']: - tf = session.query( - TopicFollower - ).where( - TopicFollower.follower == userslug - ).filter( - TopicFollower.topic == tpc - ).first() - if not tf: - tf = TopicFollower.create( - topic=tpc, - follower=userslug, - auto=True - ) - session.add(tf) + try: + tf = session.query( + TopicFollower + ).where( + TopicFollower.follower == userslug + ).filter( + TopicFollower.topic == tpc + ).first() + if not tf: + tf = TopicFollower.create( + topic=tpc, + follower=userslug, + auto=True + ) + session.add(tf) + except IntegrityError: + print('[migration.shout] skipped by topic ' + tpc) + return entry["topics"] = r["topics"] entry["cover"] = r["cover"] @@ -205,7 +203,6 @@ async def migrate(entry, storage): user = None del shout_dict["topics"] with local_session() as session: - # c = session.query(Community).all().pop() if not user and userslug: user = session.query(User).filter(User.slug == userslug).first() if not user and userdata: diff --git a/migration/tables/replacements.json b/migration/tables/replacements.json index d23a25aa..15d5b340 100644 --- a/migration/tables/replacements.json +++ b/migration/tables/replacements.json @@ -200,7 +200,6 @@ "ecology": "ecology", "economics": "economics", "eda": "food", - "editing": "editing", "editorial-statements": "editorial-statements", "eduard-limonov": "eduard-limonov", "education": "education", @@ -597,7 +596,6 @@ "r-b": "rnb", "rasizm": "racism", "realizm": "realism", - "redaktura": "editorial", "refleksiya": "reflection", "reggi": "reggae", "religion": "religion", diff --git a/migration/tables/topics.py b/migration/tables/topics.py index bda4ba97..4b563716 100644 --- a/migration/tables/topics.py +++ b/migration/tables/topics.py @@ -1,6 +1,6 @@ from base.orm import local_session from migration.extract import extract_md, html2text -from orm import Topic, Community +from orm import Topic def migrate(entry): @@ -8,9 +8,7 @@ def migrate(entry): topic_dict = { "slug": entry["slug"], "oid": entry["_id"], - "title": entry["title"].replace(" ", " "), - "children": [], - "community": Community.default_community.slug, + "title": entry["title"].replace(" ", " ") } topic_dict["body"] = extract_md(html2text(body_orig), entry["_id"]) with local_session() as session: diff --git a/migration/tables/users.py b/migration/tables/users.py index 16fbfeb8..fe9b7374 100644 --- a/migration/tables/users.py +++ b/migration/tables/users.py @@ -36,6 +36,7 @@ def migrate(entry): ) bio = BeautifulSoup(entry.get("profile").get("bio") or "", features="lxml").text if bio.startswith('<'): + print('[migration] bio! ' + bio) bio = BeautifulSoup(bio, features="lxml").text bio = bio.replace('\(', '(').replace('\)', ')') diff --git a/orm/__init__.py b/orm/__init__.py index 4ac7cee3..dfe0a323 100644 --- a/orm/__init__.py +++ b/orm/__init__.py @@ -6,6 +6,9 @@ from orm.reaction import Reaction from orm.shout import Shout from orm.topic import Topic, TopicFollower from orm.user import User, UserRating +from orm.viewed import ViewedEntry + +# NOTE: keep orm module isolated __all__ = [ "User", @@ -19,13 +22,18 @@ __all__ = [ "Notification", "Reaction", "UserRating" + "ViewedEntry" ] -Base.metadata.create_all(engine) -Operation.init_table() -Resource.init_table() -User.init_table() -Community.init_table() -Role.init_table() -# NOTE: keep orm module isolated +def init_tables(): + Base.metadata.create_all(engine) + Operation.init_table() + Resource.init_table() + User.init_table() + Community.init_table() + UserRating.init_table() + Shout.init_table() + Role.init_table() + ViewedEntry.init_table() + print("[orm] tables initialized") diff --git a/orm/collab.py b/orm/collab.py index 4b205134..3e3a5839 100644 --- a/orm/collab.py +++ b/orm/collab.py @@ -1,8 +1,9 @@ from datetime import datetime from sqlalchemy import Boolean, Column, String, ForeignKey, DateTime - +from sqlalchemy.orm import relationship from base.orm import Base +from orm.user import User class CollabAuthor(Base): @@ -21,5 +22,6 @@ class Collab(Base): title = Column(String, nullable=True, comment="Title") body = Column(String, nullable=True, comment="Body") pic = Column(String, nullable=True, comment="Picture") + authors = relationship(lambda: User, secondary=CollabAuthor.__tablename__) createdAt = Column(DateTime, default=datetime.now, comment="Created At") createdBy = Column(ForeignKey("user.id"), comment="Created By") diff --git a/orm/community.py b/orm/community.py index c3c80837..d789d0e3 100644 --- a/orm/community.py +++ b/orm/community.py @@ -32,12 +32,14 @@ class Community(Base): @staticmethod def init_table(): with local_session() as session: - default = ( + d = ( session.query(Community).filter(Community.slug == "discours").first() ) - if not default: - default = Community.create( - name="Дискурс", slug="discours", createdBy="discours" - ) - - Community.default_community = default + if not d: + d = Community.create( + name="Дискурс", slug="discours", createdBy="anonymous" + ) + session.add(d) + session.commit() + Community.default_community = d + print('[orm] default community id: %s' % d.id) diff --git a/orm/rbac.py b/orm/rbac.py index 903585a3..fccfc5ae 100644 --- a/orm/rbac.py +++ b/orm/rbac.py @@ -50,7 +50,7 @@ class Role(Base): default = Role.create( name="author", desc="Role for author", - community=Community.default_community.id, + community=1, ) Role.default_role = default diff --git a/orm/shout.py b/orm/shout.py index 2a3909ad..f65b1268 100644 --- a/orm/shout.py +++ b/orm/shout.py @@ -1,9 +1,9 @@ from datetime import datetime -from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, JSON +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, String, JSON from sqlalchemy.orm import relationship -from base.orm import Base +from base.orm import Base, local_session from orm.reaction import Reaction from orm.topic import Topic from orm.user import User @@ -43,7 +43,7 @@ class Shout(Base): __tablename__ = "shout" slug = Column(String, unique=True) - community = Column(Integer, ForeignKey("community.id"), nullable=False, comment="Community") + community = Column(ForeignKey("community.id"), default=1) lang = Column(String, nullable=False, default='ru', comment="Language") body = Column(String, nullable=False, comment="Body") title = Column(String, nullable=True) @@ -56,7 +56,6 @@ class Shout(Base): reactions = relationship(lambda: Reaction) visibility = Column(String, nullable=True) # owner authors community public versionOf = Column(ForeignKey("shout.slug"), nullable=True) - lang = Column(String, default='ru') oid = Column(String, nullable=True) media = Column(JSON, nullable=True) @@ -64,3 +63,18 @@ class Shout(Base): updatedAt = Column(DateTime, nullable=True, comment="Updated at") publishedAt = Column(DateTime, nullable=True) deletedAt = Column(DateTime, nullable=True) + + @staticmethod + def init_table(): + with local_session() as session: + s = session.query(Shout).first() + if not s: + entry = { + "slug": "genesis-block", + "body": "", + "title": "Ничего", + "lang": "ru" + } + s = Shout.create(**entry) + session.add(s) + session.commit() diff --git a/orm/topic.py b/orm/topic.py index 7aa86a4a..f4d094b7 100644 --- a/orm/topic.py +++ b/orm/topic.py @@ -1,6 +1,5 @@ from datetime import datetime -from sqlalchemy import JSON as JSONType from sqlalchemy import Boolean, Column, DateTime, ForeignKey, String from base.orm import Base @@ -25,10 +24,7 @@ class Topic(Base): title = Column(String, nullable=False, comment="Title") body = Column(String, nullable=True, comment="Body") pic = Column(String, nullable=True, comment="Picture") - children = Column( - JSONType, nullable=True, default=[], comment="list of children topics" - ) community = Column( - ForeignKey("community.slug"), nullable=False, comment="Community" + ForeignKey("community.id"), default=1, comment="Community" ) oid = Column(String, nullable=True, comment="Old ID") diff --git a/orm/user.py b/orm/user.py index 93694500..75a2d748 100644 --- a/orm/user.py +++ b/orm/user.py @@ -25,6 +25,10 @@ class UserRating(Base): user = Column(ForeignKey("user.slug"), primary_key=True) value = Column(Integer) + @staticmethod + def init_table(): + pass + class UserRole(Base): __tablename__ = "user_role" @@ -48,6 +52,7 @@ class AuthorFollower(Base): class User(Base): __tablename__ = "user" + default_user = None email = Column(String, unique=True, nullable=False, comment="Email") username = Column(String, nullable=False, comment="Login") diff --git a/orm/viewed.py b/orm/viewed.py index 1e0d69dc..6d089234 100644 --- a/orm/viewed.py +++ b/orm/viewed.py @@ -1,13 +1,24 @@ from datetime import datetime -from sqlalchemy import Column, DateTime, ForeignKey -from base.orm import Base +from sqlalchemy import Column, DateTime, ForeignKey, Integer +from base.orm import Base, local_session class ViewedEntry(Base): __tablename__ = "viewed" viewer = Column(ForeignKey("user.slug"), default='anonymous') - shout = Column(ForeignKey("shout.slug")) + shout = Column(ForeignKey("shout.slug"), default="genesis-block") + amount = Column(Integer, default=1) createdAt = Column( DateTime, nullable=False, default=datetime.now, comment="Created at" ) + + @staticmethod + def init_table(): + with local_session() as session: + entry = { + "amount": 0 + } + viewed = ViewedEntry.create(**entry) + session.add(viewed) + session.commit() diff --git a/requirements.txt b/requirements.txt index e47130af..4b36b809 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,3 +25,4 @@ DateTime~=4.7 asyncio~=3.4.3 python-dateutil~=2.8.2 beautifulsoup4~=4.11.1 +lxml diff --git a/resolvers/profile.py b/resolvers/profile.py index 15efc50d..62b50e72 100644 --- a/resolvers/profile.py +++ b/resolvers/profile.py @@ -11,7 +11,6 @@ from orm.topic import Topic, TopicFollower from orm.user import AuthorFollower, Role, User, UserRating, UserRole from services.stat.reacted import ReactedStorage from services.stat.topicstat import TopicStat -from services.zine.authors import AuthorsStorage from services.zine.shoutauthor import ShoutAuthor # from .community import followed_communities @@ -33,7 +32,7 @@ async def get_author_stat(slug): # TODO: implement author stat with local_session() as session: return { - "shouts": session.query(ShoutAuthor).where(ShoutAuthor.author == slug).count(), + "shouts": session.query(ShoutAuthor).where(ShoutAuthor.user == slug).count(), "followers": session.query(AuthorFollower).where(AuthorFollower.author == slug).count(), "followings": session.query(AuthorFollower).where(AuthorFollower.follower == slug).count(), "rating": session.query(func.sum(UserRating.value)).where(UserRating.user == slug).first(), @@ -175,12 +174,22 @@ def author_unfollow(user, slug): @query.field("authorsAll") async def get_authors_all(_, _info): - authors = await AuthorsStorage.get_all_authors() - for author in authors: - author.stat = await get_author_stat(author.slug) + with local_session() as session: + authors = session.query(User).join(ShoutAuthor).all() + for author in authors: + author.stat = await get_author_stat(author.slug) return authors +@query.field("getAuthor") +async def get_author(_, _info, slug): + with local_session() as session: + author = session.query(User).join(ShoutAuthor).where(User.slug == slug).first() + for author in author: + author.stat = await get_author_stat(author.slug) + return author + + @query.field("loadAuthorsBy") async def load_authors_by(_, info, by, limit, offset): authors = [] diff --git a/resolvers/zine.py b/resolvers/zine.py index 45161aae..5ad71850 100644 --- a/resolvers/zine.py +++ b/resolvers/zine.py @@ -1,12 +1,12 @@ +#!/usr/bin/env python3.10 from datetime import datetime, timedelta import sqlalchemy as sa from sqlalchemy.orm import selectinload -from sqlalchemy.sql.expression import or_, desc, asc, select, case -from timeit import default_timer as timer +from sqlalchemy.sql.expression import desc, asc, select, case from auth.authenticate import login_required from base.orm import local_session from base.resolvers import mutation, query -from orm.shout import Shout, ShoutAuthor +from orm.shout import Shout from orm.reaction import Reaction, ReactionKind # from resolvers.community import community_follow, community_unfollow from resolvers.profile import author_follow, author_unfollow @@ -55,7 +55,7 @@ async def load_shouts_by(_, info, options): """ q = select(Shout).options( - # TODO add cation + # TODO add caption selectinload(Shout.authors), selectinload(Shout.topics), ).where( @@ -67,10 +67,7 @@ async def load_shouts_by(_, info, options): user = info.context["request"].user q.join(Reaction, Reaction.createdBy == user.slug) if options.get("filters").get("visibility"): - q = q.filter(or_( - Shout.visibility.ilike(f"%{options.get('filters').get('visibility')}%"), - Shout.visibility.ilike(f"%{'public'}%"), - )) + q = q.filter(Shout.visibility == options.get("filters").get("visibility")) if options.get("filters").get("layout"): q = q.filter(Shout.layout == options.get("filters").get("layout")) if options.get("filters").get("author"): @@ -84,49 +81,45 @@ async def load_shouts_by(_, info, options): if options.get("filters").get("days"): before = datetime.now() - timedelta(days=int(options.get("filter").get("days")) or 30) q = q.filter(Shout.createdAt > before) - - if options.get("order_by") == 'comments': - q = q.join(Reaction, Shout.slug == Reaction.shout and Reaction.body.is_not(None)).add_columns( - sa.func.count(Reaction.id).label(options.get("order_by"))) - if options.get("order_by") == 'reacted': - q = q.join(Reaction).add_columns(sa.func.max(Reaction.createdAt).label(options.get("order_by"))) - if options.get("order_by") == "rating": - q = q.join(Reaction).add_columns(sa.func.sum(case( - (Reaction.kind == ReactionKind.AGREE, 1), - (Reaction.kind == ReactionKind.DISAGREE, -1), - (Reaction.kind == ReactionKind.PROOF, 1), - (Reaction.kind == ReactionKind.DISPROOF, -1), - (Reaction.kind == ReactionKind.ACCEPT, 1), - (Reaction.kind == ReactionKind.REJECT, -1), - (Reaction.kind == ReactionKind.LIKE, 1), - (Reaction.kind == ReactionKind.DISLIKE, -1), - else_=0 - )).label(options.get("order_by"))) - # if order_by == 'views': - # TODO dump ackee data to db periodically - - order_by = options.get("order_by") if options.get("order_by") else 'createdAt' - - order_by_desc = True if options.get('order_by_desc') is None else options.get('order_by_desc') - - query_order_by = desc(order_by) if order_by_desc else asc(order_by) - - q = q.group_by(Shout.id).order_by(query_order_by).limit(options.get("limit")).offset( - options.get("offset") if options.get("offset") else 0) + o = options.get("order_by") + if o: + q = q.add_columns(sa.func.count(Reaction.id).label(o)) + if o == 'comments': + q = q.join(Reaction, Shout.slug == Reaction.shout) + q = q.filter(Reaction.body.is_not(None)) + elif o == 'reacted': + q = q.join( + Reaction + ).add_columns( + sa.func.max(Reaction.createdAt).label(o) + ) + elif o == "rating": + q = q.join(Reaction).add_columns(sa.func.sum(case( + (Reaction.kind == ReactionKind.AGREE, 1), + (Reaction.kind == ReactionKind.DISAGREE, -1), + (Reaction.kind == ReactionKind.PROOF, 1), + (Reaction.kind == ReactionKind.DISPROOF, -1), + (Reaction.kind == ReactionKind.ACCEPT, 1), + (Reaction.kind == ReactionKind.REJECT, -1), + (Reaction.kind == ReactionKind.LIKE, 1), + (Reaction.kind == ReactionKind.DISLIKE, -1), + else_=0 + )).label(o)) + order_by = o + else: + order_by = 'createdAt' + query_order_by = desc(order_by) if options.get("order_by_desc") else asc(order_by) + offset = options.get("offset", 0) + limit = options.get("limit", 10) + q = q.group_by(Shout.id).order_by(query_order_by).limit(limit).offset(offset) with local_session() as session: - # post query stats and author's captions - # start = timer() shouts = list(map(lambda r: r.Shout, session.execute(q))) for s in shouts: s.stat = await ReactedStorage.get_shout_stat(s.slug) for a in s.authors: a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug) - # end = timer() - # print(end - start) - # print(q) - return shouts diff --git a/schema.graphql b/schema.graphql index 214547bf..3b0cdb89 100644 --- a/schema.graphql +++ b/schema.graphql @@ -37,6 +37,7 @@ type AuthorStat { followers: Int rating: Int commented: Int + shouts: Int } @@ -116,8 +117,8 @@ input TopicInput { title: String body: String pic: String - children: [String] - parents: [String] + # children: [String] + # parents: [String] } input ReactionInput { @@ -481,7 +482,7 @@ type TopicStat { shouts: Int! followers: Int! authors: Int! - viewed: Int! + viewed: Int reacted: Int! commented: Int rating: Int @@ -492,8 +493,6 @@ type Topic { title: String body: String pic: String - parents: [String] # NOTE: topic can have parent topics - children: [String] # and children community: Community! stat: TopicStat oid: String diff --git a/server.py b/server.py index 7f92c3c9..584fc7e1 100644 --- a/server.py +++ b/server.py @@ -4,6 +4,50 @@ import uvicorn from settings import PORT +log_settings = { + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'default': { + '()': 'uvicorn.logging.DefaultFormatter', + 'fmt': '%(levelprefix)s %(message)s', + 'use_colors': None + }, + 'access': { + '()': 'uvicorn.logging.AccessFormatter', + 'fmt': '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s' + } + }, + 'handlers': { + 'default': { + 'formatter': 'default', + 'class': 'logging.StreamHandler', + 'stream': 'ext://sys.stderr' + }, + 'access': { + 'formatter': 'access', + 'class': 'logging.StreamHandler', + 'stream': 'ext://sys.stdout' + } + }, + 'loggers': { + 'uvicorn': { + 'handlers': ['default'], + 'level': 'INFO' + }, + 'uvicorn.error': { + 'level': 'INFO', + 'handlers': ['default'], + 'propagate': True + }, + 'uvicorn.access': { + 'handlers': ['access'], + 'level': 'INFO', + 'propagate': False + } + } +} + if __name__ == "__main__": x = "" if len(sys.argv) > 1: @@ -21,11 +65,23 @@ if __name__ == "__main__": ("Access-Control-Allow-Credentials", "true"), ] uvicorn.run( - "main:app", host="localhost", port=8080, headers=headers + "main:app", + host="localhost", + port=8080, + headers=headers, + # log_config=LOGGING_CONFIG, + log_level=None, + access_log=True ) # , ssl_keyfile="discours.key", ssl_certfile="discours.crt", reload=True) elif x == "migrate": from migration import migrate migrate() else: - uvicorn.run("main:app", host="0.0.0.0", port=PORT) + uvicorn.run( + "main:app", + host="0.0.0.0", + port=PORT, + proxy_headers=True, + server_header=True + ) diff --git a/services/main.py b/services/main.py index bcd886f0..ad7cbde5 100644 --- a/services/main.py +++ b/services/main.py @@ -3,12 +3,14 @@ from services.auth.roles import RoleStorage from services.auth.users import UserStorage from services.zine.topics import TopicStorage from services.search import SearchService +from services.stat.viewed import ViewedStorage from base.orm import local_session async def storages_init(): with local_session() as session: print('[main] initialize storages') + ViewedStorage.init() ReactedStorage.init(session) RoleStorage.init(session) UserStorage.init(session) diff --git a/services/stat/reacted.py b/services/stat/reacted.py index fd3d1d93..00f4fb66 100644 --- a/services/stat/reacted.py +++ b/services/stat/reacted.py @@ -181,12 +181,11 @@ class ReactedStorage: c += len(siblings) await self.recount(siblings) - print("[stat.reacted] %d reactions total" % c) - print("[stat.reacted] %d shouts" % len(self.modified_shouts)) + print("[stat.reacted] %d reactions recounted" % c) + print("[stat.reacted] %d shouts modified" % len(self.modified_shouts)) print("[stat.reacted] %d topics" % len(self.reacted["topics"].values())) - print("[stat.reacted] %d shouts" % len(self.reacted["shouts"])) print("[stat.reacted] %d authors" % len(self.reacted["authors"].values())) - print("[stat.reacted] %d reactions replied" % len(self.reacted["reactions"])) + print("[stat.reacted] %d replies" % len(self.reacted["reactions"])) self.modified_shouts = set([]) @staticmethod diff --git a/services/stat/viewed.py b/services/stat/viewed.py index d54a9f78..fc984f5b 100644 --- a/services/stat/viewed.py +++ b/services/stat/viewed.py @@ -2,10 +2,10 @@ import asyncio from gql import Client, gql from gql.transport.aiohttp import AIOHTTPTransport - from base.orm import local_session +from sqlalchemy import func, select +from orm.shout import ShoutTopic from orm.viewed import ViewedEntry -from services.zine.topics import TopicStorage from ssl import create_default_context @@ -42,42 +42,75 @@ ssl = create_default_context() class ViewedStorage: lock = asyncio.Lock() + by_shouts = {} by_topics = {} period = 5 * 60 # 5 minutes client = None transport = None @staticmethod - async def load_views(session): + def init(): + ViewedStorage.transport = AIOHTTPTransport(url="https://ackee.discours.io/", ssl=ssl) + ViewedStorage.client = Client(transport=ViewedStorage.transport, fetch_schema_from_transport=True) + + @staticmethod + async def update_views(session): # TODO: when the struture of payload will be transparent # TODO: perhaps ackee token getting here - self = ViewedStorage() + self = ViewedStorage async with self.lock: - self.transport = AIOHTTPTransport(url="https://ackee.discours.io/", ssl=ssl) - self.client = Client(transport=self.transport, fetch_schema_from_transport=True) domains = await self.client.execute_async(query_ackee_views) print("[stat.ackee] loaded domains") print(domains) print('\n\n# TODO: something here...\n\n') + @staticmethod + async def get_shout(shout_slug): + self = ViewedStorage + async with self.lock: + r = self.by_shouts.get(shout_slug) + if not r: + with local_session() as session: + shout_views = 0 + shout_views_q = select(func.sum(ViewedEntry.amount)).where( + ViewedEntry.shout == shout_slug + ) + shout_views = session.execute(shout_views_q) + self.by_shouts[shout_slug] = shout_views + return shout_views + else: + return r + + @staticmethod + async def get_topic(topic_slug): + self = ViewedStorage + topic_views = 0 + async with self.lock: + topic_views_by_shouts = self.by_topics.get(topic_slug) or {} + for shout in topic_views_by_shouts: + topic_views += shout + return topic_views + @staticmethod async def increment(shout_slug, amount=1, viewer='anonymous'): self = ViewedStorage async with self.lock: with local_session() as session: - viewed = ViewedEntry.create({ + viewed = ViewedEntry.create(**{ "viewer": viewer, - "shout": shout_slug + "shout": shout_slug, + "amount": amount }) session.add(viewed) session.commit() - - shout_topics = await TopicStorage.get_topics_by_slugs([shout_slug, ]) - for t in shout_topics: - self.by_topics[t] = self.by_topics.get(t) or {} - self.by_topics[t][shout_slug] = self.by_topics[t].get(shout_slug) or 0 - self.by_topics[t][shout_slug] += amount + self.by_shouts[shout_slug] = self.by_shouts.get(shout_slug, 0) + amount + topics = session.query(ShoutTopic).where(ShoutTopic.shout == shout_slug).all() + for t in topics: + tpc = t.topic + if not self.by_topics.get(tpc): + self.by_topics[tpc] = {} + self.by_topics[tpc][shout_slug] = self.by_shouts[shout_slug] @staticmethod async def worker(): @@ -85,8 +118,8 @@ class ViewedStorage: while True: try: with local_session() as session: - await self.load_views(session) + await self.update_views(session) + print("[stat.viewed] next renew in %d minutes" % (self.period / 60)) except Exception as err: - print("[stat.viewed] : %s" % (err)) - print("[stat.viewed] renew period: %d minutes" % (self.period / 60)) + print("[stat.viewed] %s" % (err)) await asyncio.sleep(self.period) diff --git a/services/stat/views.py b/services/stat/views.py deleted file mode 100644 index 722d745d..00000000 --- a/services/stat/views.py +++ /dev/null @@ -1,127 +0,0 @@ -import asyncio -import json - -from gql import Client, gql -from gql.transport.aiohttp import AIOHTTPTransport - -from base.redis import redis -from services.zine.topics import TopicStorage -from ssl import create_default_context - - -query_ackee_views = gql( - """ - query getDomainsFacts { - domains { - statistics { - views { - id - count - } - pages { - id - count - created - } - } - facts { - activeVisitors - # averageViews - # averageDuration - viewsToday - viewsMonth - viewsYear - } - } - } - """ -) - -ssl = create_default_context() - - -class ViewStat: - lock = asyncio.Lock() - by_slugs = {} - by_topics = {} - period = 5 * 60 # 5 minutes - transport = AIOHTTPTransport(url="https://ackee.discours.io/", ssl=ssl) - client = Client(transport=transport, fetch_schema_from_transport=True) - - @staticmethod - async def load_views(): - # TODO: when the struture of paylod will be transparent - # TODO: perhaps ackee token getting here - - self = ViewStat - async with self.lock: - self.by_topics = await redis.execute("GET", "views_by_topics") - if self.by_topics: - self.by_topics = dict(json.loads(self.by_topics)) - else: - self.by_topics = {} - self.by_slugs = await redis.execute("GET", "views_by_shouts") - if self.by_slugs: - self.by_slugs = dict(json.loads(self.by_slugs)) - else: - self.by_slugs = {} - domains = await self.client.execute_async(query_ackee_views) - print("[stat.ackee] loaded domains") - print(domains) - - print('\n\n# TODO: something here...\n\n') - - @staticmethod - async def get_shout(shout_slug): - self = ViewStat - async with self.lock: - return self.by_slugs.get(shout_slug) or 0 - - @staticmethod - async def get_topic(topic_slug): - self = ViewStat - async with self.lock: - shouts = self.by_topics.get(topic_slug) or {} - topic_views = 0 - for v in shouts.values(): - topic_views += v - return topic_views - - @staticmethod - async def increment(shout_slug, amount=1): - self = ViewStat - async with self.lock: - self.by_slugs[shout_slug] = self.by_slugs.get(shout_slug) or 0 - self.by_slugs[shout_slug] += amount - await redis.execute( - "SET", - f"views_by_shouts/{shout_slug}", - str(self.by_slugs[shout_slug]) - ) - shout_topics = await TopicStorage.get_topics_by_slugs([shout_slug, ]) - for t in shout_topics: - self.by_topics[t] = self.by_topics.get(t) or {} - self.by_topics[t][shout_slug] = self.by_topics[t].get(shout_slug) or 0 - self.by_topics[t][shout_slug] += amount - await redis.execute( - "SET", - f"views_by_topics/{t}/{shout_slug}", - str(self.by_topics[t][shout_slug]) - ) - - @staticmethod - async def reset(): - self = ViewStat - self.by_topics = {} - self.by_slugs = {} - - @staticmethod - async def worker(): - self = ViewStat - while True: - try: - await self.load_views() - except Exception as err: - print("[stat.ackee] : %s" % (err)) - print("[stat.ackee] renew period: %d minutes" % (ViewStat.period / 60)) - await asyncio.sleep(self.period) diff --git a/services/zine/authors.py b/services/zine/authors.py deleted file mode 100644 index b91e788e..00000000 --- a/services/zine/authors.py +++ /dev/null @@ -1,12 +0,0 @@ -from base.orm import local_session -from orm.user import User -from orm.shout import ShoutAuthor - - -class AuthorsStorage: - @staticmethod - async def get_all_authors(): - with local_session() as session: - query = session.query(User).join(ShoutAuthor) - result = query.all() - return result diff --git a/services/zine/topics.py b/services/zine/topics.py index 97773414..13889899 100644 --- a/services/zine/topics.py +++ b/services/zine/topics.py @@ -12,19 +12,20 @@ class TopicStorage: topics = session.query(Topic) self.topics = dict([(topic.slug, topic) for topic in topics]) for tpc in self.topics.values(): - self.load_parents(tpc) + # self.load_parents(tpc) + pass print("[zine.topics] %d precached" % len(self.topics.keys())) - @staticmethod - def load_parents(topic): - self = TopicStorage - parents = [] - for parent in self.topics.values(): - if topic.slug in parent.children: - parents.append(parent.slug) - topic.parents = parents - return topic + # @staticmethod + # def load_parents(topic): + # self = TopicStorage + # parents = [] + # for parent in self.topics.values(): + # if topic.slug in parent.children: + # parents.append(parent.slug) + # topic.parents = parents + # return topic @staticmethod async def get_topics_all(): @@ -64,4 +65,4 @@ class TopicStorage: self = TopicStorage async with self.lock: self.topics[topic.slug] = topic - self.load_parents(topic) + # self.load_parents(topic)