Merge pull request #42 from Discours/dev

some improves

main.py

@@ -8,6 +8,7 @@ from starlette.middleware import Middleware
 from starlette.middleware.authentication import AuthenticationMiddleware
 from starlette.middleware.sessions import SessionMiddleware
 from starlette.routing import Route
+from orm import init_tables
 
 from auth.authenticate import JWTAuthenticate
 from auth.oauth import oauth_login, oauth_authorize
@@ -30,6 +31,7 @@ middleware = [
 
 
 async def start_up():
+    init_tables()
     await redis.connect()
     await storages_init()
     views_stat_task = asyncio.create_task(ViewedStorage().worker())

migration/__init__.py

@@ -7,7 +7,6 @@ import sys
 from datetime import datetime
 
 import bs4
-from base.redis import redis
 from migration.tables.comments import migrate as migrateComment
 from migration.tables.comments import migrate_2stage as migrateComment_2stage
 from migration.tables.content_items import get_shout_slug
@@ -17,6 +16,7 @@ from migration.tables.users import migrate as migrateUser
 from migration.tables.users import migrate_2stage as migrateUser_2stage
 from orm.reaction import Reaction
 from settings import DB_URL
+from orm import init_tables
 
 # from export import export_email_subscriptions
 from .export import export_mdx, export_slug
@@ -84,6 +84,7 @@ async def shouts_handle(storage, args):
     discours_author = 0
     anonymous_author = 0
     pub_counter = 0
+    ignored = 0
     topics_dataset_bodies = []
     topics_dataset_tlist = []
     for entry in storage["shouts"]["data"]:
@@ -96,6 +97,7 @@ async def shouts_handle(storage, args):
 
         # migrate
         shout = await migrateShout(entry, storage)
+        if shout:
             storage["shouts"]["by_oid"][entry["_id"]] = shout
             storage["shouts"]["by_slug"][shout["slug"]] = shout
             # shouts.topics
@@ -125,11 +127,14 @@ async def shouts_handle(storage, args):
             texts = texts + b.findAll(text=True)
             topics_dataset_bodies.append(" ".join([x.strip().lower() for x in texts]))
             topics_dataset_tlist.append(shout["topics"])
+        else:
+            ignored += 1
 
     # np.savetxt('topics_dataset.csv', (topics_dataset_bodies, topics_dataset_tlist), delimiter=',
     # ', fmt='%s')
 
     print("[migration] " + str(counter) + " content items were migrated")
+    print("[migration] " + str(ignored) + " content items were ignored")
     print("[migration] " + str(pub_counter) + " have been published")
     print("[migration] " + str(discours_author) + " authored by @discours")
     print("[migration] " + str(anonymous_author) + " authored by @anonymous")
@@ -182,8 +187,6 @@ async def all_handle(storage, args):
     await users_handle(storage)
     await topics_handle(storage)
     print("[migration] users and topics are migrated")
-    await redis.connect()
-    print("[migration] redis connected")
     await shouts_handle(storage, args)
     print("[migration] migrating comments")
     await comments_handle(storage)
@@ -314,6 +317,7 @@ async def main():
         cmd = sys.argv[1]
         if type(cmd) == str:
             print("[migration] command: " + cmd)
+        init_tables()
         await handle_auto()
     else:
         print("[migration] usage: python server.py migrate")

migration/tables/content_items.py

@@ -3,10 +3,8 @@ import json
 from dateutil.parser import parse as date_parse
 from sqlalchemy.exc import IntegrityError
 from transliterate import translit
-
 from base.orm import local_session
 from migration.extract import prepare_html_body
-from orm.community import Community
 from orm.reaction import Reaction, ReactionKind
 from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower
 from orm.user import User
@@ -103,12 +101,8 @@ async def migrate(entry, storage):
     r = {
         "layout": type2layout[entry["type"]],
         "title": entry["title"],
-        "community": Community.default_community.id,
         "authors": [],
-        "topics": set([]),
-        # 'rating': 0,
-        # 'ratings': [],
-        "createdAt": [],
+        "topics": set([])
     }
     topics_by_oid = storage["topics"]["by_oid"]
     users_by_oid = storage["users"]["by_oid"]
@@ -177,6 +171,7 @@ async def migrate(entry, storage):
     # add author as TopicFollower
     with local_session() as session:
         for tpc in r['topics']:
+            try:
                 tf = session.query(
                     TopicFollower
                 ).where(
@@ -191,6 +186,9 @@ async def migrate(entry, storage):
                         auto=True
                     )
                     session.add(tf)
+            except IntegrityError:
+                print('[migration.shout] skipped by topic ' + tpc)
+                return
 
     entry["topics"] = r["topics"]
     entry["cover"] = r["cover"]
@@ -205,7 +203,6 @@ async def migrate(entry, storage):
     user = None
     del shout_dict["topics"]
     with local_session() as session:
-        # c = session.query(Community).all().pop()
         if not user and userslug:
             user = session.query(User).filter(User.slug == userslug).first()
         if not user and userdata:

migration/tables/replacements.json

@@ -200,7 +200,6 @@
     "ecology": "ecology",
     "economics": "economics",
     "eda": "food",
-    "editing": "editing",
     "editorial-statements": "editorial-statements",
     "eduard-limonov": "eduard-limonov",
     "education": "education",
@@ -597,7 +596,6 @@
     "r-b": "rnb",
     "rasizm": "racism",
     "realizm": "realism",
-    "redaktura": "editorial",
     "refleksiya": "reflection",
     "reggi": "reggae",
     "religion": "religion",

migration/tables/topics.py

@@ -1,6 +1,6 @@
 from base.orm import local_session
 from migration.extract import extract_md, html2text
-from orm import Topic, Community
+from orm import Topic
 
 
 def migrate(entry):
@@ -8,9 +8,7 @@ def migrate(entry):
     topic_dict = {
         "slug": entry["slug"],
         "oid": entry["_id"],
-        "title": entry["title"].replace(" ", " "),
-        "children": [],
-        "community": Community.default_community.slug,
+        "title": entry["title"].replace(" ", " ")
     }
     topic_dict["body"] = extract_md(html2text(body_orig), entry["_id"])
     with local_session() as session:

migration/tables/users.py

@@ -36,6 +36,7 @@ def migrate(entry):
         )
         bio = BeautifulSoup(entry.get("profile").get("bio") or "", features="lxml").text
         if bio.startswith('<'):
+            print('[migration] bio! ' + bio)
             bio = BeautifulSoup(bio, features="lxml").text
         bio = bio.replace('\(', '(').replace('\)', ')')
 

orm/__init__.py

@@ -6,6 +6,9 @@ from orm.reaction import Reaction
 from orm.shout import Shout
 from orm.topic import Topic, TopicFollower
 from orm.user import User, UserRating
+from orm.viewed import ViewedEntry
+
+# NOTE: keep orm module isolated
 
 __all__ = [
     "User",
@@ -19,13 +22,18 @@ __all__ = [
     "Notification",
     "Reaction",
     "UserRating"
+    "ViewedEntry"
 ]
 
+
+def init_tables():
     Base.metadata.create_all(engine)
     Operation.init_table()
     Resource.init_table()
     User.init_table()
     Community.init_table()
+    UserRating.init_table()
+    Shout.init_table()
     Role.init_table()
-
-# NOTE: keep orm module isolated
+    ViewedEntry.init_table()
+    print("[orm] tables initialized")

orm/collab.py

@@ -1,8 +1,9 @@
 from datetime import datetime
 
 from sqlalchemy import Boolean, Column, String, ForeignKey, DateTime
-
+from sqlalchemy.orm import relationship
 from base.orm import Base
+from orm.user import User
 
 
 class CollabAuthor(Base):
@@ -21,5 +22,6 @@ class Collab(Base):
     title = Column(String, nullable=True, comment="Title")
     body = Column(String, nullable=True, comment="Body")
     pic = Column(String, nullable=True, comment="Picture")
+    authors = relationship(lambda: User, secondary=CollabAuthor.__tablename__)
     createdAt = Column(DateTime, default=datetime.now, comment="Created At")
     createdBy = Column(ForeignKey("user.id"), comment="Created By")

orm/community.py

@@ -32,12 +32,14 @@ class Community(Base):
     @staticmethod
     def init_table():
         with local_session() as session:
-            default = (
+            d = (
                 session.query(Community).filter(Community.slug == "discours").first()
             )
-        if not default:
-            default = Community.create(
-                name="Дискурс", slug="discours", createdBy="discours"
+            if not d:
+                d = Community.create(
+                    name="Дискурс", slug="discours", createdBy="anonymous"
                 )
-
-        Community.default_community = default
+                session.add(d)
+                session.commit()
+            Community.default_community = d
+            print('[orm] default community id: %s' % d.id)

orm/rbac.py

@@ -50,7 +50,7 @@ class Role(Base):
         default = Role.create(
             name="author",
             desc="Role for author",
-            community=Community.default_community.id,
+            community=1,
         )
 
         Role.default_role = default

orm/shout.py

@@ -1,9 +1,9 @@
 from datetime import datetime
 
-from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, JSON
+from sqlalchemy import Boolean, Column, DateTime, ForeignKey, String, JSON
 from sqlalchemy.orm import relationship
 
-from base.orm import Base
+from base.orm import Base, local_session
 from orm.reaction import Reaction
 from orm.topic import Topic
 from orm.user import User
@@ -43,7 +43,7 @@ class Shout(Base):
     __tablename__ = "shout"
 
     slug = Column(String, unique=True)
-    community = Column(Integer, ForeignKey("community.id"), nullable=False, comment="Community")
+    community = Column(ForeignKey("community.id"), default=1)
     lang = Column(String, nullable=False, default='ru', comment="Language")
     body = Column(String, nullable=False, comment="Body")
     title = Column(String, nullable=True)
@@ -56,7 +56,6 @@ class Shout(Base):
     reactions = relationship(lambda: Reaction)
     visibility = Column(String, nullable=True)  # owner authors community public
     versionOf = Column(ForeignKey("shout.slug"), nullable=True)
-    lang = Column(String, default='ru')
     oid = Column(String, nullable=True)
     media = Column(JSON, nullable=True)
 
@@ -64,3 +63,18 @@ class Shout(Base):
     updatedAt = Column(DateTime, nullable=True, comment="Updated at")
     publishedAt = Column(DateTime, nullable=True)
     deletedAt = Column(DateTime, nullable=True)
+
+    @staticmethod
+    def init_table():
+        with local_session() as session:
+            s = session.query(Shout).first()
+            if not s:
+                entry = {
+                    "slug": "genesis-block",
+                    "body": "",
+                    "title": "Ничего",
+                    "lang": "ru"
+                }
+                s = Shout.create(**entry)
+                session.add(s)
+                session.commit()

orm/topic.py

@@ -1,6 +1,5 @@
 from datetime import datetime
 
-from sqlalchemy import JSON as JSONType
 from sqlalchemy import Boolean, Column, DateTime, ForeignKey, String
 
 from base.orm import Base
@@ -25,10 +24,7 @@ class Topic(Base):
     title = Column(String, nullable=False, comment="Title")
     body = Column(String, nullable=True, comment="Body")
     pic = Column(String, nullable=True, comment="Picture")
-    children = Column(
-        JSONType, nullable=True, default=[], comment="list of children topics"
-    )
     community = Column(
-        ForeignKey("community.slug"), nullable=False, comment="Community"
+        ForeignKey("community.id"), default=1, comment="Community"
     )
     oid = Column(String, nullable=True, comment="Old ID")

orm/user.py

@@ -25,6 +25,10 @@ class UserRating(Base):
     user = Column(ForeignKey("user.slug"), primary_key=True)
     value = Column(Integer)
 
+    @staticmethod
+    def init_table():
+        pass
+
 
 class UserRole(Base):
     __tablename__ = "user_role"
@@ -48,6 +52,7 @@ class AuthorFollower(Base):
 
 class User(Base):
     __tablename__ = "user"
+    default_user = None
 
     email = Column(String, unique=True, nullable=False, comment="Email")
     username = Column(String, nullable=False, comment="Login")

orm/viewed.py

@@ -1,13 +1,24 @@
 from datetime import datetime
-from sqlalchemy import Column, DateTime, ForeignKey
-from base.orm import Base
+from sqlalchemy import Column, DateTime, ForeignKey, Integer
+from base.orm import Base, local_session
 
 
 class ViewedEntry(Base):
     __tablename__ = "viewed"
 
     viewer = Column(ForeignKey("user.slug"), default='anonymous')
-    shout = Column(ForeignKey("shout.slug"))
+    shout = Column(ForeignKey("shout.slug"), default="genesis-block")
+    amount = Column(Integer, default=1)
     createdAt = Column(
         DateTime, nullable=False, default=datetime.now, comment="Created at"
     )
+
+    @staticmethod
+    def init_table():
+        with local_session() as session:
+            entry = {
+                "amount": 0
+            }
+            viewed = ViewedEntry.create(**entry)
+            session.add(viewed)
+            session.commit()

requirements.txt

@@ -25,3 +25,4 @@ DateTime~=4.7
 asyncio~=3.4.3
 python-dateutil~=2.8.2
 beautifulsoup4~=4.11.1
+lxml

resolvers/profile.py

@@ -11,7 +11,6 @@ from orm.topic import Topic, TopicFollower
 from orm.user import AuthorFollower, Role, User, UserRating, UserRole
 from services.stat.reacted import ReactedStorage
 from services.stat.topicstat import TopicStat
-from services.zine.authors import AuthorsStorage
 from services.zine.shoutauthor import ShoutAuthor
 
 # from .community import followed_communities
@@ -33,7 +32,7 @@ async def get_author_stat(slug):
     # TODO: implement author stat
     with local_session() as session:
         return {
-            "shouts": session.query(ShoutAuthor).where(ShoutAuthor.author == slug).count(),
+            "shouts": session.query(ShoutAuthor).where(ShoutAuthor.user == slug).count(),
             "followers": session.query(AuthorFollower).where(AuthorFollower.author == slug).count(),
             "followings": session.query(AuthorFollower).where(AuthorFollower.follower == slug).count(),
             "rating": session.query(func.sum(UserRating.value)).where(UserRating.user == slug).first(),
@@ -175,12 +174,22 @@ def author_unfollow(user, slug):
 
 @query.field("authorsAll")
 async def get_authors_all(_, _info):
-    authors = await AuthorsStorage.get_all_authors()
+    with local_session() as session:
+        authors = session.query(User).join(ShoutAuthor).all()
         for author in authors:
             author.stat = await get_author_stat(author.slug)
     return authors
 
 
+@query.field("getAuthor")
+async def get_author(_, _info, slug):
+    with local_session() as session:
+        author = session.query(User).join(ShoutAuthor).where(User.slug == slug).first()
+        for author in author:
+            author.stat = await get_author_stat(author.slug)
+    return author
+
+
 @query.field("loadAuthorsBy")
 async def load_authors_by(_, info, by, limit, offset):
     authors = []

resolvers/zine.py

@@ -1,12 +1,12 @@
+#!/usr/bin/env python3.10
 from datetime import datetime, timedelta
 import sqlalchemy as sa
 from sqlalchemy.orm import selectinload
-from sqlalchemy.sql.expression import or_, desc, asc, select, case
-from timeit import default_timer as timer
+from sqlalchemy.sql.expression import desc, asc, select, case
 from auth.authenticate import login_required
 from base.orm import local_session
 from base.resolvers import mutation, query
-from orm.shout import Shout, ShoutAuthor
+from orm.shout import Shout
 from orm.reaction import Reaction, ReactionKind
 # from resolvers.community import community_follow, community_unfollow
 from resolvers.profile import author_follow, author_unfollow
@@ -55,7 +55,7 @@ async def load_shouts_by(_, info, options):
     """
 
     q = select(Shout).options(
-        # TODO add cation
+        # TODO add caption
         selectinload(Shout.authors),
         selectinload(Shout.topics),
     ).where(
@@ -67,10 +67,7 @@ async def load_shouts_by(_, info, options):
             user = info.context["request"].user
             q.join(Reaction, Reaction.createdBy == user.slug)
         if options.get("filters").get("visibility"):
-            q = q.filter(or_(
-                Shout.visibility.ilike(f"%{options.get('filters').get('visibility')}%"),
-                Shout.visibility.ilike(f"%{'public'}%"),
-            ))
+            q = q.filter(Shout.visibility == options.get("filters").get("visibility"))
         if options.get("filters").get("layout"):
             q = q.filter(Shout.layout == options.get("filters").get("layout"))
         if options.get("filters").get("author"):
@@ -84,13 +81,19 @@ async def load_shouts_by(_, info, options):
         if options.get("filters").get("days"):
             before = datetime.now() - timedelta(days=int(options.get("filter").get("days")) or 30)
             q = q.filter(Shout.createdAt > before)
-
-    if options.get("order_by") == 'comments':
-        q = q.join(Reaction, Shout.slug == Reaction.shout and Reaction.body.is_not(None)).add_columns(
-            sa.func.count(Reaction.id).label(options.get("order_by")))
-    if options.get("order_by") == 'reacted':
-        q = q.join(Reaction).add_columns(sa.func.max(Reaction.createdAt).label(options.get("order_by")))
-    if options.get("order_by") == "rating":
+    o = options.get("order_by")
+    if o:
+        q = q.add_columns(sa.func.count(Reaction.id).label(o))
+        if o == 'comments':
+            q = q.join(Reaction, Shout.slug == Reaction.shout)
+            q = q.filter(Reaction.body.is_not(None))
+        elif o == 'reacted':
+            q = q.join(
+                Reaction
+            ).add_columns(
+                sa.func.max(Reaction.createdAt).label(o)
+            )
+        elif o == "rating":
             q = q.join(Reaction).add_columns(sa.func.sum(case(
                 (Reaction.kind == ReactionKind.AGREE, 1),
                 (Reaction.kind == ReactionKind.DISAGREE, -1),
@@ -101,32 +104,22 @@ async def load_shouts_by(_, info, options):
                 (Reaction.kind == ReactionKind.LIKE, 1),
                 (Reaction.kind == ReactionKind.DISLIKE, -1),
                 else_=0
-        )).label(options.get("order_by")))
-    # if order_by == 'views':
-    # TODO dump ackee data to db periodically
-
-    order_by = options.get("order_by") if options.get("order_by") else 'createdAt'
-
-    order_by_desc = True if options.get('order_by_desc') is None else options.get('order_by_desc')
-
-    query_order_by = desc(order_by) if order_by_desc else asc(order_by)
-
-    q = q.group_by(Shout.id).order_by(query_order_by).limit(options.get("limit")).offset(
-        options.get("offset") if options.get("offset") else 0)
+            )).label(o))
+        order_by = o
+    else:
+        order_by = 'createdAt'
+    query_order_by = desc(order_by) if options.get("order_by_desc") else asc(order_by)
+    offset = options.get("offset", 0)
+    limit = options.get("limit", 10)
+    q = q.group_by(Shout.id).order_by(query_order_by).limit(limit).offset(offset)
 
     with local_session() as session:
-        # post query stats and author's captions
-        # start = timer()
         shouts = list(map(lambda r: r.Shout, session.execute(q)))
         for s in shouts:
             s.stat = await ReactedStorage.get_shout_stat(s.slug)
             for a in s.authors:
                 a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug)
 
-        # end = timer()
-        # print(end - start)
-        # print(q)
-
     return shouts
 
 

schema.graphql

@@ -37,6 +37,7 @@ type AuthorStat {
     followers: Int
     rating: Int
     commented: Int
+    shouts: Int
 }
 
 
@@ -116,8 +117,8 @@ input TopicInput {
   title: String
   body: String
   pic: String
-  children: [String]
-  parents: [String]
+  # children: [String]
+  # parents: [String]
 }
 
 input ReactionInput {
@@ -481,7 +482,7 @@ type TopicStat {
   shouts: Int!
   followers: Int!
   authors: Int!
-  viewed: Int!
+  viewed: Int
   reacted: Int!
   commented: Int
   rating: Int
@@ -492,8 +493,6 @@ type Topic {
   title: String
   body: String
   pic: String
-  parents: [String] # NOTE: topic can have parent topics
-  children: [String] # and children
   community: Community!
   stat: TopicStat
   oid: String

server.py

@@ -4,6 +4,50 @@ import uvicorn
 
 from settings import PORT
 
+log_settings = {
+    'version': 1,
+    'disable_existing_loggers': True,
+    'formatters': {
+        'default': {
+            '()': 'uvicorn.logging.DefaultFormatter',
+            'fmt': '%(levelprefix)s %(message)s',
+            'use_colors': None
+        },
+        'access': {
+            '()': 'uvicorn.logging.AccessFormatter',
+            'fmt': '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s'
+        }
+    },
+    'handlers': {
+        'default': {
+            'formatter': 'default',
+            'class': 'logging.StreamHandler',
+            'stream': 'ext://sys.stderr'
+        },
+        'access': {
+            'formatter': 'access',
+            'class': 'logging.StreamHandler',
+            'stream': 'ext://sys.stdout'
+        }
+    },
+    'loggers': {
+        'uvicorn': {
+            'handlers': ['default'],
+            'level': 'INFO'
+        },
+        'uvicorn.error': {
+            'level': 'INFO',
+            'handlers': ['default'],
+            'propagate': True
+        },
+        'uvicorn.access': {
+            'handlers': ['access'],
+            'level': 'INFO',
+            'propagate': False
+        }
+    }
+}
+
 if __name__ == "__main__":
     x = ""
     if len(sys.argv) > 1:
@@ -21,11 +65,23 @@ if __name__ == "__main__":
             ("Access-Control-Allow-Credentials", "true"),
         ]
         uvicorn.run(
-            "main:app", host="localhost", port=8080, headers=headers
+            "main:app",
+            host="localhost",
+            port=8080,
+            headers=headers,
+            # log_config=LOGGING_CONFIG,
+            log_level=None,
+            access_log=True
         )  # , ssl_keyfile="discours.key", ssl_certfile="discours.crt", reload=True)
     elif x == "migrate":
         from migration import migrate
 
         migrate()
     else:
-        uvicorn.run("main:app", host="0.0.0.0", port=PORT)
+        uvicorn.run(
+            "main:app",
+            host="0.0.0.0",
+            port=PORT,
+            proxy_headers=True,
+            server_header=True
+        )

services/main.py

@@ -3,12 +3,14 @@ from services.auth.roles import RoleStorage
 from services.auth.users import UserStorage
 from services.zine.topics import TopicStorage
 from services.search import SearchService
+from services.stat.viewed import ViewedStorage
 from base.orm import local_session
 
 
 async def storages_init():
     with local_session() as session:
         print('[main] initialize storages')
+        ViewedStorage.init()
         ReactedStorage.init(session)
         RoleStorage.init(session)
         UserStorage.init(session)

services/stat/reacted.py

@@ -181,12 +181,11 @@ class ReactedStorage:
                 c += len(siblings)
                 await self.recount(siblings)
 
-            print("[stat.reacted] %d reactions total" % c)
-            print("[stat.reacted] %d shouts" % len(self.modified_shouts))
+            print("[stat.reacted] %d reactions recounted" % c)
+            print("[stat.reacted] %d shouts modified" % len(self.modified_shouts))
             print("[stat.reacted] %d topics" % len(self.reacted["topics"].values()))
-            print("[stat.reacted] %d shouts" % len(self.reacted["shouts"]))
             print("[stat.reacted] %d authors" % len(self.reacted["authors"].values()))
-            print("[stat.reacted] %d reactions replied" % len(self.reacted["reactions"]))
+            print("[stat.reacted] %d replies" % len(self.reacted["reactions"]))
             self.modified_shouts = set([])
 
     @staticmethod

services/stat/viewed.py

@@ -2,10 +2,10 @@ import asyncio
 
 from gql import Client, gql
 from gql.transport.aiohttp import AIOHTTPTransport
-
 from base.orm import local_session
+from sqlalchemy import func, select
+from orm.shout import ShoutTopic
 from orm.viewed import ViewedEntry
-from services.zine.topics import TopicStorage
 from ssl import create_default_context
 
 
@@ -42,42 +42,75 @@ ssl = create_default_context()
 
 class ViewedStorage:
     lock = asyncio.Lock()
+    by_shouts = {}
     by_topics = {}
     period = 5 * 60  # 5 minutes
     client = None
     transport = None
 
     @staticmethod
-    async def load_views(session):
+    def init():
+        ViewedStorage.transport = AIOHTTPTransport(url="https://ackee.discours.io/", ssl=ssl)
+        ViewedStorage.client = Client(transport=ViewedStorage.transport, fetch_schema_from_transport=True)
+
+    @staticmethod
+    async def update_views(session):
         # TODO: when the struture of payload will be transparent
         # TODO: perhaps ackee token getting here
 
-        self = ViewedStorage()
+        self = ViewedStorage
         async with self.lock:
-            self.transport = AIOHTTPTransport(url="https://ackee.discours.io/", ssl=ssl)
-            self.client = Client(transport=self.transport, fetch_schema_from_transport=True)
             domains = await self.client.execute_async(query_ackee_views)
             print("[stat.ackee] loaded domains")
             print(domains)
             print('\n\n# TODO: something here...\n\n')
 
+    @staticmethod
+    async def get_shout(shout_slug):
+        self = ViewedStorage
+        async with self.lock:
+            r = self.by_shouts.get(shout_slug)
+            if not r:
+                with local_session() as session:
+                    shout_views = 0
+                    shout_views_q = select(func.sum(ViewedEntry.amount)).where(
+                        ViewedEntry.shout == shout_slug
+                    )
+                    shout_views = session.execute(shout_views_q)
+                    self.by_shouts[shout_slug] = shout_views
+                    return shout_views
+            else:
+                return r
+
+    @staticmethod
+    async def get_topic(topic_slug):
+        self = ViewedStorage
+        topic_views = 0
+        async with self.lock:
+            topic_views_by_shouts = self.by_topics.get(topic_slug) or {}
+            for shout in topic_views_by_shouts:
+                topic_views += shout
+        return topic_views
+
     @staticmethod
     async def increment(shout_slug, amount=1, viewer='anonymous'):
         self = ViewedStorage
         async with self.lock:
             with local_session() as session:
-                viewed = ViewedEntry.create({
+                viewed = ViewedEntry.create(**{
                     "viewer": viewer,
-                    "shout": shout_slug
+                    "shout": shout_slug,
+                    "amount": amount
                 })
                 session.add(viewed)
                 session.commit()
-
-                shout_topics = await TopicStorage.get_topics_by_slugs([shout_slug, ])
-                for t in shout_topics:
-                    self.by_topics[t] = self.by_topics.get(t) or {}
-                    self.by_topics[t][shout_slug] = self.by_topics[t].get(shout_slug) or 0
-                    self.by_topics[t][shout_slug] += amount
+                self.by_shouts[shout_slug] = self.by_shouts.get(shout_slug, 0) + amount
+                topics = session.query(ShoutTopic).where(ShoutTopic.shout == shout_slug).all()
+                for t in topics:
+                    tpc = t.topic
+                    if not self.by_topics.get(tpc):
+                        self.by_topics[tpc] = {}
+                    self.by_topics[tpc][shout_slug] = self.by_shouts[shout_slug]
 
     @staticmethod
     async def worker():
@@ -85,8 +118,8 @@ class ViewedStorage:
         while True:
             try:
                 with local_session() as session:
-                    await self.load_views(session)
+                    await self.update_views(session)
+                    print("[stat.viewed] next renew in %d minutes" % (self.period / 60))
             except Exception as err:
-                print("[stat.viewed] : %s" % (err))
-            print("[stat.viewed] renew period: %d minutes" % (self.period / 60))
+                print("[stat.viewed] %s" % (err))
             await asyncio.sleep(self.period)

services/stat/views.py

@@ -1,127 +0,0 @@
-import asyncio
-import json
-
-from gql import Client, gql
-from gql.transport.aiohttp import AIOHTTPTransport
-
-from base.redis import redis
-from services.zine.topics import TopicStorage
-from ssl import create_default_context
-
-
-query_ackee_views = gql(
-    """
-    query getDomainsFacts {
-        domains {
-            statistics {
-                views {
-                    id
-                    count
-                }
-                pages {
-                    id
-                    count
-                    created
-                }
-            }
-            facts {
-                activeVisitors
-                # averageViews
-                # averageDuration
-                viewsToday
-                viewsMonth
-                viewsYear
-            }
-        }
-    }
-    """
-)
-
-ssl = create_default_context()
-
-
-class ViewStat:
-    lock = asyncio.Lock()
-    by_slugs = {}
-    by_topics = {}
-    period = 5 * 60  # 5 minutes
-    transport = AIOHTTPTransport(url="https://ackee.discours.io/", ssl=ssl)
-    client = Client(transport=transport, fetch_schema_from_transport=True)
-
-    @staticmethod
-    async def load_views():
-        # TODO: when the struture of paylod will be transparent
-        # TODO: perhaps ackee token getting here
-
-        self = ViewStat
-        async with self.lock:
-            self.by_topics = await redis.execute("GET", "views_by_topics")
-            if self.by_topics:
-                self.by_topics = dict(json.loads(self.by_topics))
-            else:
-                self.by_topics = {}
-            self.by_slugs = await redis.execute("GET", "views_by_shouts")
-            if self.by_slugs:
-                self.by_slugs = dict(json.loads(self.by_slugs))
-            else:
-                self.by_slugs = {}
-            domains = await self.client.execute_async(query_ackee_views)
-            print("[stat.ackee] loaded domains")
-            print(domains)
-
-            print('\n\n# TODO: something here...\n\n')
-
-    @staticmethod
-    async def get_shout(shout_slug):
-        self = ViewStat
-        async with self.lock:
-            return self.by_slugs.get(shout_slug) or 0
-
-    @staticmethod
-    async def get_topic(topic_slug):
-        self = ViewStat
-        async with self.lock:
-            shouts = self.by_topics.get(topic_slug) or {}
-            topic_views = 0
-            for v in shouts.values():
-                topic_views += v
-            return topic_views
-
-    @staticmethod
-    async def increment(shout_slug, amount=1):
-        self = ViewStat
-        async with self.lock:
-            self.by_slugs[shout_slug] = self.by_slugs.get(shout_slug) or 0
-            self.by_slugs[shout_slug] += amount
-            await redis.execute(
-                "SET",
-                f"views_by_shouts/{shout_slug}",
-                str(self.by_slugs[shout_slug])
-            )
-            shout_topics = await TopicStorage.get_topics_by_slugs([shout_slug, ])
-            for t in shout_topics:
-                self.by_topics[t] = self.by_topics.get(t) or {}
-                self.by_topics[t][shout_slug] = self.by_topics[t].get(shout_slug) or 0
-                self.by_topics[t][shout_slug] += amount
-                await redis.execute(
-                    "SET",
-                    f"views_by_topics/{t}/{shout_slug}",
-                    str(self.by_topics[t][shout_slug])
-                )
-
-    @staticmethod
-    async def reset():
-        self = ViewStat
-        self.by_topics = {}
-        self.by_slugs = {}
-
-    @staticmethod
-    async def worker():
-        self = ViewStat
-        while True:
-            try:
-                await self.load_views()
-            except Exception as err:
-                print("[stat.ackee] : %s" % (err))
-            print("[stat.ackee] renew period: %d minutes" % (ViewStat.period / 60))
-            await asyncio.sleep(self.period)

services/zine/authors.py

@@ -1,12 +0,0 @@
-from base.orm import local_session
-from orm.user import User
-from orm.shout import ShoutAuthor
-
-
-class AuthorsStorage:
-    @staticmethod
-    async def get_all_authors():
-        with local_session() as session:
-            query = session.query(User).join(ShoutAuthor)
-            result = query.all()
-            return result

services/zine/topics.py

@@ -12,19 +12,20 @@ class TopicStorage:
         topics = session.query(Topic)
         self.topics = dict([(topic.slug, topic) for topic in topics])
         for tpc in self.topics.values():
-            self.load_parents(tpc)
+            # self.load_parents(tpc)
+            pass
 
         print("[zine.topics] %d precached" % len(self.topics.keys()))
 
-    @staticmethod
-    def load_parents(topic):
-        self = TopicStorage
-        parents = []
-        for parent in self.topics.values():
-            if topic.slug in parent.children:
-                parents.append(parent.slug)
-        topic.parents = parents
-        return topic
+    # @staticmethod
+    # def load_parents(topic):
+    #     self = TopicStorage
+    #     parents = []
+    #     for parent in self.topics.values():
+    #         if topic.slug in parent.children:
+    #             parents.append(parent.slug)
+    #     topic.parents = parents
+    #     return topic
 
     @staticmethod
     async def get_topics_all():
@@ -64,4 +65,4 @@ class TopicStorage:
         self = TopicStorage
         async with self.lock:
             self.topics[topic.slug] = topic
-            self.load_parents(topic)
+            # self.load_parents(topic)