Merge pull request #38 from Discours/universal-shouts-query-debug

universal shouts query

resolvers/profile.py

@@ -12,6 +12,7 @@ from orm.user import AuthorFollower, Role, User, UserRating, UserRole
 from services.auth.users import UserStorage
 from services.stat.reacted import ReactedStorage
 from services.stat.topicstat import TopicStat
+from services.zine.authors import AuthorsStorage
 from services.zine.shoutauthor import ShoutAuthorStorage
 
 # from .community import followed_communities
@@ -174,12 +175,9 @@ def author_unfollow(user, slug):
 
 @query.field("authorsAll")
 async def get_authors_all(_, _info):
-    users = await UserStorage.get_all_users()
+    authors = await AuthorsStorage.get_all_authors()
-    authors = []
+    for author in authors:
-    for author in users:
+        author.stat = await get_author_stat(author.slug)
-        if ShoutAuthorStorage.shouts_by_author.get(author.slug):
-            author.stat = await get_author_stat(author.slug)
-            authors.append(author)
     return authors
 
 
@@ -212,5 +210,5 @@ async def load_authors_by(_, info, by, limit, offset):
             for a in authors:
                 a.stat = await get_author_stat(a.slug)
     authors = list(set(authors))
-    authors = sorted(authors, key=lambda a: a["stat"].get(by.get("stat")))
+    # authors = sorted(authors, key=lambda a: a["stat"].get(by.get("stat")))
     return authors

resolvers/reactions.py

@@ -217,13 +217,10 @@ async def load_reactions_by(_, info, by, limit=50, offset=0):
     :return: Reaction[]
     """
 
-    q = select(Reaction).options(
+    q = select(Reaction).join(
-        selectinload(Reaction.shout),
+        Shout
     ).where(
         Reaction.deletedAt.is_(None)
-    ).join(
-        Shout,
-        Shout.slug == Reaction.shout
     )
     if by.get("slug"):
         q = q.filter(Shout.slug == by["slug"])
@@ -243,8 +240,9 @@ async def load_reactions_by(_, info, by, limit=50, offset=0):
         if by.get("days"):
             before = datetime.now() - timedelta(days=int(by["days"]) or 30)
             q = q.filter(Reaction.createdAt > before)
-        q = q.group_by(Shout.id).order_by(
+
-            desc(by.get("order") or "createdAt")
+        q = q.group_by(Reaction.id).order_by(
+            desc(by.get("order") or Reaction.createdAt)
         ).limit(limit).offset(offset)
 
     rrr = []

resolvers/zine.py

@@ -1,13 +1,13 @@
 from datetime import datetime, timedelta
-
+import sqlalchemy as sa
 from sqlalchemy.orm import selectinload
-from sqlalchemy.sql.expression import or_, desc, select
+from sqlalchemy.sql.expression import or_, desc, asc, select, case
-
+from timeit import default_timer as timer
 from auth.authenticate import login_required
 from base.orm import local_session
 from base.resolvers import mutation, query
-from orm.shout import Shout
+from orm.shout import Shout, ShoutAuthor
-from orm.reaction import Reaction
+from orm.reaction import Reaction, ReactionKind
 # from resolvers.community import community_follow, community_unfollow
 from resolvers.profile import author_follow, author_unfollow
 from resolvers.reactions import reactions_follow, reactions_unfollow
@@ -16,74 +16,115 @@ from services.zine.shoutauthor import ShoutAuthorStorage
 from services.stat.reacted import ReactedStorage
 
 
-@query.field("loadShoutsBy")
+@query.field("loadShout")
-async def load_shouts_by(_, info, by, limit=50, offset=0):
+async def load_shout(_, info, slug):
+    with local_session() as session:
+        shout = session.query(Shout).options(
+            # TODO add cation
+            selectinload(Shout.authors),
+            selectinload(Shout.topics),
+        ).filter(
+            Shout.slug == slug
+        ).filter(
+            Shout.deletedAt.is_(None)
+        ).one()
+
+        return shout
+
+
+@query.field("loadShouts")
+async def load_shouts_by(_, info, options):
     """
-    :param by: {
+    :param options: {
-        layout: 'audio',
+        filters: {
-        visibility: "public",
+            layout: 'audio',
-        author: 'discours',
+            visibility: "public",
-        topic: 'culture',
+            author: 'discours',
-        title: 'something',
+            topic: 'culture',
-        body: 'something else',
+            title: 'something',
-        stat: 'rating' | 'comments' | 'reacted' | 'views',
+            body: 'something else',
-        days: 30
+            days: 30
+        }
+        offset: 0
+        limit: 50
+        order_by: 'createdAt'
+        order_by_desc: tr
+
     }
-    :param limit: int amount of shouts
-    :param offset: int offset in this order
     :return: Shout[]
     """
 
-    q = select(Shout, Reaction).options(
+    q = select(Shout).options(
+        # TODO add cation
         selectinload(Shout.authors),
         selectinload(Shout.topics),
-        selectinload(Shout.reactions)
     ).where(
         Shout.deletedAt.is_(None)
-    ).join(
-        Reaction, Reaction.shout == Shout.slug
     )
-    if by.get("slug"):
+
-        q = q.filter(Shout.slug == by["slug"])
+    if options.get("filters"):
-    else:
+        if options.get("filters").get("reacted"):
-        if by.get("reacted"):
+            user = info.context["request"].user
-            try:
+            q.join(Reaction, Reaction.createdBy == user.slug)
-                user = info.context["request"].user
+        if options.get("filters").get("visibility"):
-                q = q.filter(Reaction.createdBy == user.slug)
-            except Exception:
-                pass
-        if by.get("visibility"):
             q = q.filter(or_(
-                Shout.visibility.ilike(f"%{by.get('visibility')}%"),
+                Shout.visibility.ilike(f"%{options.get('filters').get('visibility')}%"),
                 Shout.visibility.ilike(f"%{'public'}%"),
             ))
-        if by.get("layout"):
+        if options.get("filters").get("layout"):
-            q = q.filter(Shout.layout == by["layout"])
+            q = q.filter(Shout.layout == options.get("filters").get("layout"))
-        if by.get("author"):
+        if options.get("filters").get("author"):
-            q = q.filter(Shout.authors.contains(by["author"]))
+            q = q.filter(Shout.authors.any(slug=options.get("filters").get("author")))
-        if by.get("topic"):
+        if options.get("filters").get("topic"):
-            q = q.filter(Shout.topics.contains(by["topic"]))
+            q = q.filter(Shout.topics.any(slug=options.get("filters").get("topic")))
-        if by.get("title"):
+        if options.get("filters").get("title"):
-            q = q.filter(Shout.title.ilike(f'%{by["title"]}%'))
+            q = q.filter(Shout.title.ilike(f'%{options.get("filters").get("title")}%'))
-        if by.get("body"):
+        if options.get("filters").get("body"):
-            q = q.filter(Shout.body.ilike(f'%{by["body"]}%'))
+            q = q.filter(Shout.body.ilike(f'%{options.get("filters").get("body")}%s'))
-        if by.get("days"):
+        if options.get("filters").get("days"):
-            before = datetime.now() - timedelta(days=int(by["days"]) or 30)
+            before = datetime.now() - timedelta(days=int(options.get("filter").get("days")) or 30)
             q = q.filter(Shout.createdAt > before)
-        q = q.group_by(Shout.id, Reaction.id).order_by(
+
-            desc(by.get("order") or "createdAt")
+    if options.get("order_by") == 'comments':
-        ).limit(limit).offset(offset)
+        q = q.join(Reaction, Shout.slug == Reaction.shout and Reaction.body.is_not(None)).add_columns(
-    print(q)
+            sa.func.count(Reaction.id).label(options.get("order_by")))
-    shouts = []
+    if options.get("order_by") == 'reacted':
+        q = q.join(Reaction).add_columns(sa.func.max(Reaction.createdAt).label(options.get("order_by")))
+    if options.get("order_by") == "rating":
+        q = q.join(Reaction).add_columns(sa.func.sum(case(
+            (Reaction.kind == ReactionKind.AGREE, 1),
+            (Reaction.kind == ReactionKind.DISAGREE, -1),
+            (Reaction.kind == ReactionKind.PROOF, 1),
+            (Reaction.kind == ReactionKind.DISPROOF, -1),
+            (Reaction.kind == ReactionKind.ACCEPT, 1),
+            (Reaction.kind == ReactionKind.REJECT, -1),
+            (Reaction.kind == ReactionKind.LIKE, 1),
+            (Reaction.kind == ReactionKind.DISLIKE, -1),
+            else_=0
+        )).label(options.get("order_by")))
+    # if order_by == 'views':
+    # TODO dump ackee data to db periodically
+
+    order_by = options.get("order_by") if options.get("order_by") else 'createdAt'
+
+    query_order_by = desc(order_by) if options.get('order_by_desc') else asc(order_by)
+
+    q = q.group_by(Shout.id).order_by(query_order_by).limit(options.get("limit")).offset(
+        options.get("offset") if options.get("offset") else 0)
+
     with local_session() as session:
         # post query stats and author's captions
-        for s in list(map(lambda r: r.Shout, session.execute(q))):
+        # start = timer()
+        shouts = list(map(lambda r: r.Shout, session.execute(q)))
+        for s in shouts:
             s.stat = await ReactedStorage.get_shout_stat(s.slug)
             for a in s.authors:
                 a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug)
-            shouts.append(s)
+
-        if by.get("stat"):
+        # end = timer()
-            shouts.sort(lambda s: s.stat.get(by["stat"]) or s.createdAt)
+        # print(end - start)
+        # print(q)
+
     return shouts
 
 

schema.graphql

@@ -211,19 +211,37 @@ input AuthorsBy {
     stat: String
 }
 
-input ShoutsBy {
+input ShoutsFilterBy {
-    slug: String
+  slug: String
-    title: String
+  title: String
-    body: String
+  body: String
-    topic: String
+  topic: String
-    topics: [String]
+  topics: [String]
-    author: String
+  author: String
-    authors: [String]
+  authors: [String]
-    layout: String
+  layout: String
-    visibility: String
+  visibility: String
-    order: String
+  days: Int
-    days: Int
+  stat: String
-    stat: String
+}
+
+input LoadShoutsFilters {
+  title: String
+  body: String
+  topic: String
+  author: String
+  layout: String
+  visibility: String
+  days: Int
+  reacted: Boolean
+}
+
+input LoadShoutsOptions {
+  filters: LoadShoutsFilters
+  limit: Int!
+  offset: Int
+  order_by: String
+  order_by_desc: Boolean
 }
 
 input ReactionBy {
@@ -252,13 +270,14 @@ type Query {
 
   # zine
   loadAuthorsBy(by: AuthorsBy, limit: Int, offset: Int): [Author]!
-  loadShoutsBy(by: ShoutsBy, limit: Int, offset: Int): [Shout]!
+  loadShout(slug: String!): Shout
+  loadShouts(options: LoadShoutsOptions): [Shout]!
   loadReactionsBy(by: ReactionBy!, limit: Int, offset: Int): [Reaction]!
   userFollowers(slug: String!): [Author]!
   userFollowedAuthors(slug: String!): [Author]!
   userFollowedTopics(slug: String!): [Topic]!
   authorsAll: [Author]!
-  getAuthor(slug: String!): User!
+  getAuthor(slug: String!): User
 
   # collab
   getCollabs: [Collab]!
@@ -267,7 +286,7 @@ type Query {
   markdownBody(body: String!): String!
 
   # topics
-  getTopic(slug: String!): Topic!
+  getTopic(slug: String!): Topic
   topicsAll: [Topic]!
   topicsRandom(amount: Int): [Topic]!
   topicsByCommunity(community: String!): [Topic]!

services/zine/authors.py

@@ -0,0 +1,12 @@
+from base.orm import local_session
+from orm.user import User
+from orm.shout import ShoutAuthor
+
+
+class AuthorsStorage:
+    @staticmethod
+    async def get_all_authors():
+        with local_session() as session:
+            query = session.query(User).join(ShoutAuthor)
+            result = query.all()
+            return result

services/zine/shoutauthor.py

@@ -6,7 +6,6 @@ from orm.shout import ShoutAuthor, Shout
 
 class ShoutAuthorStorage:
     authors_by_shout = {}
-    shouts_by_author = {}
     lock = asyncio.Lock()
     period = 30 * 60  # sec
 
@@ -17,10 +16,7 @@ class ShoutAuthorStorage:
         for sa in sas:
             self.authors_by_shout[sa.shout] = self.authors_by_shout.get(sa.shout, [])
             self.authors_by_shout[sa.shout].append([sa.user, sa.caption])
-            self.shouts_by_author[sa.user] = self.shouts_by_author.get(sa.user, [])
-            self.shouts_by_author[sa.user].append(sa.shout)
         print("[zine.authors] %d shouts indexed by authors" % len(self.authors_by_shout))
-        print("[zine.authors] %d authors indexed by shouts" % len(self.shouts_by_author))
 
     @staticmethod
     async def get_authors(shout):