discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

Merge pull request #38 from Discours/universal-shouts-query-debug

Browse Source 
universal shouts query
This commit is contained in:
Tony 2022-11-18 07:31:29 +03:00 committed by GitHub
commit 43c3c1b28b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 152 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
resolvers/profile.py
View File

@ -12,6 +12,7 @@ from orm.user import AuthorFollower, Role, User, UserRating, UserRole
from services.auth.users import UserStorage
from services.stat.reacted import ReactedStorage
from services.stat.topicstat import TopicStat
from services.zine.authors import AuthorsStorage
from services.zine.shoutauthor import ShoutAuthorStorage
# from .community import followed_communities
@ -174,12 +175,9 @@ def author_unfollow(user, slug):
@query.field("authorsAll")
async def get_authors_all(_, _info):
users = await UserStorage.get_all_users()
authors = []
for author in users:
if ShoutAuthorStorage.shouts_by_author.get(author.slug):
author.stat = await get_author_stat(author.slug)
authors.append(author)
authors = await AuthorsStorage.get_all_authors()
for author in authors:
author.stat = await get_author_stat(author.slug)
return authors
@ -212,5 +210,5 @@ async def load_authors_by(_, info, by, limit, offset):
for a in authors:
a.stat = await get_author_stat(a.slug)
authors = list(set(authors))
authors = sorted(authors, key=lambda a: a["stat"].get(by.get("stat")))
# authors = sorted(authors, key=lambda a: a["stat"].get(by.get("stat")))
return authors

12
resolvers/reactions.py
View File

@ -217,13 +217,10 @@ async def load_reactions_by(_, info, by, limit=50, offset=0):
:return: Reaction[]
"""
q = select(Reaction).options(
selectinload(Reaction.shout),
q = select(Reaction).join(
Shout
).where(
Reaction.deletedAt.is_(None)
).join(
Shout,
Shout.slug == Reaction.shout
)
if by.get("slug"):
q = q.filter(Shout.slug == by["slug"])
@ -243,8 +240,9 @@ async def load_reactions_by(_, info, by, limit=50, offset=0):
if by.get("days"):
before = datetime.now() - timedelta(days=int(by["days"]) or 30)
q = q.filter(Reaction.createdAt > before)
q = q.group_by(Shout.id).order_by(
desc(by.get("order") or "createdAt")
q = q.group_by(Reaction.id).order_by(
desc(by.get("order") or Reaction.createdAt)
).limit(limit).offset(offset)
rrr = []

149
resolvers/zine.py
View File

@ -1,13 +1,13 @@
from datetime import datetime, timedelta
import sqlalchemy as sa
from sqlalchemy.orm import selectinload
from sqlalchemy.sql.expression import or_, desc, select
from sqlalchemy.sql.expression import or_, desc, asc, select, case
from timeit import default_timer as timer
from auth.authenticate import login_required
from base.orm import local_session
from base.resolvers import mutation, query
from orm.shout import Shout
from orm.reaction import Reaction
from orm.shout import Shout, ShoutAuthor
from orm.reaction import Reaction, ReactionKind
# from resolvers.community import community_follow, community_unfollow
from resolvers.profile import author_follow, author_unfollow
from resolvers.reactions import reactions_follow, reactions_unfollow
@ -16,74 +16,115 @@ from services.zine.shoutauthor import ShoutAuthorStorage
from services.stat.reacted import ReactedStorage
@query.field("loadShoutsBy")
async def load_shouts_by(_, info, by, limit=50, offset=0):
@query.field("loadShout")
async def load_shout(_, info, slug):
with local_session() as session:
shout = session.query(Shout).options(
# TODO add cation
selectinload(Shout.authors),
selectinload(Shout.topics),
).filter(
Shout.slug == slug
).filter(
Shout.deletedAt.is_(None)
).one()
return shout
@query.field("loadShouts")
async def load_shouts_by(_, info, options):
"""
:param by: {
layout: 'audio',
visibility: "public",
author: 'discours',
topic: 'culture',
title: 'something',
body: 'something else',
stat: 'rating' | 'comments' | 'reacted' | 'views',
days: 30
:param options: {
filters: {
layout: 'audio',
visibility: "public",
author: 'discours',
topic: 'culture',
title: 'something',
body: 'something else',
days: 30
}
offset: 0
limit: 50
order_by: 'createdAt'
order_by_desc: tr
}
:param limit: int amount of shouts
:param offset: int offset in this order
:return: Shout[]
"""
q = select(Shout, Reaction).options(
q = select(Shout).options(
# TODO add cation
selectinload(Shout.authors),
selectinload(Shout.topics),
selectinload(Shout.reactions)
).where(
Shout.deletedAt.is_(None)
).join(
Reaction, Reaction.shout == Shout.slug
)
if by.get("slug"):
q = q.filter(Shout.slug == by["slug"])
else:
if by.get("reacted"):
try:
user = info.context["request"].user
q = q.filter(Reaction.createdBy == user.slug)
except Exception:
pass
if by.get("visibility"):
if options.get("filters"):
if options.get("filters").get("reacted"):
user = info.context["request"].user
q.join(Reaction, Reaction.createdBy == user.slug)
if options.get("filters").get("visibility"):
q = q.filter(or_(
Shout.visibility.ilike(f"%{by.get('visibility')}%"),
Shout.visibility.ilike(f"%{options.get('filters').get('visibility')}%"),
Shout.visibility.ilike(f"%{'public'}%"),
))
if by.get("layout"):
q = q.filter(Shout.layout == by["layout"])
if by.get("author"):
q = q.filter(Shout.authors.contains(by["author"]))
if by.get("topic"):
q = q.filter(Shout.topics.contains(by["topic"]))
if by.get("title"):
q = q.filter(Shout.title.ilike(f'%{by["title"]}%'))
if by.get("body"):
q = q.filter(Shout.body.ilike(f'%{by["body"]}%'))
if by.get("days"):
before = datetime.now() - timedelta(days=int(by["days"]) or 30)
if options.get("filters").get("layout"):
q = q.filter(Shout.layout == options.get("filters").get("layout"))
if options.get("filters").get("author"):
q = q.filter(Shout.authors.any(slug=options.get("filters").get("author")))
if options.get("filters").get("topic"):
q = q.filter(Shout.topics.any(slug=options.get("filters").get("topic")))
if options.get("filters").get("title"):
q = q.filter(Shout.title.ilike(f'%{options.get("filters").get("title")}%'))
if options.get("filters").get("body"):
q = q.filter(Shout.body.ilike(f'%{options.get("filters").get("body")}%s'))
if options.get("filters").get("days"):
before = datetime.now() - timedelta(days=int(options.get("filter").get("days")) or 30)
q = q.filter(Shout.createdAt > before)
q = q.group_by(Shout.id, Reaction.id).order_by(
desc(by.get("order") or "createdAt")
).limit(limit).offset(offset)
print(q)
shouts = []
if options.get("order_by") == 'comments':
q = q.join(Reaction, Shout.slug == Reaction.shout and Reaction.body.is_not(None)).add_columns(
sa.func.count(Reaction.id).label(options.get("order_by")))
if options.get("order_by") == 'reacted':
q = q.join(Reaction).add_columns(sa.func.max(Reaction.createdAt).label(options.get("order_by")))
if options.get("order_by") == "rating":
q = q.join(Reaction).add_columns(sa.func.sum(case(
(Reaction.kind == ReactionKind.AGREE, 1),
(Reaction.kind == ReactionKind.DISAGREE, -1),
(Reaction.kind == ReactionKind.PROOF, 1),
(Reaction.kind == ReactionKind.DISPROOF, -1),
(Reaction.kind == ReactionKind.ACCEPT, 1),
(Reaction.kind == ReactionKind.REJECT, -1),
(Reaction.kind == ReactionKind.LIKE, 1),
(Reaction.kind == ReactionKind.DISLIKE, -1),
else_=0
)).label(options.get("order_by")))
# if order_by == 'views':
# TODO dump ackee data to db periodically
order_by = options.get("order_by") if options.get("order_by") else 'createdAt'
query_order_by = desc(order_by) if options.get('order_by_desc') else asc(order_by)
q = q.group_by(Shout.id).order_by(query_order_by).limit(options.get("limit")).offset(
options.get("offset") if options.get("offset") else 0)
with local_session() as session:
# post query stats and author's captions
for s in list(map(lambda r: r.Shout, session.execute(q))):
# start = timer()
shouts = list(map(lambda r: r.Shout, session.execute(q)))
for s in shouts:
s.stat = await ReactedStorage.get_shout_stat(s.slug)
for a in s.authors:
a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug)
shouts.append(s)
if by.get("stat"):
shouts.sort(lambda s: s.stat.get(by["stat"]) or s.createdAt)
# end = timer()
# print(end - start)
# print(q)
return shouts

51
schema.graphql
View File

@ -211,19 +211,37 @@ input AuthorsBy {
stat: String
}
input ShoutsBy {
slug: String
title: String
body: String
topic: String
topics: [String]
author: String
authors: [String]
layout: String
visibility: String
order: String
days: Int
stat: String
input ShoutsFilterBy {
slug: String
title: String
body: String
topic: String
topics: [String]
author: String
authors: [String]
layout: String
visibility: String
days: Int
stat: String
}
input LoadShoutsFilters {
title: String
body: String
topic: String
author: String
layout: String
visibility: String
days: Int
reacted: Boolean
}
input LoadShoutsOptions {
filters: LoadShoutsFilters
limit: Int!
offset: Int
order_by: String
order_by_desc: Boolean
}
input ReactionBy {
@ -252,13 +270,14 @@ type Query {
# zine
loadAuthorsBy(by: AuthorsBy, limit: Int, offset: Int): [Author]!
loadShoutsBy(by: ShoutsBy, limit: Int, offset: Int): [Shout]!
loadShout(slug: String!): Shout
loadShouts(options: LoadShoutsOptions): [Shout]!
loadReactionsBy(by: ReactionBy!, limit: Int, offset: Int): [Reaction]!
userFollowers(slug: String!): [Author]!
userFollowedAuthors(slug: String!): [Author]!
userFollowedTopics(slug: String!): [Topic]!
authorsAll: [Author]!
getAuthor(slug: String!): User!
getAuthor(slug: String!): User
# collab
getCollabs: [Collab]!
@ -267,7 +286,7 @@ type Query {
markdownBody(body: String!): String!
# topics
getTopic(slug: String!): Topic!
getTopic(slug: String!): Topic
topicsAll: [Topic]!
topicsRandom(amount: Int): [Topic]!
topicsByCommunity(community: String!): [Topic]!

12
services/zine/authors.py Normal file
View File

@ -0,0 +1,12 @@
from base.orm import local_session
from orm.user import User
from orm.shout import ShoutAuthor
class AuthorsStorage:
@staticmethod
async def get_all_authors():
with local_session() as session:
query = session.query(User).join(ShoutAuthor)
result = query.all()
return result

4
services/zine/shoutauthor.py
View File

@ -6,7 +6,6 @@ from orm.shout import ShoutAuthor, Shout
class ShoutAuthorStorage:
authors_by_shout = {}
shouts_by_author = {}
lock = asyncio.Lock()
period = 30 * 60 # sec
@ -17,10 +16,7 @@ class ShoutAuthorStorage:
for sa in sas:
self.authors_by_shout[sa.shout] = self.authors_by_shout.get(sa.shout, [])
self.authors_by_shout[sa.shout].append([sa.user, sa.caption])
self.shouts_by_author[sa.user] = self.shouts_by_author.get(sa.user, [])
self.shouts_by_author[sa.user].append(sa.shout)
print("[zine.authors] %d shouts indexed by authors" % len(self.authors_by_shout))
print("[zine.authors] %d authors indexed by shouts" % len(self.shouts_by_author))
@staticmethod
async def get_authors(shout):