discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

-reactions.storage, +collectionShouts.query, fixes

Browse Source
This commit is contained in:
tonyrewin
2022-08-13 12:48:07 +03:00
parent 5859a4db40
commit f0b625af53
19 changed files with 277 additions and 381 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
services/zine/gittask.py
View File

@@ -53,10 +53,10 @@ class GitTask:
@staticmethod
async def git_task_worker():
print("[resolvers.git] worker start")
print("[service.git] worker start")
while True:
task = await GitTask.queue.get()
try:
task.execute()
except Exception as err:
print("[resolvers.git] worker error: %s" % (err))
print("[service.git] worker error: %s" % (err))

159
services/zine/reactions.py
View File

@@ -1,159 +0,0 @@
import asyncio
from sqlalchemy import and_, desc, func
from sqlalchemy.orm import joinedload
from base.orm import local_session
from orm.reaction import Reaction, ReactionKind
from orm.topic import ShoutTopic
def kind_to_rate(kind) -> int:
if kind in [
ReactionKind.AGREE,
ReactionKind.LIKE,
ReactionKind.PROOF,
ReactionKind.ACCEPT
]: return 1
elif kind in [
ReactionKind.DISAGREE,
ReactionKind.DISLIKE,
ReactionKind.DISPROOF,
ReactionKind.REJECT
]: return -1
else: return 0
class ReactionsStorage:
limit = 200
reactions = []
rating_by_shout = {}
reactions_by_shout = {}
reactions_by_topic = {} # TODO: get sum reactions for all shouts in topic
reactions_by_author = {}
lock = asyncio.Lock()
period = 3*60 # 3 mins
@staticmethod
async def prepare_all(session):
stmt = session.query(Reaction).\
filter(Reaction.deletedAt == None).\
order_by(desc("createdAt")).\
limit(ReactionsStorage.limit)
reactions = []
for row in session.execute(stmt):
reaction = row.Reaction
reactions.append(reaction)
reactions.sort(key=lambda x: x.createdAt, reverse=True)
async with ReactionsStorage.lock:
print("[service.reactions] %d recently published reactions " % len(reactions))
ReactionsStorage.reactions = reactions
@staticmethod
async def prepare_by_author(session):
try:
by_authors = session.query(Reaction.createdBy, func.count('*').label("count")).\
where(and_(Reaction.deletedAt == None)).\
group_by(Reaction.createdBy).all()
except Exception as e:
print(e)
by_authors = {}
async with ReactionsStorage.lock:
ReactionsStorage.reactions_by_author = dict([stat for stat in by_authors])
print("[service.reactions] %d reacted users" % len(by_authors))
@staticmethod
async def prepare_by_shout(session):
try:
by_shouts = session.query(Reaction.shout, func.count('*').label("count")).\
where(and_(Reaction.deletedAt == None)).\
group_by(Reaction.shout).all()
except Exception as e:
print(e)
by_shouts = {}
async with ReactionsStorage.lock:
ReactionsStorage.reactions_by_shout = dict([stat for stat in by_shouts])
print("[service.reactions] %d reacted shouts" % len(by_shouts))
@staticmethod
async def calc_ratings(session):
rating_by_shout = {}
for shout in ReactionsStorage.reactions_by_shout.keys():
rating_by_shout[shout] = 0
shout_reactions_by_kinds = session.query(Reaction).\
where(and_(Reaction.deletedAt == None, Reaction.shout == shout)).\
group_by(Reaction.kind, Reaction.id).all()
for reaction in shout_reactions_by_kinds:
rating_by_shout[shout] += kind_to_rate(reaction.kind)
async with ReactionsStorage.lock:
ReactionsStorage.rating_by_shout = rating_by_shout
@staticmethod
async def prepare_by_topic(session):
# TODO: optimize
by_topics = session.query(Reaction, func.count('*').label("count")).\
options(
joinedload(ShoutTopic),
joinedload(Reaction.shout)
).\
join(ShoutTopic, ShoutTopic.shout == Reaction.shout).\
filter(Reaction.deletedAt == None).\
group_by(ShoutTopic.topic).all()
reactions_by_topic = {}
for t, reactions in by_topics:
if not reactions_by_topic.get(t):
reactions_by_topic[t] = 0
for r in reactions:
reactions_by_topic[t] += r.count
async with ReactionsStorage.lock:
ReactionsStorage.reactions_by_topic = reactions_by_topic
@staticmethod
async def recent():
async with ReactionsStorage.lock:
return ReactionsStorage.reactions
@staticmethod
async def total():
async with ReactionsStorage.lock:
return len(ReactionsStorage.reactions)
@staticmethod
async def by_shout(shout):
async with ReactionsStorage.lock:
stat = ReactionsStorage.reactions_by_shout.get(shout)
stat = stat if stat else 0
return stat
@staticmethod
async def shout_rating(shout):
async with ReactionsStorage.lock:
return ReactionsStorage.rating_by_shout.get(shout)
@staticmethod
async def by_author(slug):
async with ReactionsStorage.lock:
stat = ReactionsStorage.reactions_by_author.get(slug)
stat = stat if stat else 0
return stat
@staticmethod
async def by_topic(topic):
async with ReactionsStorage.lock:
stat = ReactionsStorage.reactions_by_topic.get(topic)
stat = stat if stat else 0
return stat
@staticmethod
async def worker():
while True:
try:
with local_session() as session:
await ReactionsStorage.prepare_all(session)
print("[service.reactions] all reactions prepared")
await ReactionsStorage.prepare_by_shout(session)
print("[service.reactions] reactions by shouts prepared")
await ReactionsStorage.calc_ratings(session)
print("[service.reactions] reactions ratings prepared")
await ReactionsStorage.prepare_by_topic(session)
print("[service.reactions] reactions topics prepared")
except Exception as err:
print("[service.reactions] errror: %s" % (err))
await asyncio.sleep(ReactionsStorage.period)

7
services/zine/shoutauthor.py
View File

@@ -20,8 +20,7 @@ class ShoutAuthorStorage:
self.authors_by_shout[shout].append(user)
else:
self.authors_by_shout[shout] = [user]
print('[service.shoutauthor] %d authors ' % len(self.authors_by_shout))
# FIXME: [service.shoutauthor] 4251 authors
print('[zine.authors] %d shouts preprocessed' % len(self.authors_by_shout))
@staticmethod
async def get_authors(shout):
@@ -37,7 +36,7 @@ class ShoutAuthorStorage:
with local_session() as session:
async with self.lock:
await self.load(session)
print("[service.shoutauthor] updated")
print("[zine.authors] state updated")
except Exception as err:
print("[service.shoutauthor] errror: %s" % (err))
print("[zine.authors] errror: %s" % (err))
await asyncio.sleep(self.period)

33
services/zine/shoutscache.py
View File

@@ -2,12 +2,11 @@
import asyncio
from datetime import datetime, timedelta
from sqlalchemy import and_, desc, func, select
from sqlalchemy.orm import selectinload, joinedload
from sqlalchemy.orm import selectinload
from base.orm import local_session
from orm.reaction import Reaction
from orm.shout import Shout
from orm.topic import Topic
from services.zine.reactions import ReactionsStorage
from services.stat.reacted import ReactedStorage
from services.stat.viewed import ViewedByDay
@@ -27,11 +26,11 @@ class ShoutsCache:
shouts = []
for row in session.execute(stmt):
shout = row.Shout
shout.rating = await ReactionsStorage.shout_rating(shout.slug) or 0
shout.rating = await ReactedStorage.get_rating(shout.slug) or 0
shouts.append(shout)
async with ShoutsCache.lock:
ShoutsCache.recent_published = shouts
print("[service.shoutscache] %d recently published shouts " % len(shouts))
print("[zine.cache] %d recently published shouts " % len(shouts))
@staticmethod
async def prepare_recent_all():
@@ -47,11 +46,11 @@ class ShoutsCache:
for row in session.execute(stmt):
shout = row.Shout
# shout.topics = [t.slug for t in shout.topics]
shout.rating = await ReactionsStorage.shout_rating(shout.slug) or 0
shout.rating = await ReactedStorage.get_rating(shout.slug) or 0
shouts.append(shout)
async with ShoutsCache.lock:
ShoutsCache.recent_all = shouts
print("[service.shoutscache] %d recently created shouts " % len(shouts))
print("[zine.cache] %d recently created shouts " % len(shouts))
@staticmethod
async def prepare_recent_reacted():
@@ -70,11 +69,11 @@ class ShoutsCache:
for row in session.execute(stmt):
shout = row.Shout
# shout.topics = [t.slug for t in shout.topics]
shout.rating = await ReactionsStorage.shout_rating(shout.slug) or 0
shout.rating = await ReactedStorage.get_rating(shout.slug) or 0
shouts.append(shout)
async with ShoutsCache.lock:
ShoutsCache.recent_reacted = shouts
print("[service.shoutscache] %d recently reacted shouts " % len(shouts))
print("[zine.cache] %d recently reacted shouts " % len(shouts))
@staticmethod
@@ -93,11 +92,11 @@ class ShoutsCache:
# with rating synthetic counter
for row in session.execute(stmt):
shout = row.Shout
shout.rating = await ReactionsStorage.shout_rating(shout.slug) or 0
shout.rating = await ReactedStorage.get_rating(shout.slug) or 0
shouts.append(shout)
shouts.sort(key = lambda shout: shout.rating, reverse = True)
async with ShoutsCache.lock:
print("[service.shoutscache] %d top shouts " % len(shouts))
print("[zine.cache] %d top shouts " % len(shouts))
ShoutsCache.top_overall = shouts
@staticmethod
@@ -114,11 +113,11 @@ class ShoutsCache:
shouts = []
for row in session.execute(stmt):
shout = row.Shout
shout.rating = await ReactionsStorage.shout_rating(shout.slug) or 0
shout.rating = await ReactedStorage.get_rating(shout.slug) or 0
shouts.append(shout)
shouts.sort(key = lambda shout: shout.rating, reverse = True)
async with ShoutsCache.lock:
print("[service.shoutscache] %d top month shouts " % len(shouts))
print("[zine.cache] %d top month shouts " % len(shouts))
ShoutsCache.top_month = shouts
@staticmethod
@@ -135,11 +134,11 @@ class ShoutsCache:
shouts = []
for row in session.execute(stmt):
shout = row.Shout
shout.rating = await ReactionsStorage.shout_rating(shout.slug) or 0
shout.rating = await ReactedStorage.get_rating(shout.slug) or 0
shouts.append(shout)
# shouts.sort(key = lambda shout: shout.viewed, reverse = True)
async with ShoutsCache.lock:
print("[service.shoutscache] %d top viewed shouts " % len(shouts))
print("[zine.cache] %d top viewed shouts " % len(shouts))
ShoutsCache.top_viewed = shouts
@staticmethod
@@ -152,8 +151,8 @@ class ShoutsCache:
await ShoutsCache.prepare_recent_published()
await ShoutsCache.prepare_recent_all()
await ShoutsCache.prepare_recent_reacted()
print("[service.shoutscache] updated")
print("[zine.cache] updated")
except Exception as err:
print("[service.shoutscache] error: %s" % (err))
print("[zine.cache] error: %s" % (err))
raise err
await asyncio.sleep(ShoutsCache.period)

4
services/zine/topics.py
View File

@@ -12,9 +12,9 @@ class TopicStorage:
topics = session.query(Topic)
self.topics = dict([(topic.slug, topic) for topic in topics])
for topic in self.topics.values():
self.load_parents(topic) # TODO: test
self.load_parents(topic)
print('[service.topics] %d ' % len(self.topics.keys()))
print('[zine.topics] %d ' % len(self.topics.keys()))
@staticmethod
def load_parents(topic):