working-on
This commit is contained in:
126
services/stat/reacted.py
Normal file
126
services/stat/reacted.py
Normal file
@@ -0,0 +1,126 @@
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, DateTime, ForeignKey, Integer
|
||||
from sqlalchemy.orm.attributes import flag_modified
|
||||
from base.orm import Base, local_session
|
||||
from orm.reaction import ReactionKind
|
||||
|
||||
class ReactedByDay(Base):
|
||||
__tablename__ = "reacted_by_day"
|
||||
|
||||
id = None
|
||||
reaction = Column(ForeignKey("reaction.id"), primary_key = True)
|
||||
shout = Column(ForeignKey('shout.slug'), primary_key=True)
|
||||
reply = Column(ForeignKey('reaction.id'), primary_key=True, nullable=True)
|
||||
kind = Column(ReactionKind, primary_key=True)
|
||||
day = Column(DateTime, primary_key=True, default=datetime.now)
|
||||
|
||||
class ReactedStorage:
|
||||
reacted = {
|
||||
'shouts': {
|
||||
'total': 0,
|
||||
'today': 0,
|
||||
'month': 0,
|
||||
# TODO: need an opionated metrics list
|
||||
},
|
||||
'topics': {} # TODO: get sum reactions for all shouts in topic
|
||||
}
|
||||
this_day_reactions = {}
|
||||
to_flush = []
|
||||
period = 30*60 # sec
|
||||
lock = asyncio.Lock()
|
||||
|
||||
@staticmethod
|
||||
def init(session):
|
||||
self = ReactedStorage
|
||||
reactions = session.query(ReactedByDay).all()
|
||||
|
||||
for reaction in reactions:
|
||||
shout = reaction.shout
|
||||
value = reaction.value
|
||||
if shout:
|
||||
old_value = self.reacted['shouts'].get(shout, 0)
|
||||
self.reacted['shouts'][shout] = old_value + value
|
||||
if not shout in self.this_day_reactions:
|
||||
self.this_day_reactions[shout] = reaction
|
||||
this_day_reaction = self.this_day_reactions[shout]
|
||||
if this_day_reaction.day < reaction.day:
|
||||
self.this_day_reactions[shout] = reaction
|
||||
|
||||
print('[service.reacted] watching %d shouts' % len(reactions))
|
||||
# TODO: add reactions ?
|
||||
|
||||
@staticmethod
|
||||
async def get_shout(shout_slug):
|
||||
self = ReactedStorage
|
||||
async with self.lock:
|
||||
return self.reacted['shouts'].get(shout_slug, 0)
|
||||
|
||||
# NOTE: this method is never called
|
||||
@staticmethod
|
||||
async def get_reaction(reaction_id):
|
||||
self = ReactedStorage
|
||||
async with self.lock:
|
||||
return self.reacted['reactions'].get(reaction_id, 0)
|
||||
|
||||
@staticmethod
|
||||
async def inc_shout(shout_slug):
|
||||
self = ReactedStorage
|
||||
async with self.lock:
|
||||
this_day_reaction = self.this_day_reactions.get(shout_slug)
|
||||
day_start = datetime.now().replace(hour=0, minute=0, second=0)
|
||||
if not this_day_reaction or this_day_reaction.day < day_start:
|
||||
if this_day_reaction and getattr(this_day_reaction, "modified", False):
|
||||
self.to_flush.append(this_day_reaction)
|
||||
this_day_reaction = ReactedByDay.create(shout=shout_slug, value=1)
|
||||
self.this_day_reactions[shout_slug] = this_day_reaction
|
||||
else:
|
||||
this_day_reaction.value = this_day_reaction.value + 1
|
||||
this_day_reaction.modified = True
|
||||
old_value = self.reacted['shouts'].get(shout_slug, 0)
|
||||
self.reacted['shotus'][shout_slug] = old_value + 1
|
||||
|
||||
@staticmethod
|
||||
async def inc_reaction(shout_slug, reaction_id):
|
||||
self = ReactedStorage
|
||||
async with self.lock:
|
||||
this_day_reaction = self.this_day_reactions.get(reaction_id)
|
||||
day_start = datetime.now().replace(hour=0, minute=0, second=0)
|
||||
if not this_day_reaction or this_day_reaction.day < day_start:
|
||||
if this_day_reaction and getattr(this_day_reaction, "modified", False):
|
||||
self.to_flush.append(this_day_reaction)
|
||||
this_day_reaction = ReactedByDay.create(
|
||||
shout=shout_slug, reaction=reaction_id, value=1)
|
||||
self.this_day_reactions[shout_slug] = this_day_reaction
|
||||
else:
|
||||
this_day_reaction.value = this_day_reaction.value + 1
|
||||
this_day_reaction.modified = True
|
||||
old_value = self.reacted['shouts'].get(shout_slug, 0)
|
||||
self.reacted['shouts'][shout_slug] = old_value + 1
|
||||
old_value = self.reacted['reactions'].get(shout_slug, 0)
|
||||
self.reacted['reaction'][reaction_id] = old_value + 1
|
||||
|
||||
@staticmethod
|
||||
async def flush_changes(session):
|
||||
self = ReactedStorage
|
||||
async with self.lock:
|
||||
for reaction in self.this_day_reactions.values():
|
||||
if getattr(reaction, "modified", False):
|
||||
session.add(reaction)
|
||||
flag_modified(reaction, "value")
|
||||
reaction.modified = False
|
||||
for reaction in self.to_flush:
|
||||
session.add(reaction)
|
||||
self.to_flush.clear()
|
||||
session.commit()
|
||||
|
||||
@staticmethod
|
||||
async def worker():
|
||||
while True:
|
||||
try:
|
||||
with local_session() as session:
|
||||
await ReactedStorage.flush_changes(session)
|
||||
print("[service.reacted] service flushed changes")
|
||||
except Exception as err:
|
||||
print("[service.reacted] errror: %s" % (err))
|
||||
await asyncio.sleep(ReactedStorage.period)
|
||||
83
services/stat/topicstat.py
Normal file
83
services/stat/topicstat.py
Normal file
@@ -0,0 +1,83 @@
|
||||
import asyncio
|
||||
from base.orm import local_session
|
||||
from services.zine.shoutauthor import ShoutAuthorStorage
|
||||
from orm.topic import ShoutTopic, TopicFollower
|
||||
from typing import Dict
|
||||
|
||||
class TopicStat:
|
||||
shouts_by_topic = {}
|
||||
authors_by_topic = {}
|
||||
followers_by_topic = {}
|
||||
reactions_by_topic = {}
|
||||
lock = asyncio.Lock()
|
||||
period = 30*60 #sec
|
||||
|
||||
@staticmethod
|
||||
async def load_stat(session):
|
||||
self = TopicStat
|
||||
self.shouts_by_topic = {}
|
||||
self.authors_by_topic = {}
|
||||
shout_topics = session.query(ShoutTopic)
|
||||
for shout_topic in shout_topics:
|
||||
topic = shout_topic.topic
|
||||
shout = shout_topic.shout
|
||||
if topic in self.shouts_by_topic:
|
||||
self.shouts_by_topic[topic].append(shout)
|
||||
else:
|
||||
self.shouts_by_topic[topic] = [shout]
|
||||
|
||||
authors = await ShoutAuthorStorage.get_authors(shout)
|
||||
if topic in self.authors_by_topic:
|
||||
self.authors_by_topic[topic].update(authors)
|
||||
else:
|
||||
self.authors_by_topic[topic] = set(authors)
|
||||
|
||||
print('[service.topicstat] authors sorted')
|
||||
print('[service.topicstat] shouts sorted')
|
||||
|
||||
self.followers_by_topic = {}
|
||||
followings = session.query(TopicFollower)
|
||||
for flw in followings:
|
||||
topic = flw.topic
|
||||
user = flw.follower
|
||||
if topic in self.followers_by_topic:
|
||||
self.followers_by_topic[topic].append(user)
|
||||
else:
|
||||
self.followers_by_topic[topic] = [user]
|
||||
print('[service.topicstat] followers sorted')
|
||||
|
||||
@staticmethod
|
||||
async def get_shouts(topic):
|
||||
self = TopicStat
|
||||
async with self.lock:
|
||||
return self.shouts_by_topic.get(topic, [])
|
||||
|
||||
@staticmethod
|
||||
async def get_stat(topic):
|
||||
self = TopicStat
|
||||
async with self.lock:
|
||||
shouts = self.shouts_by_topic.get(topic, [])
|
||||
followers = self.followers_by_topic.get(topic, [])
|
||||
authors = self.authors_by_topic.get(topic, [])
|
||||
reactions = self.reactions_by_topic.get(topic, [])
|
||||
|
||||
return {
|
||||
"shouts" : len(shouts),
|
||||
"authors" : len(authors),
|
||||
"followers" : len(followers),
|
||||
"reactions" : len(reactions)
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
async def worker():
|
||||
self = TopicStat
|
||||
while True:
|
||||
try:
|
||||
with local_session() as session:
|
||||
async with self.lock:
|
||||
await self.load_stat(session)
|
||||
print("[service.topicstat] updated")
|
||||
except Exception as err:
|
||||
print("[service.topicstat] errror: %s" % (err))
|
||||
await asyncio.sleep(self.period)
|
||||
|
||||
121
services/stat/viewed.py
Normal file
121
services/stat/viewed.py
Normal file
@@ -0,0 +1,121 @@
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, DateTime, ForeignKey, Integer
|
||||
from sqlalchemy.orm.attributes import flag_modified
|
||||
from base.orm import Base, local_session
|
||||
|
||||
|
||||
class ViewedByDay(Base):
|
||||
__tablename__ = "viewed_by_day"
|
||||
|
||||
id = None
|
||||
shout = Column(ForeignKey('shout.slug'), primary_key=True)
|
||||
day = Column(DateTime, primary_key=True, default=datetime.now)
|
||||
value = Column(Integer)
|
||||
|
||||
|
||||
class ViewedStorage:
|
||||
viewed = {
|
||||
'shouts': {},
|
||||
# TODO: ? 'reactions': {},
|
||||
'topics': {} # TODO: get sum views for all shouts in topic
|
||||
}
|
||||
this_day_views = {}
|
||||
to_flush = []
|
||||
period = 30*60 # sec
|
||||
lock = asyncio.Lock()
|
||||
|
||||
@staticmethod
|
||||
def init(session):
|
||||
self = ViewedStorage
|
||||
views = session.query(ViewedByDay).all()
|
||||
|
||||
for view in views:
|
||||
shout = view.shout
|
||||
value = view.value
|
||||
if shout:
|
||||
old_value = self.viewed['shouts'].get(shout, 0)
|
||||
self.viewed['shouts'][shout] = old_value + value
|
||||
if not shout in self.this_day_views:
|
||||
self.this_day_views[shout] = view
|
||||
this_day_view = self.this_day_views[shout]
|
||||
if this_day_view.day < view.day:
|
||||
self.this_day_views[shout] = view
|
||||
|
||||
print('[service.viewed] watching %d shouts' % len(views))
|
||||
# TODO: add reactions ?
|
||||
|
||||
@staticmethod
|
||||
async def get_shout(shout_slug):
|
||||
self = ViewedStorage
|
||||
async with self.lock:
|
||||
return self.viewed['shouts'].get(shout_slug, 0)
|
||||
|
||||
# NOTE: this method is never called
|
||||
@staticmethod
|
||||
async def get_reaction(reaction_id):
|
||||
self = ViewedStorage
|
||||
async with self.lock:
|
||||
return self.viewed['reactions'].get(reaction_id, 0)
|
||||
|
||||
@staticmethod
|
||||
async def inc_shout(shout_slug):
|
||||
self = ViewedStorage
|
||||
async with self.lock:
|
||||
this_day_view = self.this_day_views.get(shout_slug)
|
||||
day_start = datetime.now().replace(hour=0, minute=0, second=0)
|
||||
if not this_day_view or this_day_view.day < day_start:
|
||||
if this_day_view and getattr(this_day_view, "modified", False):
|
||||
self.to_flush.append(this_day_view)
|
||||
this_day_view = ViewedByDay.create(shout=shout_slug, value=1)
|
||||
self.this_day_views[shout_slug] = this_day_view
|
||||
else:
|
||||
this_day_view.value = this_day_view.value + 1
|
||||
this_day_view.modified = True
|
||||
old_value = self.viewed['shouts'].get(shout_slug, 0)
|
||||
self.viewed['shotus'][shout_slug] = old_value + 1
|
||||
|
||||
@staticmethod
|
||||
async def inc_reaction(shout_slug, reaction_id):
|
||||
self = ViewedStorage
|
||||
async with self.lock:
|
||||
this_day_view = self.this_day_views.get(reaction_id)
|
||||
day_start = datetime.now().replace(hour=0, minute=0, second=0)
|
||||
if not this_day_view or this_day_view.day < day_start:
|
||||
if this_day_view and getattr(this_day_view, "modified", False):
|
||||
self.to_flush.append(this_day_view)
|
||||
this_day_view = ViewedByDay.create(
|
||||
shout=shout_slug, reaction=reaction_id, value=1)
|
||||
self.this_day_views[shout_slug] = this_day_view
|
||||
else:
|
||||
this_day_view.value = this_day_view.value + 1
|
||||
this_day_view.modified = True
|
||||
old_value = self.viewed['shouts'].get(shout_slug, 0)
|
||||
self.viewed['shouts'][shout_slug] = old_value + 1
|
||||
old_value = self.viewed['reactions'].get(shout_slug, 0)
|
||||
self.viewed['reaction'][reaction_id] = old_value + 1
|
||||
|
||||
@staticmethod
|
||||
async def flush_changes(session):
|
||||
self = ViewedStorage
|
||||
async with self.lock:
|
||||
for view in self.this_day_views.values():
|
||||
if getattr(view, "modified", False):
|
||||
session.add(view)
|
||||
flag_modified(view, "value")
|
||||
view.modified = False
|
||||
for view in self.to_flush:
|
||||
session.add(view)
|
||||
self.to_flush.clear()
|
||||
session.commit()
|
||||
|
||||
@staticmethod
|
||||
async def worker():
|
||||
while True:
|
||||
try:
|
||||
with local_session() as session:
|
||||
await ViewedStorage.flush_changes(session)
|
||||
print("[service.viewed] service flushed changes")
|
||||
except Exception as err:
|
||||
print("[service.viewed] errror: %s" % (err))
|
||||
await asyncio.sleep(ViewedStorage.period)
|
||||
Reference in New Issue
Block a user