discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

migration, auth, refactoring, formatting

Browse Source
This commit is contained in:
tonyrewin
2022-09-17 21:12:14 +03:00
parent 6b4c00d9e7
commit 3136eecd7e
68 changed files with 968 additions and 930 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
services/auth/roles.py
View File

@@ -1,5 +1,7 @@
import asyncio
from sqlalchemy.orm import selectinload
from orm.rbac import Role

2
services/auth/users.py
View File

@@ -1,5 +1,7 @@
import asyncio
from sqlalchemy.orm import selectinload
from orm.user import User

17
services/main.py Normal file
View File

@@ -0,0 +1,17 @@
from services.stat.viewed import ViewedStorage
from services.stat.reacted import ReactedStorage
from services.auth.roles import RoleStorage
from services.auth.users import UserStorage
from services.zine.topics import TopicStorage
from base.orm import local_session
async def storages_init():
with local_session() as session:
print('[main] initialize storages')
ViewedStorage.init(session)
ReactedStorage.init(session)
RoleStorage.init(session)
UserStorage.init(session)
TopicStorage.init(session)
session.commit()

58
services/stat/reacted.py
View File

@@ -1,11 +1,13 @@
import asyncio
from datetime import datetime
from enum import Enum as Enumeration
from sqlalchemy import Column, DateTime, ForeignKey, Boolean
from sqlalchemy.orm.attributes import flag_modified
from sqlalchemy.types import Enum as ColumnEnum
from base.orm import Base, local_session
from orm.topic import ShoutTopic
from enum import Enum as Enumeration
from sqlalchemy.types import Enum as ColumnEnum
class ReactionKind(Enumeration):
@@ -139,26 +141,23 @@ class ReactedStorage:
self = ReactedStorage
async with self.lock:
reactions = self.reacted["shouts"].get(reaction.shout)
if reaction.replyTo:
reactions = self.reacted["reactions"].get(reaction.id)
for r in reactions.values():
r = {
"day": datetime.now().replace(
hour=0, minute=0, second=0, microsecond=0
),
"reaction": reaction.id,
"kind": reaction.kind,
"shout": reaction.shout,
}
if reaction.replyTo:
r["replyTo"] = reaction.replyTo
if reaction.body:
r["comment"] = True
reaction: ReactedByDay = ReactedByDay.create(**r) # type: ignore
self.reacted["shouts"][reaction.shout] = self.reacted["shouts"].get(
reaction.shout, []
)
reactions = {}
# iterate sibling reactions
reactions = self.reacted["shouts"].get(reaction.shout, {})
for r in reactions.values():
reaction = ReactedByDay.create({
"day": datetime.now().replace(
hour=0, minute=0, second=0, microsecond=0
),
"reaction": r.id,
"kind": r.kind,
"shout": r.shout,
"comment": bool(r.body),
"replyTo": r.replyTo
})
# renew sorted by shouts store
self.reacted["shouts"][reaction.shout] = self.reacted["shouts"].get(reaction.shout, [])
self.reacted["shouts"][reaction.shout].append(reaction)
if reaction.replyTo:
self.reacted["reaction"][reaction.replyTo] = self.reacted[
@@ -169,11 +168,12 @@ class ReactedStorage:
"reactions"
].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind)
else:
# rate only by root reactions on shout
self.rating["shouts"][reaction.replyTo] = self.rating["shouts"].get(
reaction.shout, 0
) + kind_to_rate(reaction.kind)
flag_modified(r, "value")
flag_modified(reaction, "value")
@staticmethod
def init(session):
@@ -218,16 +218,20 @@ class ReactedStorage:
async def flush_changes(session):
self = ReactedStorage
async with self.lock:
for slug in dict(self.reacted['shouts']).keys():
topics = session.query(ShoutTopic.topic).where(ShoutTopic.shout == slug).all()
reactions = self.reacted['shouts'].get(slug, [])
for slug in dict(self.reacted["shouts"]).keys():
topics = (
session.query(ShoutTopic.topic)
.where(ShoutTopic.shout == slug)
.all()
)
reactions = self.reacted["shouts"].get(slug, [])
# print('[stat.reacted] shout {' + str(slug) + "}: " + str(len(reactions)))
for ts in list(topics):
tslug = ts[0]
topic_reactions = self.reacted["topics"].get(tslug, [])
topic_reactions += reactions
# print('[stat.reacted] topic {' + str(tslug) + "}: " + str(len(topic_reactions)))
reactions += list(self.reacted['reactions'].values())
reactions += list(self.reacted["reactions"].values())
for reaction in reactions:
if getattr(reaction, "modified", False):
session.add(reaction)

20
services/stat/topicstat.py
View File

@@ -1,19 +1,11 @@
import asyncio
from base.orm import local_session
from orm.shout import Shout
from orm.topic import ShoutTopic, TopicFollower
from services.stat.reacted import ReactedStorage
from services.stat.viewed import ViewedStorage
from services.zine.shoutauthor import ShoutAuthorStorage
from orm.topic import ShoutTopic, TopicFollower
def unique(list1):
# insert the list to the set
list_set = set(list1)
# convert the set to the list
unique_list = (list(list_set))
return unique_list
class TopicStat:
@@ -27,7 +19,7 @@ class TopicStat:
async def load_stat(session):
self = TopicStat
shout_topics = session.query(ShoutTopic).all()
print('[stat.topics] shout topics amount', len(shout_topics))
print("[stat.topics] shout topics amount", len(shout_topics))
for shout_topic in shout_topics:
# shouts by topics
@@ -35,7 +27,11 @@ class TopicStat:
shout = shout_topic.shout
sss = set(self.shouts_by_topic.get(topic, []))
shout = session.query(Shout).where(Shout.slug == shout).first()
sss.union([shout, ])
sss.union(
[
shout,
]
)
self.shouts_by_topic[topic] = list(sss)
# authors by topics

2
services/stat/viewed.py
View File

@@ -1,7 +1,9 @@
import asyncio
from datetime import datetime
from sqlalchemy import Column, DateTime, ForeignKey, Integer
from sqlalchemy.orm.attributes import flag_modified
from base.orm import Base, local_session
from orm.topic import ShoutTopic

3
services/zine/gittask.py
View File

@@ -1,6 +1,7 @@
import asyncio
import subprocess
from pathlib import Path
import asyncio
from settings import SHOUTS_REPO

1
services/zine/shoutauthor.py
View File

@@ -1,4 +1,5 @@
import asyncio
from base.orm import local_session
from orm.shout import ShoutAuthor

168
services/zine/shoutscache.py
View File

@@ -1,7 +1,9 @@
import asyncio
from datetime import datetime, timedelta
from sqlalchemy import and_, desc, func, select
from sqlalchemy.orm import selectinload
from base.orm import local_session
from orm.reaction import Reaction
from orm.shout import Shout, ShoutAuthor, ShoutTopic
@@ -27,6 +29,7 @@ class ShoutsCache:
top_month = []
top_overall = []
top_viewed = []
top_commented = []
by_author = {}
by_topic = {}
@@ -34,14 +37,17 @@ class ShoutsCache:
@staticmethod
async def prepare_recent_published():
with local_session() as session:
shouts = await prepare_shouts(session, (
select(Shout)
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.where(bool(Shout.publishedAt))
.group_by(Shout.slug)
.order_by(desc("publishedAt"))
.limit(ShoutsCache.limit)
))
shouts = await prepare_shouts(
session,
(
select(Shout)
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.where(bool(Shout.publishedAt))
.group_by(Shout.slug)
.order_by(desc("publishedAt"))
.limit(ShoutsCache.limit)
),
)
async with ShoutsCache.lock:
ShoutsCache.recent_published = shouts
print("[zine.cache] %d recently published shouts " % len(shouts))
@@ -49,14 +55,17 @@ class ShoutsCache:
@staticmethod
async def prepare_recent_all():
with local_session() as session:
shouts = await prepare_shouts(session, (
select(Shout)
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("createdAt"))
.limit(ShoutsCache.limit)
))
shouts = await prepare_shouts(
session,
(
select(Shout)
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("createdAt"))
.limit(ShoutsCache.limit)
),
)
async with ShoutsCache.lock:
ShoutsCache.recent_all = shouts
print("[zine.cache] %d recently created shouts " % len(shouts))
@@ -64,18 +73,23 @@ class ShoutsCache:
@staticmethod
async def prepare_recent_reacted():
with local_session() as session:
shouts = await prepare_shouts(session, (
select(Shout, func.max(Reaction.createdAt).label("reactionCreatedAt"))
.options(
selectinload(Shout.authors),
selectinload(Shout.topics),
)
.join(Reaction, Reaction.shout == Shout.slug)
.where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("reactionCreatedAt"))
.limit(ShoutsCache.limit)
))
shouts = await prepare_shouts(
session,
(
select(
Shout, func.max(Reaction.createdAt).label("reactionCreatedAt")
)
.options(
selectinload(Shout.authors),
selectinload(Shout.topics),
)
.join(Reaction, Reaction.shout == Shout.slug)
.where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("reactionCreatedAt"))
.limit(ShoutsCache.limit)
),
)
async with ShoutsCache.lock:
ShoutsCache.recent_reacted = shouts
print("[zine.cache] %d recently reacted shouts " % len(shouts))
@@ -84,20 +98,23 @@ class ShoutsCache:
async def prepare_top_overall():
with local_session() as session:
# with reacted times counter
shouts = await prepare_shouts(session, (
select(Shout, func.count(Reaction.id).label("reacted"))
.options(
selectinload(Shout.authors),
selectinload(Shout.topics),
selectinload(Shout.reactions),
)
.join(Reaction)
.where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("reacted"))
.limit(ShoutsCache.limit)
))
shouts.sort(key=lambda s: s.stats['rating'], reverse=True)
shouts = await prepare_shouts(
session,
(
select(Shout, func.count(Reaction.id).label("reacted"))
.options(
selectinload(Shout.authors),
selectinload(Shout.topics),
selectinload(Shout.reactions),
)
.join(Reaction)
.where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("reacted"))
.limit(ShoutsCache.limit)
),
)
shouts.sort(key=lambda s: s.stats["rating"], reverse=True)
async with ShoutsCache.lock:
print("[zine.cache] %d top shouts " % len(shouts))
ShoutsCache.top_overall = shouts
@@ -106,34 +123,61 @@ class ShoutsCache:
async def prepare_top_month():
month_ago = datetime.now() - timedelta(days=30)
with local_session() as session:
shouts = await prepare_shouts(session, (
select(Shout, func.count(Reaction.id).label("reacted"))
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.join(Reaction)
.where(and_(Shout.createdAt > month_ago, bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("reacted"))
.limit(ShoutsCache.limit)
))
shouts.sort(key=lambda s: s.stats['rating'], reverse=True)
shouts = await prepare_shouts(
session,
(
select(Shout, func.count(Reaction.id).label("reacted"))
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.join(Reaction)
.where(and_(Shout.createdAt > month_ago, bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("reacted"))
.limit(ShoutsCache.limit)
),
)
shouts.sort(key=lambda s: s.stats["rating"], reverse=True)
async with ShoutsCache.lock:
print("[zine.cache] %d top month shouts " % len(shouts))
ShoutsCache.top_month = shouts
@staticmethod
async def prepare_top_commented():
month_ago = datetime.now() - timedelta(days=30)
with local_session() as session:
shouts = await prepare_shouts(
session,
(
select(Shout, Reaction)
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.join(Reaction)
.where(and_(Shout.createdAt > month_ago, bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("commented"))
.limit(ShoutsCache.limit)
),
)
shouts.sort(key=lambda s: s.stats["commented"], reverse=True)
async with ShoutsCache.lock:
print("[zine.cache] %d top commented shouts " % len(shouts))
ShoutsCache.top_viewed = shouts
@staticmethod
async def prepare_top_viewed():
month_ago = datetime.now() - timedelta(days=30)
with local_session() as session:
shouts = await prepare_shouts(session, (
select(Shout, func.sum(ViewedByDay.value).label("viewed"))
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.join(ViewedByDay)
.where(and_(Shout.createdAt > month_ago, bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("viewed"))
.limit(ShoutsCache.limit)
))
shouts.sort(key=lambda s: s.stats['viewed'], reverse=True)
shouts = await prepare_shouts(
session,
(
select(Shout, func.sum(ViewedByDay.value).label("viewed"))
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.join(ViewedByDay)
.where(and_(Shout.createdAt > month_ago, bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("viewed"))
.limit(ShoutsCache.limit)
),
)
shouts.sort(key=lambda s: s.stats["viewed"], reverse=True)
async with ShoutsCache.lock:
print("[zine.cache] %d top viewed shouts " % len(shouts))
ShoutsCache.top_viewed = shouts