working-on

This commit is contained in:
tonyrewin 2022-08-11 12:09:57 +03:00
parent 8aec6c6e07
commit 83f5f280b2
30 changed files with 229 additions and 86 deletions

View File

@ -10,7 +10,7 @@ from auth.jwtcodec import JWTCodec
from auth.authorize import Authorize, TokenStorage
from base.exceptions import InvalidToken
from orm.user import User
from storages.users import UserStorage
from services.auth.users import UserStorage
from base.orm import local_session
from settings import JWT_AUTH_HEADER, EMAIL_TOKEN_LIFE_SPAN

View File

@ -6,7 +6,7 @@ from settings import BACKEND_URL, MAILGUN_API_KEY, MAILGUN_DOMAIN, RESET_PWD_URL
CONFIRM_EMAIL_URL, ERROR_URL_ON_FRONTEND
MAILGUN_API_URL = "https://api.mailgun.net/v3/%s/messages" % (MAILGUN_DOMAIN)
MAILGUN_FROM = "postmaster <postmaster@%s>" % (MAILGUN_DOMAIN)
MAILGUN_FROM = "discours.io <noreply@%s>" % (MAILGUN_DOMAIN)
AUTH_URL = "%s/email_authorize" % (BACKEND_URL)
@ -14,7 +14,7 @@ email_templates = {"confirm_email" : "", "auth_email" : "", "reset_password_emai
def load_email_templates():
for name in email_templates:
filename = "templates/%s.tmpl" % name
filename = "auth/templates/%s.tmpl" % name # TODO: check path
with open(filename) as f:
email_templates[name] = f.read()
print("[auth.email] templates loaded")

View File

@ -1,10 +1,8 @@
from typing import TypeVar, Any, Dict, Generic, Callable
from sqlalchemy import create_engine, Column, Integer
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session
from sqlalchemy.sql.schema import Table
from settings import DB_URL
if DB_URL.startswith('sqlite'):

View File

@ -1,7 +1,6 @@
import aioredis
from settings import REDIS_URL
class Redis:
def __init__(self, uri=REDIS_URL):
self._uri: str = uri
@ -29,29 +28,6 @@ class Redis:
return await self._instance.mget(key, *keys)
async def test():
redis = Redis()
from datetime import datetime
await redis.connect()
await redis.execute("SET", "1-KEY1", 1)
await redis.execute("SET", "1-KEY2", 1)
await redis.execute("SET", "1-KEY3", 1)
await redis.execute("SET", "1-KEY4", 1)
await redis.execute("EXPIREAT", "1-KEY4", int(datetime.utcnow().timestamp()))
v = await redis.execute("KEYS", "1-*")
print(v)
await redis.execute("DEL", *v)
v = await redis.execute("KEYS", "1-*")
print(v)
if __name__ == '__main__':
import asyncio
asyncio.run(test())
redis = Redis()
__all__ = ['redis']

12
main.py
View File

@ -12,11 +12,11 @@ from auth.email import email_authorize
from base.redis import redis
from base.resolvers import resolvers
from resolvers.zine import ShoutsCache
from storages.viewed import ViewedStorage
# from storages.gittask import GitTask
from storages.topicstat import TopicStat
from storages.shoutauthor import ShoutAuthorStorage
from storages.reactions import ReactionsStorage
from services.stat.viewed import ViewedStorage
from services.zine.gittask import GitTask
from services.stat.topicstat import TopicStat
from services.zine.shoutauthor import ShoutAuthorStorage
from services.zine.reactions import ReactionsStorage
import asyncio
import_module('resolvers')
@ -34,7 +34,7 @@ async def start_up():
reaction_stat_task = asyncio.create_task(ReactionsStorage.worker())
shout_author_task = asyncio.create_task(ShoutAuthorStorage.worker())
topic_stat_task = asyncio.create_task(TopicStat.worker())
# FIXME git_task = asyncio.create_task(GitTask.git_task_worker())
git_task = asyncio.create_task(GitTask.git_task_worker())
async def shutdown():
await redis.disconnect()

View File

@ -1,14 +1,14 @@
from orm.rbac import Operation, Resource, Permission, Role
from storages.roles import RoleStorage
from services.auth.roles import RoleStorage
from orm.community import Community
from orm.user import User, UserRating
from orm.topic import Topic, TopicFollower
from orm.notification import Notification
from orm.shout import Shout
from orm.reaction import Reaction
from storages.topics import TopicStorage
from storages.users import UserStorage
from storages.viewed import ViewedStorage
from services.zine.topics import TopicStorage
from services.auth.users import UserStorage
from services.stat.viewed import ViewedStorage
from base.orm import Base, engine, local_session
__all__ = ["User", "Role", "Operation", "Permission", \

20
orm/collection.py Normal file
View File

@ -0,0 +1,20 @@
from datetime import datetime
from sqlalchemy import Column, String, ForeignKey, DateTime, JSON as JSONType
from base.orm import Base
class ShoutCollection(Base):
__tablename__ = 'shout_collection'
id = None
shout = Column(ForeignKey('shout.slug'), primary_key = True)
collection = Column(ForeignKey('collection.slug'), primary_key = True)
class Collection(Base):
__tablename__ = 'collection'
id = None
slug: str = Column(String, primary_key = True)
title: str = Column(String, nullable=False, comment="Title")
body: str = Column(String, nullable=True, comment="Body")
pic: str = Column(String, nullable=True, comment="Picture")

View File

@ -3,7 +3,7 @@ from sqlalchemy import Column, String, ForeignKey, DateTime
from base.orm import Base, local_session
import enum
from sqlalchemy import Enum
from storages.viewed import ViewedStorage
from services.stat.viewed import ViewedStorage
class ReactionKind(enum.Enum):
AGREE = 1 # +1

View File

@ -4,8 +4,8 @@ from sqlalchemy.orm import relationship
from orm.user import User
from orm.topic import Topic, ShoutTopic
from orm.reaction import Reaction
from storages.reactions import ReactionsStorage
from storages.viewed import ViewedStorage
from services.zine.reactions import ReactionsStorage
from services.stat.viewed import ViewedStorage
from base.orm import Base

View File

@ -3,7 +3,7 @@ from sqlalchemy import Column, Integer, String, ForeignKey, Boolean, DateTime, J
from sqlalchemy.orm import relationship
from base.orm import Base, local_session
from orm.rbac import Role
from storages.roles import RoleStorage
from services.auth.roles import RoleStorage
class UserNotifications(Base):
__tablename__ = 'user_notifications'

View File

@ -7,7 +7,7 @@ from base.resolvers import mutation
from resolvers.reactions import reactions_follow, reactions_unfollow
from auth.authenticate import login_required
from datetime import datetime
from storages.gittask import GitTask
from services.zine.gittask import GitTask
@mutation.field("createShout")

View File

@ -1,7 +1,7 @@
from auth.authenticate import login_required
from base.orm import local_session
from sqlalchemy import and_, desc, query
from orm.reaction import Reaction
from base.resolvers import query
from sqlalchemy import and_, desc
from orm.shout import Shout, ShoutAuthor, ShoutTopic
from orm.topic import TopicFollower
from orm.user import AuthorFollower

View File

@ -1,5 +1,5 @@
from orm.user import User, UserRole, Role, UserRating, AuthorFollower
from storages.users import UserStorage
from services.auth.users import UserStorage
from orm.shout import Shout
from orm.reaction import Reaction
from base.orm import local_session

View File

@ -5,8 +5,8 @@ from orm.user import User
from base.resolvers import mutation, query
from auth.authenticate import login_required
from datetime import datetime
from storages.reactions import ReactionsStorage
from storages.viewed import ViewedStorage
from services.zine.reactions import ReactionsStorage
from services.stat.viewed import ViewedStorage
def reactions_follow(user, slug, auto=False):
with local_session() as session:

View File

@ -1,8 +1,8 @@
from orm.topic import Topic, TopicFollower
from storages.topics import TopicStorage
from services.zine.topics import TopicStorage
from orm.shout import Shout
from orm.user import User
from storages.topicstat import TopicStat
from services.stat.topicstat import TopicStat
from base.orm import local_session
from base.resolvers import mutation, query
from auth.authenticate import login_required

View File

@ -2,8 +2,8 @@ from orm.shout import Shout, ShoutAuthor, ShoutTopic
from orm.topic import Topic
from base.orm import local_session
from base.resolvers import mutation, query
from storages.shoutscache import ShoutsCache
from storages.viewed import ViewedStorage
from services.zine.shoutscache import ShoutsCache
from services.stat.viewed import ViewedStorage
from resolvers.profile import author_follow, author_unfollow
from resolvers.topics import topic_follow, topic_unfollow
from resolvers.community import community_follow, community_unfollow

View File

@ -97,6 +97,12 @@ input CommunityInput {
pic: String
}
input CollectionInput {
title: String!
desc: String
pic: String
}
input TopicInput {
slug: String!
community: String!
@ -169,6 +175,11 @@ type Mutation {
updateCommunity(community: CommunityInput!): Result!
deleteCommunity(slug: String!): Result!
# collection
createCollection(collection: CollectionInput!): Result!
updateCollection(collection: CollectionInput!): Result!
deleteCollection(slug: String!): Result!
# collab
inviteAuthor(author: String!, shout: String!): Result!
removeAuthor(author: String!, shout: String!): Result!
@ -239,6 +250,9 @@ type Query {
topicsByCommunity(community: String!): [Topic]!
topicsByAuthor(author: String!): [Topic]!
# collection
getCollection(author: String!, slug: String!): Collection!
# communities
getCommunity(slug: String): Community!
getCommunities: [Community]! # all
@ -404,6 +418,15 @@ type Community {
createdBy: User!
}
type Collection {
slug: String!
title: String!
desc: String
pic: String!
createdAt: DateTime!
createdBy: User!
}
type TopicStat {
shouts: Int!
followers: Int!

View File

@ -13,7 +13,7 @@ class RoleStorage:
roles = session.query(Role).\
options(selectinload(Role.permissions)).all()
self.roles = dict([(role.id, role) for role in roles])
print('[storage.roles] %d ' % len(roles))
print('[service.auth] %d roles' % len(roles))
@staticmethod

View File

@ -14,7 +14,7 @@ class UserStorage:
options(selectinload(User.roles), selectinload(User.ratings)).all()
# TODO: add shouts and reactions counters
self.users = dict([(user.id, user) for user in users])
print('[storage.users] %d ' % len(self.users))
print('[service.auth] %d users' % len(self.users))
@staticmethod
async def get_user(id):

126
services/stat/reacted.py Normal file
View File

@ -0,0 +1,126 @@
import asyncio
from datetime import datetime
from sqlalchemy import Column, DateTime, ForeignKey, Integer
from sqlalchemy.orm.attributes import flag_modified
from base.orm import Base, local_session
from orm.reaction import ReactionKind
class ReactedByDay(Base):
__tablename__ = "reacted_by_day"
id = None
reaction = Column(ForeignKey("reaction.id"), primary_key = True)
shout = Column(ForeignKey('shout.slug'), primary_key=True)
reply = Column(ForeignKey('reaction.id'), primary_key=True, nullable=True)
kind = Column(ReactionKind, primary_key=True)
day = Column(DateTime, primary_key=True, default=datetime.now)
class ReactedStorage:
reacted = {
'shouts': {
'total': 0,
'today': 0,
'month': 0,
# TODO: need an opionated metrics list
},
'topics': {} # TODO: get sum reactions for all shouts in topic
}
this_day_reactions = {}
to_flush = []
period = 30*60 # sec
lock = asyncio.Lock()
@staticmethod
def init(session):
self = ReactedStorage
reactions = session.query(ReactedByDay).all()
for reaction in reactions:
shout = reaction.shout
value = reaction.value
if shout:
old_value = self.reacted['shouts'].get(shout, 0)
self.reacted['shouts'][shout] = old_value + value
if not shout in self.this_day_reactions:
self.this_day_reactions[shout] = reaction
this_day_reaction = self.this_day_reactions[shout]
if this_day_reaction.day < reaction.day:
self.this_day_reactions[shout] = reaction
print('[service.reacted] watching %d shouts' % len(reactions))
# TODO: add reactions ?
@staticmethod
async def get_shout(shout_slug):
self = ReactedStorage
async with self.lock:
return self.reacted['shouts'].get(shout_slug, 0)
# NOTE: this method is never called
@staticmethod
async def get_reaction(reaction_id):
self = ReactedStorage
async with self.lock:
return self.reacted['reactions'].get(reaction_id, 0)
@staticmethod
async def inc_shout(shout_slug):
self = ReactedStorage
async with self.lock:
this_day_reaction = self.this_day_reactions.get(shout_slug)
day_start = datetime.now().replace(hour=0, minute=0, second=0)
if not this_day_reaction or this_day_reaction.day < day_start:
if this_day_reaction and getattr(this_day_reaction, "modified", False):
self.to_flush.append(this_day_reaction)
this_day_reaction = ReactedByDay.create(shout=shout_slug, value=1)
self.this_day_reactions[shout_slug] = this_day_reaction
else:
this_day_reaction.value = this_day_reaction.value + 1
this_day_reaction.modified = True
old_value = self.reacted['shouts'].get(shout_slug, 0)
self.reacted['shotus'][shout_slug] = old_value + 1
@staticmethod
async def inc_reaction(shout_slug, reaction_id):
self = ReactedStorage
async with self.lock:
this_day_reaction = self.this_day_reactions.get(reaction_id)
day_start = datetime.now().replace(hour=0, minute=0, second=0)
if not this_day_reaction or this_day_reaction.day < day_start:
if this_day_reaction and getattr(this_day_reaction, "modified", False):
self.to_flush.append(this_day_reaction)
this_day_reaction = ReactedByDay.create(
shout=shout_slug, reaction=reaction_id, value=1)
self.this_day_reactions[shout_slug] = this_day_reaction
else:
this_day_reaction.value = this_day_reaction.value + 1
this_day_reaction.modified = True
old_value = self.reacted['shouts'].get(shout_slug, 0)
self.reacted['shouts'][shout_slug] = old_value + 1
old_value = self.reacted['reactions'].get(shout_slug, 0)
self.reacted['reaction'][reaction_id] = old_value + 1
@staticmethod
async def flush_changes(session):
self = ReactedStorage
async with self.lock:
for reaction in self.this_day_reactions.values():
if getattr(reaction, "modified", False):
session.add(reaction)
flag_modified(reaction, "value")
reaction.modified = False
for reaction in self.to_flush:
session.add(reaction)
self.to_flush.clear()
session.commit()
@staticmethod
async def worker():
while True:
try:
with local_session() as session:
await ReactedStorage.flush_changes(session)
print("[service.reacted] service flushed changes")
except Exception as err:
print("[service.reacted] errror: %s" % (err))
await asyncio.sleep(ReactedStorage.period)

View File

@ -1,6 +1,6 @@
import asyncio
from base.orm import local_session
from storages.shoutauthor import ShoutAuthorStorage
from services.zine.shoutauthor import ShoutAuthorStorage
from orm.topic import ShoutTopic, TopicFollower
from typing import Dict
@ -32,8 +32,8 @@ class TopicStat:
else:
self.authors_by_topic[topic] = set(authors)
print('[storage.topicstat] authors sorted')
print('[storage.topicstat] shouts sorted')
print('[service.topicstat] authors sorted')
print('[service.topicstat] shouts sorted')
self.followers_by_topic = {}
followings = session.query(TopicFollower)
@ -44,7 +44,7 @@ class TopicStat:
self.followers_by_topic[topic].append(user)
else:
self.followers_by_topic[topic] = [user]
print('[storage.topicstat] followers sorted')
print('[service.topicstat] followers sorted')
@staticmethod
async def get_shouts(topic):
@ -76,8 +76,8 @@ class TopicStat:
with local_session() as session:
async with self.lock:
await self.load_stat(session)
print("[storage.topicstat] updated")
print("[service.topicstat] updated")
except Exception as err:
print("[storage.topicstat] errror: %s" % (err))
print("[service.topicstat] errror: %s" % (err))
await asyncio.sleep(self.period)

View File

@ -42,7 +42,7 @@ class ViewedStorage:
if this_day_view.day < view.day:
self.this_day_views[shout] = view
print('[storage.viewed] watching %d shouts' % len(views))
print('[service.viewed] watching %d shouts' % len(views))
# TODO: add reactions ?
@staticmethod
@ -115,7 +115,7 @@ class ViewedStorage:
try:
with local_session() as session:
await ViewedStorage.flush_changes(session)
print("[storage.viewed] storage flushed changes")
print("[service.viewed] service flushed changes")
except Exception as err:
print("[storage.viewed] errror: %s" % (err))
print("[service.viewed] errror: %s" % (err))
await asyncio.sleep(ViewedStorage.period)

View File

@ -43,7 +43,7 @@ class ReactionsStorage:
reactions.append(reaction)
reactions.sort(key=lambda x: x.createdAt, reverse=True)
async with ReactionsStorage.lock:
print("[storage.reactions] %d recently published reactions " % len(reactions))
print("[service.reactions] %d recently published reactions " % len(reactions))
ReactionsStorage.reactions = reactions
@staticmethod
@ -57,7 +57,7 @@ class ReactionsStorage:
by_authors = {}
async with ReactionsStorage.lock:
ReactionsStorage.reactions_by_author = dict([stat for stat in by_authors])
print("[storage.reactions] %d reacted users" % len(by_authors))
print("[service.reactions] %d reacted users" % len(by_authors))
@staticmethod
async def prepare_by_shout(session):
@ -70,7 +70,7 @@ class ReactionsStorage:
by_shouts = {}
async with ReactionsStorage.lock:
ReactionsStorage.reactions_by_shout = dict([stat for stat in by_shouts])
print("[storage.reactions] %d reacted shouts" % len(by_shouts))
print("[service.reactions] %d reacted shouts" % len(by_shouts))
@staticmethod
async def calc_ratings(session):
@ -147,13 +147,13 @@ class ReactionsStorage:
try:
with local_session() as session:
await ReactionsStorage.prepare_all(session)
print("[storage.reactions] all reactions prepared")
print("[service.reactions] all reactions prepared")
await ReactionsStorage.prepare_by_shout(session)
print("[storage.reactions] reactions by shouts prepared")
print("[service.reactions] reactions by shouts prepared")
await ReactionsStorage.calc_ratings(session)
print("[storage.reactions] reactions ratings prepared")
print("[service.reactions] reactions ratings prepared")
await ReactionsStorage.prepare_by_topic(session)
print("[storage.reactions] reactions topics prepared")
print("[service.reactions] reactions topics prepared")
except Exception as err:
print("[storage.reactions] errror: %s" % (err))
print("[service.reactions] errror: %s" % (err))
await asyncio.sleep(ReactionsStorage.period)

View File

@ -20,7 +20,7 @@ class ShoutAuthorStorage:
self.authors_by_shout[shout].append(user)
else:
self.authors_by_shout[shout] = [user]
print('[storage.shoutauthor] %d shouts ' % len(self.authors_by_shout))
print('[service.shoutauthor] %d shouts ' % len(self.authors_by_shout))
@staticmethod
async def get_authors(shout):
@ -36,7 +36,7 @@ class ShoutAuthorStorage:
with local_session() as session:
async with self.lock:
await self.load(session)
print("[storage.shoutauthor] updated")
print("[service.shoutauthor] updated")
except Exception as err:
print("[storage.shoutauthor] errror: %s" % (err))
print("[service.shoutauthor] errror: %s" % (err))
await asyncio.sleep(self.period)

View File

@ -6,8 +6,8 @@ from sqlalchemy.orm import selectinload
from base.orm import local_session
from orm.reaction import Reaction
from orm.shout import Shout
from storages.reactions import ReactionsStorage
from storages.viewed import ViewedByDay
from services.zine.reactions import ReactionsStorage
from services.stat.viewed import ViewedByDay
class ShoutsCache:
@ -30,7 +30,7 @@ class ShoutsCache:
shouts.append(shout)
async with ShoutsCache.lock:
ShoutsCache.recent_published = shouts
print("[storage.shoutscache] %d recently published shouts " % len(shouts))
print("[service.shoutscache] %d recently published shouts " % len(shouts))
@staticmethod
async def prepare_recent_all():
@ -46,7 +46,7 @@ class ShoutsCache:
shouts.append(shout)
async with ShoutsCache.lock:
ShoutsCache.recent_all = shouts
print("[storage.shoutscache] %d recently created shouts " % len(shouts))
print("[service.shoutscache] %d recently created shouts " % len(shouts))
@staticmethod
async def prepare_recent_reacted():
@ -68,7 +68,7 @@ class ShoutsCache:
shouts.append(shout)
async with ShoutsCache.lock:
ShoutsCache.recent_reacted = shouts
print("[storage.shoutscache] %d recently reacted shouts " % len(shouts))
print("[service.shoutscache] %d recently reacted shouts " % len(shouts))
@staticmethod
@ -91,7 +91,7 @@ class ShoutsCache:
shouts.append(shout)
shouts.sort(key = lambda shout: shout.rating, reverse = True)
async with ShoutsCache.lock:
print("[storage.shoutscache] %d top shouts " % len(shouts))
print("[service.shoutscache] %d top shouts " % len(shouts))
ShoutsCache.top_overall = shouts
@staticmethod
@ -112,7 +112,7 @@ class ShoutsCache:
shouts.append(shout)
shouts.sort(key = lambda shout: shout.rating, reverse = True)
async with ShoutsCache.lock:
print("[storage.shoutscache] %d top month shouts " % len(shouts))
print("[service.shoutscache] %d top month shouts " % len(shouts))
ShoutsCache.top_month = shouts
@staticmethod
@ -133,7 +133,7 @@ class ShoutsCache:
shouts.append(shout)
# shouts.sort(key = lambda shout: shout.viewed, reverse = True)
async with ShoutsCache.lock:
print("[storage.shoutscache] %d top viewed shouts " % len(shouts))
print("[service.shoutscache] %d top viewed shouts " % len(shouts))
ShoutsCache.top_viewed = shouts
@staticmethod
@ -146,8 +146,8 @@ class ShoutsCache:
await ShoutsCache.prepare_recent_published()
await ShoutsCache.prepare_recent_all()
await ShoutsCache.prepare_recent_reacted()
print("[storage.shoutscache] updated")
print("[service.shoutscache] updated")
except Exception as err:
print("[storage.shoutscache] error: %s" % (err))
print("[service.shoutscache] error: %s" % (err))
raise err
await asyncio.sleep(ShoutsCache.period)

View File

@ -14,7 +14,7 @@ class TopicStorage:
for topic in self.topics.values():
self.load_parents(topic) # TODO: test
print('[storage.topics] %d ' % len(self.topics.keys()))
print('[service.topics] %d ' % len(self.topics.keys()))
@staticmethod
def load_parents(topic):