diff --git a/base/redis.py b/base/redis.py index 17c6d6e3..39f5d47a 100644 --- a/base/redis.py +++ b/base/redis.py @@ -16,7 +16,7 @@ class RedisCache: async def disconnect(self): if self._instance is None: return - self._instance.close() + await self._instance.close() # await self._instance.wait_closed() # deprecated self._instance = None diff --git a/migration/__init__.py b/migration/__init__.py index 2f99fe7b..127b134d 100644 --- a/migration/__init__.py +++ b/migration/__init__.py @@ -81,6 +81,7 @@ async def shouts_handle(storage, args): """migrating content items one by one""" counter = 0 discours_author = 0 + anonymous_author = 0 pub_counter = 0 topics_dataset_bodies = [] topics_dataset_tlist = [] @@ -104,6 +105,8 @@ async def shouts_handle(storage, args): author: str = shout["authors"][0].dict() if author["slug"] == "discours": discours_author += 1 + if author["slug"] == "anonymous": + anonymous_author += 1 # print('[migration] ' + shout['slug'] + ' with author ' + author) if entry.get("published"): @@ -128,6 +131,7 @@ async def shouts_handle(storage, args): print("[migration] " + str(counter) + " content items were migrated") print("[migration] " + str(pub_counter) + " have been published") print("[migration] " + str(discours_author) + " authored by @discours") + print("[migration] " + str(anonymous_author) + " authored by @anonymous") async def comments_handle(storage): diff --git a/migration/tables/__init__.py b/migration/tables/__init__.py index b7ca72c5..8e7ee938 100644 --- a/migration/tables/__init__.py +++ b/migration/tables/__init__.py @@ -1 +1 @@ -__all__ = (["users", "tags", "content_items", "comments"],) +__all__ = (["users", "topics", "content_items", "comments"],) diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index b1128090..51d4b65d 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -8,10 +8,11 @@ from base.orm import local_session from migration.extract import prepare_html_body from orm.community import Community from orm.reaction import Reaction, ReactionKind -from orm.shout import Shout, ShoutTopic, User, ShoutReactionsFollower +from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower +from orm.user import User from orm.topic import TopicFollower from services.stat.reacted import ReactedStorage -from services.stat.viewed import ViewedByDay +from services.stat.viewed import ViewedStorage from services.zine.topics import TopicStorage OLD_DATE = "2016-03-05 22:22:00.350000" @@ -137,8 +138,7 @@ async def migrate(entry, storage): if userdata: userslug = userdata.get('slug') else: - userslug = "discours" # bad old id slug is used here to change later - print('DISCOURS AUTHORED: ' + oid) + userslug = "anonymous" # bad old id slug was found r["authors"] = [userslug, ] # slug @@ -336,7 +336,7 @@ async def migrate(entry, storage): raise Exception("[migration] content_item.ratings error: \n%r" % content_rating) # shout views - ViewedByDay.create(shout=shout_dict["slug"], value=entry.get("views", 1)) + ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1)) # del shout_dict['ratings'] shout_dict["oid"] = entry.get("_id") storage["shouts"]["by_oid"][entry["_id"]] = shout_dict diff --git a/migration/tables/replacements.json b/migration/tables/replacements.json index e47bb837..169c36b5 100644 --- a/migration/tables/replacements.json +++ b/migration/tables/replacements.json @@ -1,12 +1,8 @@ { + "207": "207", + "90-e": "90s", "1990-e": "90s", "2000-e": "2000s", - "90-e": "90s", - "207": "207", - "kartochki-rubinshteyna": "rubinstein-cards", - "Georgia": "georgia", - "Japan": "japan", - "Sweden": "sweden", "abstraktsiya": "abstract", "absurdism": "absurdism", "acclimatization": "acclimatisation", @@ -14,8 +10,8 @@ "adolf-gitler": "adolf-hitler", "afrika": "africa", "agata-kristi": "agatha-christie", - "agressiya": "agression", "agressivnoe-povedenie": "agression", + "agressiya": "agression", "aktsii": "actions", "aktsionizm": "actionism", "alber-kamyu": "albert-kamus", @@ -40,6 +36,7 @@ "andrey-tarkovskiy": "andrey-tarkovsky", "angliyskie-istorii": "english-stories", "angliyskiy-yazyk": "english-langugae", + "ango": "ango", "animation": "animation", "animatsiya": "animation", "anime": "anime", @@ -57,12 +54,13 @@ "aristotel": "aristotle", "arktika": "arctic", "armiya": "army", + "armiya-1": "army", "art": "art", "art-is": "art-is", "artists": "artists", "ateizm": "atheism", - "audiopoeziya": "audio-poetry", "audio-poetry": "audio-poetry", + "audiopoeziya": "audio-poetry", "audiospektakl": "audio-spectacles", "auktsyon": "auktsyon", "avangard": "avantgarde", @@ -74,6 +72,7 @@ "bannye-chteniya": "sauna-reading", "bardsongs": "bardsongs", "bdsm": "bdsm", + "beecake": "beecake", "belarus": "belarus", "belgiya": "belgium", "bertold-breht": "berttold-brecht", @@ -85,6 +84,7 @@ "biznes": "business", "blizhniy-vostok": "middle-east", "blizost": "closeness", + "blocked-in-russia": "blocked-in-russia", "blokada": "blockade", "bob-dilan": "bob-dylan", "bog": "god", @@ -152,30 +152,40 @@ "demonstrations": "demonstrations", "depression": "depression", "derevnya": "village", + "derrida": "derrida", "design": "design", "detskie-doma": "orphanages", "detstvo": "childhood", + "devyanostye": "90s", + "dialog": "dialogue", "digital": "digital", "digital-art": "digital-art", + "dinozavry": "dinosaurs", "directing": "directing", "diskurs": "discours", "diskurs-1": "discourse", + "diskurs-analiz": "discourse-analytics", "dissidenty": "dissidents", "diy": "diy", "dmitriy-donskoy": "dmitriy-donskoy", "dmitriy-prigov": "dmitriy-prigov", + "dnevnik-1": "dairy", "dnevniki": "dairies", "documentary": "documentary", + "dokumentalnaya-poeziya": "documentary-poetry", "dokumenty": "doсuments", "domashnee-nasilie": "home-terror", "donald-tramp": "donald-trump", "donbass": "donbass", + "donbass-diary": "donbass-diary", "donorstvo": "donation", + "dozhd": "rain", "drama": "drama", "dramaturgy": "dramaturgy", "drawing": "drawing", "drevo-zhizni": "tree-of-life", "drugs": "drugs", + "duh": "spirit", "dzhaz": "jazz", "dzhek-keruak": "jack-keruak", "dzhim-morrison": "jim-morrison", @@ -194,6 +204,7 @@ "ekspressionizm": "expressionism", "ekstremizm": "extremism", "ekzistentsializm-1": "existentialism", + "ekzistentsiya": "existence", "elections": "elections", "electronic": "electronics", "electronics": "electronics", @@ -248,10 +259,12 @@ "futuristy": "futurists", "futurizm": "futurism", "galereya": "gallery", + "galereya-anna-nova": "gallery-anna-nova", "gdr": "gdr", "gender": "gender", "gendernyy-diskurs": "gender", "gennadiy-aygi": "gennadiy-aygi", + "Georgia": "georgia", "gerhard-rihter": "gerhard-rihter", "germaniya": "germany", "germenevtika": "hermeneutics", @@ -268,8 +281,11 @@ "gravyura": "engraving", "grazhdanskaya-oborona": "grazhdanskaya-oborona", "gretsiya": "greece", + "griby": "mushrooms", + "gruziya-2": "georgia", "gulag": "gulag", "han-batyy": "khan-batyy", + "hayku": "haiku", "health": "health", "himiya": "chemistry", "hip-hop": "hip-hop", @@ -286,6 +302,7 @@ "idm": "idm", "igil": "isis", "igor-pomerantsev": "igor-pomerantsev", + "igra": "game", "igra-prestolov": "game-of-throne", "igry": "games", "iisus-hristos": "jesus-christ", @@ -312,6 +329,7 @@ "iskusstvennyy-intellekt": "artificial-intelligence", "islam": "islam", "istoriya-moskvy": "moscow-history", + "istoriya-nauki": "history-of-sceince", "istoriya-teatra": "theatre-history", "italiya": "italy", "italyanskiy-yazyk": "italian-language", @@ -322,6 +340,7 @@ "ivan-krylov": "ivan-krylov", "izobreteniya": "inventions", "izrail-1": "israel", + "Japan": "japan", "jazz": "jazz", "john-lennon": "john-lennon", "journalism": "journalism", @@ -329,12 +348,15 @@ "k-pop": "k-pop", "kalligrafiya": "calligraphy", "karikatura": "caricatures", + "kartochki-rubinshteyna": "rubinstein-cards", "katrin-nenasheva": "katrin-nenasheva", + "kavarga": "kavarga", "kavkaz": "caucasus", "kazan": "kazan", "kiberbezopasnost": "cybersecurity", "kinoklub": "cinema-club", "kirill-serebrennikov": "kirill-serebrennikov", + "kladbische": "cemetery", "klassika": "classic", "kollektivnoe-bessoznatelnoe": "сollective-unconscious", "komediya": "comedy", @@ -342,12 +364,14 @@ "kommunizm": "communism", "kommuny": "communes", "kompyuternye-igry": "computer-games", + "konets-vesny": "end-of-spring", "konservatizm": "conservatism", "kontrkultura": "counter-culture", "kontseptualizm": "conceptualism", "korotkometrazhka": "cinema-shorts", "kosmos": "cosmos", "kraudfanding": "crowdfunding", + "kriptovalyuty": "cryptocurrencies", "krizis": "crisis", "krov": "blood", "krym": "crimea", @@ -373,12 +397,15 @@ "lirika": "lirics", "literary-studies": "literary-studies", "literature": "literature", + "literaturnyykaver": "literature-cover", "lo-fi": "lo-fi", + "lomonosov": "lomonosov", "love": "love", "luzha-goluboy-krovi": "luzha-goluboy-krovi", "lyudvig-vitgenshteyn": "ludwig-wittgenstein", "lzhedmitriy": "false-dmitry", "lzhenauka": "pseudoscience", + "magiya": "magic", "maks-veber": "max-weber", "manifests": "manifests", "manipulyatsii-soznaniem": "mind-manipulation", @@ -388,13 +415,12 @@ "marsel-dyushan": "marchel-duchamp", "martin-haydegger": "martin-hidegger", "matematika": "maths", - "vladimir-mayakovskiy": "vladimir-mayakovsky", "mayakovskiy": "vladimir-mayakovsky", - "ekzistentsiya": "existence", "media": "media", "medicine": "medicine", "memuary": "memoirs", "menedzhment": "management", + "menty": "police", "merab-mamardashvili": "merab-mamardashvili", "mest": "revenge", "metamodernizm": "metamodern", @@ -417,6 +443,7 @@ "moda": "fashion", "modernizm": "modernism", "mokyumentari": "mockumentary", + "molodezh": "youth", "moloko-plus": "moloko-plus", "money": "money", "monologs": "monologues", @@ -436,6 +463,7 @@ "muzhchiny": "man", "myshlenie": "thinking", "nagornyy-karabah": "nagorno-karabakh", + "nasilie-1": "violence", "natsionalizm": "nationalism", "natsionalnaya-ideya": "national-idea", "natsizm": "nazism", @@ -500,9 +528,12 @@ "poetry": "poetry", "poetry-of-squares": "poetry-of-squares", "poetry-slam": "poetry-slam", + "pokoy": "peace", "police": "police", "politics": "politics", + "politzaklyuchennye": "political-prisoners", "polsha": "poland", + "pomosch": "help", "pop-art": "pop-art", "pop-culture": "pop-culture", "pornografiya": "pornography", @@ -543,8 +574,10 @@ "pskov": "pskov", "psychiatry": "psychiatry", "psychology": "psychology", + "ptitsy": "birds", "punk": "punk", "r-b": "rnb", + "rasizm": "racism", "realizm": "realism", "redaktura": "editorial", "refleksiya": "reflection", @@ -555,6 +588,7 @@ "renovatsiya": "renovation", "rep": "rap", "reportage": "reportage", + "reportazh-1": "reportage", "repressions": "repressions", "research": "research", "retroveyv": "retrowave", @@ -570,13 +604,16 @@ "ronald-reygan": "ronald-reygan", "roskomnadzor": "roskomnadzor", "rossiyskoe-kino": "russian-cinema", + "rouling": "rowling", "rozhava": "rojava", "rpts": "rpts", "rus-na-grani-sryva": "rus-na-grani-sryva", "russia": "russia", "russian-language": "russian-language", "russian-literature": "russian-literature", + "russkaya-toska": "russian-toska", "russkiy-mir": "russkiy-mir", + "salo": "lard", "salvador-dali": "salvador-dali", "samoidentifikatsiya": "self-identity", "samoopredelenie": "self-definition", @@ -591,6 +628,7 @@ "second-world-war": "second-world-war", "sekond-hend": "second-hand", "seksprosvet": "sex-education", + "seksualnoe-nasilie": "sexual-violence", "sekty": "sects", "semiotics": "semiotics", "serbiya": "serbia", @@ -606,6 +644,7 @@ "siriya": "siria", "skulptura": "sculpture", "slavoy-zhizhek": "slavoj-zizek", + "smert-1": "death", "smysl": "meaning", "sny": "dreams", "sobytiya": "events", @@ -637,10 +676,12 @@ "strah": "fear", "street-art": "street-art", "stsenarii": "scenarios", + "sud": "court", "summary": "summary", "supergeroi": "superheroes", "svetlana-aleksievich": "svetlana-aleksievich", "svobodu-ivanu-golunovu": "free-ivan-golunov", + "Sweden": "sweden", "syurrealizm": "surrealism", "tales": "tales", "tanets": "dance", @@ -679,6 +720,7 @@ "tvorchestvo": "creativity", "ugnetennyy-zhilischnyy-klass": "oppressed-housing-class", "uilyam-shekspir": "william-shakespeare", + "ukraina-2": "ukraine", "ukraine": "ukraine", "university": "university", "urban-studies": "urban-studies", @@ -686,6 +728,7 @@ "usa": "usa", "ussr": "ussr", "utopiya": "utopia", + "utrata": "loss", "valter-benyamin": "valter-benyamin", "varlam-shalamov": "varlam-shalamov", "vasiliy-ii-temnyy": "basil-ii-temnyy", @@ -714,6 +757,7 @@ "visual-culture": "visual-culture", "vizualnaya-poeziya": "visual-poetry", "vladimir-lenin": "vladimir-lenin", + "vladimir-mayakovskiy": "vladimir-mayakovsky", "vladimir-nabokov": "vladimir-nabokov", "vladimir-putin": "vladimir-putin", "vladimir-sorokin": "vladimir-sorokin", @@ -723,6 +767,7 @@ "vong-karvay": "wong-karwai", "vospominaniya": "memories", "vostok": "east", + "voyna-na-ukraine": "war-in-ukraine", "vremya": "time", "vudi-allen": "woody-allen", "vynuzhdennye-otnosheniya": "forced-relationship", @@ -736,65 +781,21 @@ "yan-vermeer": "yan-vermeer", "yanka-dyagileva": "yanka-dyagileva", "yaponskaya-literatura": "japan-literature", + "yazychestvo": "paganism", "youth": "youth", "yozef-rot": "yozef-rot", "yurgen-habermas": "jorgen-habermas", "za-liniey-mannergeyma": "behind-mannerheim-line", + "zabota": "care", "zahar-prilepin": "zahar-prilepin", "zakonodatelstvo": "laws", "zakony-mira": "world-laws", "zametki": "notes", "zhelanie": "wish", - "konets-vesny": "end-of-spring", "zhivotnye": "animals", "zhoze-saramago": "jose-saramago", "zigmund-freyd": "sigmund-freud", "zolotaya-orda": "golden-horde", "zombi": "zombie", - "zombi-simpsony": "zombie-simpsons", - "rouling": "rowling", - "diskurs-analiz": "discourse-analytics", - "menty": "police", - "ptitsy": "birds", - "salo": "lard", - "rasizm": "racism", - "griby": "mushrooms", - "politzaklyuchennye": "political-prisoners", - "molodezh": "youth", - "blocked-in-russia": "blocked-in-russia", - "kavarga": "kavarga", - "galereya-anna-nova": "gallery-anna-nova", - "derrida": "derrida", - "dinozavry": "dinosaurs", - "beecake": "beecake", - "literaturnyykaver": "literature-cover", - "dialog": "dialogue", - "dozhd": "rain", - "pomosch": "help", - "igra": "game", - "reportazh-1": "reportage", - "armiya-1": "army", - "ukraina-2": "ukraine", - "nasilie-1": "violence", - "smert-1": "death", - "dnevnik-1": "dairy", - "voyna-na-ukraine": "war-in-ukraine", - "zabota": "care", - "ango": "ango", - "hayku": "haiku", - "utrata": "loss", - "pokoy": "peace", - "kladbische": "cemetery", - "lomonosov": "lomonosov", - "istoriya-nauki": "history-of-sceince", - "sud": "court", - "russkaya-toska": "russian-toska", - "duh": "spirit", - "devyanostye": "90s", - "seksualnoe-nasilie": "sexual-violence", - "gruziya-2": "georgia", - "dokumentalnaya-poeziya": "documentary-poetry", - "kriptovalyuty": "cryptocurrencies", - "magiya": "magic", - "yazychestvo": "paganism" + "zombi-simpsony": "zombie-simpsons" } diff --git a/orm/__init__.py b/orm/__init__.py index 9c11262b..8c9f6412 100644 --- a/orm/__init__.py +++ b/orm/__init__.py @@ -6,6 +6,7 @@ from orm.reaction import Reaction from orm.shout import Shout from orm.topic import Topic, TopicFollower from orm.user import User, UserRating +from orm.viewed import ViewedByDay __all__ = [ "User", @@ -19,6 +20,7 @@ __all__ = [ "Notification", "Reaction", "UserRating", + "ViewedByDay" ] Base.metadata.create_all(engine) @@ -27,3 +29,5 @@ Resource.init_table() User.init_table() Community.init_table() Role.init_table() + +# NOTE: keep orm module isolated diff --git a/orm/reaction.py b/orm/reaction.py index 69b3b37a..2fdf900a 100644 --- a/orm/reaction.py +++ b/orm/reaction.py @@ -2,10 +2,25 @@ from datetime import datetime from sqlalchemy import Column, String, ForeignKey, DateTime from sqlalchemy import Enum +from enum import Enum as Enumeration from base.orm import Base -from services.stat.reacted import ReactedStorage, ReactionKind -from services.stat.viewed import ViewedStorage + + +class ReactionKind(Enumeration): + AGREE = 1 # +1 + DISAGREE = 2 # -1 + PROOF = 3 # +1 + DISPROOF = 4 # -1 + ASK = 5 # +0 bookmark + PROPOSE = 6 # +0 + QUOTE = 7 # +0 bookmark + COMMENT = 8 # +0 + ACCEPT = 9 # +1 + REJECT = 0 # -1 + LIKE = 11 # +1 + DISLIKE = 12 # -1 + # TYPE = # rating diff class Reaction(Base): @@ -26,12 +41,3 @@ class Reaction(Base): range = Column(String, nullable=True, comment="Range in format :") kind = Column(Enum(ReactionKind), nullable=False, comment="Reaction kind") oid = Column(String, nullable=True, comment="Old ID") - - @property - async def stat(self): - return { - "viewed": await ViewedStorage.get_reaction(self.id), - "reacted": len(await ReactedStorage.get_reaction(self.id)), - "rating": await ReactedStorage.get_reaction_rating(self.id), - "commented": len(await ReactedStorage.get_reaction_comments(self.id)), - } diff --git a/orm/shout.py b/orm/shout.py index 42a45eac..c889cbba 100644 --- a/orm/shout.py +++ b/orm/shout.py @@ -5,10 +5,16 @@ from sqlalchemy.orm import relationship from base.orm import Base from orm.reaction import Reaction -from orm.topic import Topic, ShoutTopic +from orm.topic import Topic from orm.user import User -from services.stat.reacted import ReactedStorage -from services.stat.viewed import ViewedStorage + + +class ShoutTopic(Base): + __tablename__ = "shout_topic" + + id = None # type: ignore + shout = Column(ForeignKey("shout.slug"), primary_key=True) + topic = Column(ForeignKey("topic.slug"), primary_key=True) class ShoutReactionsFollower(Base): @@ -62,17 +68,9 @@ class Shout(Base): createdAt = Column(DateTime, nullable=False, default=datetime.now, comment="Created at") updatedAt = Column(DateTime, nullable=True, comment="Updated at") publishedAt = Column(DateTime, nullable=True) + deletedAt = Column(DateTime, nullable=True) versionOf = Column(ForeignKey("shout.slug"), nullable=True) draft = Column(Boolean, default=False) lang = Column(String, default='ru') oid = Column(String, nullable=True) - - @property - async def stat(self): - return { - "viewed": await ViewedStorage.get_shout(self.slug), - "reacted": len(await ReactedStorage.get_shout(self.slug)), - "commented": len(await ReactedStorage.get_comments(self.slug)), - "rating": await ReactedStorage.get_rating(self.slug), - } diff --git a/orm/topic.py b/orm/topic.py index 060451d0..2031ac5c 100644 --- a/orm/topic.py +++ b/orm/topic.py @@ -5,14 +5,6 @@ from sqlalchemy import Column, Boolean, String, ForeignKey, DateTime, JSON as JS from base.orm import Base -class ShoutTopic(Base): - __tablename__ = "shout_topic" - - id = None # type: ignore - shout = Column(ForeignKey("shout.slug"), primary_key=True) - topic = Column(ForeignKey("topic.slug"), primary_key=True) - - class TopicFollower(Base): __tablename__ = "topic_followers" diff --git a/orm/user.py b/orm/user.py index d6dd4c81..638d4a2d 100644 --- a/orm/user.py +++ b/orm/user.py @@ -82,17 +82,25 @@ class User(Base): @staticmethod def init_table(): with local_session() as session: - default = session.query(User).filter(User.slug == "discours").first() + default = session.query(User).filter(User.slug == "anonymous").first() if not default: - default = User.create( - id=0, - email="welcome@discours.io", - username="welcome@discours.io", - name="Дискурс", - slug="discours", - userpic="https://discours.io/images/logo-mini.svg", - ) - + defaul_dict = { + "email": "noreply@discours.io", + "username": "noreply@discours.io", + "name": "Аноним", + "slug": "anonymous", + } + default = User.create(**defaul_dict) + session.add(default) + discours_dict = { + "email": "welcome@discours.io", + "username": "welcome@discours.io", + "name": "Дискурс", + "slug": "discours", + } + discours = User.create(**discours_dict) + session.add(discours) + session.commit() User.default_user = default async def get_permission(self): diff --git a/orm/viewed.py b/orm/viewed.py new file mode 100644 index 00000000..7db97418 --- /dev/null +++ b/orm/viewed.py @@ -0,0 +1,12 @@ +from datetime import datetime +from sqlalchemy import Column, DateTime, ForeignKey, Integer +from base.orm import Base + + +class ViewedByDay(Base): + __tablename__ = "viewed_by_day" + + id = None + shout = Column(ForeignKey("shout.slug"), primary_key=True) + day = Column(DateTime, primary_key=True, default=datetime.now) + value = Column(Integer) diff --git a/resolvers/__init__.py b/resolvers/__init__.py index 8af58039..84313eba 100644 --- a/resolvers/__init__.py +++ b/resolvers/__init__.py @@ -5,6 +5,7 @@ from resolvers.auth import ( register, confirm_email, auth_send_link, + get_current_user, ) from resolvers.collab import remove_author, invite_author from resolvers.community import ( @@ -18,7 +19,6 @@ from resolvers.community import ( from resolvers.editor import create_shout, delete_shout, update_shout from resolvers.profile import ( get_users_by_slugs, - get_current_user, get_user_reacted_shouts, get_user_roles, get_top_authors, @@ -44,7 +44,7 @@ from resolvers.zine import ( get_shout_by_slug, follow, unfollow, - view_shout, + increment_view, top_month, top_overall, recent_published, @@ -65,8 +65,8 @@ __all__ = [ "confirm_email", "auth_send_link", "sign_out", - # profile "get_current_user", + # profile "get_users_by_slugs", "get_user_roles", "get_top_authors", @@ -80,7 +80,7 @@ __all__ = [ "top_month", "top_overall", "top_viewed", - "view_shout", + "increment_view", "get_shout_by_slug", # editor "create_shout", diff --git a/resolvers/auth.py b/resolvers/auth.py index 4b9ceeea..ead501e6 100644 --- a/resolvers/auth.py +++ b/resolvers/auth.py @@ -1,9 +1,10 @@ from urllib.parse import quote_plus +from datetime import datetime -from auth.tokenstorage import TokenStorage from graphql.type import GraphQLResolveInfo from transliterate import translit +from auth.tokenstorage import TokenStorage from auth.authenticate import login_required from auth.email import send_auth_email from auth.identity import Identity, Password @@ -20,6 +21,22 @@ from resolvers.profile import get_user_info from settings import SESSION_TOKEN_HEADER +@mutation.field("refreshSession") +@login_required +async def get_current_user(_, info): + user = info.context["request"].user + user.lastSeen = datetime.now() + with local_session() as session: + session.add(user) + session.commit() + token = await TokenStorage.create_session(user) + return { + "token": token, + "user": user, + "info": await get_user_info(user.slug), + } + + @mutation.field("confirmEmail") async def confirm_email(*_, confirm_token): """confirm owning email address""" diff --git a/resolvers/collection.py b/resolvers/collection.py index 2987b459..9eff6bbb 100644 --- a/resolvers/collection.py +++ b/resolvers/collection.py @@ -99,3 +99,6 @@ async def get_my_collections(_, info): session.query(Collection).when(Collection.createdBy == user_id).all() ) return collections + + +# TODO: get shouts list by collection diff --git a/resolvers/feed.py b/resolvers/feed.py index 3a15abd6..e2e42a73 100644 --- a/resolvers/feed.py +++ b/resolvers/feed.py @@ -13,7 +13,7 @@ from services.zine.shoutscache import prepare_shouts @query.field("shoutsForFeed") @login_required -def get_user_feed(_, info, offset, limit) -> List[Shout]: +async def get_user_feed(_, info, offset, limit) -> List[Shout]: user = info.context["request"].user shouts = [] with local_session() as session: diff --git a/resolvers/profile.py b/resolvers/profile.py index 6df879b9..3aff2f88 100644 --- a/resolvers/profile.py +++ b/resolvers/profile.py @@ -1,23 +1,39 @@ -from datetime import datetime from typing import List from sqlalchemy import and_, desc from sqlalchemy.orm import selectinload from auth.authenticate import login_required -from auth.tokenstorage import TokenStorage from base.orm import local_session from base.resolvers import mutation, query from orm.reaction import Reaction from orm.shout import Shout from orm.topic import Topic, TopicFollower from orm.user import User, UserRole, Role, UserRating, AuthorFollower -from resolvers.community import get_followed_communities -from resolvers.inbox import get_unread_counter -from resolvers.reactions import get_shout_reactions +from .community import get_followed_communities +from .inbox import get_unread_counter +from .reactions import get_shout_reactions +from .topics import get_topic_stat from services.auth.users import UserStorage +async def get_user_info(slug): + return { + "unread": await get_unread_counter(slug), # unread inbox messages counter + "topics": [t.slug for t in get_followed_topics(0, slug)], # followed topics slugs + "authors": [a.slug for a in get_followed_authors(0, slug)], # followed authors slugs + "reactions": [r.shout for r in get_shout_reactions(0, slug)], # followed reacted shouts slugs + "communities": [c.slug for c in get_followed_communities(0, slug)], # followed communities slugs + } + + +async def get_author_stat(slug): + # TODO: implement author stat + return { + + } + + @query.field("userReactedShouts") async def get_user_reacted_shouts(_, _info, slug, offset, limit) -> List[Shout]: user = await UserStorage.get_user_by_slug(slug) @@ -38,20 +54,22 @@ async def get_user_reacted_shouts(_, _info, slug, offset, limit) -> List[Shout]: @query.field("userFollowedTopics") @login_required -def get_followed_topics(_, slug) -> List[Topic]: - rows = [] +async def get_followed_topics(_, slug) -> List[Topic]: + topics = [] with local_session() as session: - rows = ( + topics = ( session.query(Topic) .join(TopicFollower) .where(TopicFollower.follower == slug) .all() ) - return rows + for topic in topics: + topic.stat = await get_topic_stat(topic.slug) + return topics @query.field("userFollowedAuthors") -def get_followed_authors(_, slug) -> List[User]: +async def get_followed_authors(_, slug) -> List[User]: authors = [] with local_session() as session: authors = ( @@ -60,6 +78,8 @@ def get_followed_authors(_, slug) -> List[User]: .where(AuthorFollower.follower == slug) .all() ) + for author in authors: + author.stat = await get_author_stat(author.slug) return authors @@ -75,33 +95,6 @@ async def user_followers(_, slug) -> List[User]: return users -# for mutation.field("refreshSession") -async def get_user_info(slug): - return { - "unread": await get_unread_counter(slug), - "topics": [t.slug for t in get_followed_topics(0, slug)], - "authors": [a.slug for a in get_followed_authors(0, slug)], - "reactions": [r.shout for r in get_shout_reactions(0, slug)], - "communities": [c.slug for c in get_followed_communities(0, slug)], - } - - -@mutation.field("refreshSession") -@login_required -async def get_current_user(_, info): - user = info.context["request"].user - user.lastSeen = datetime.now() - with local_session() as session: - session.add(user) - session.commit() - token = await TokenStorage.create_session(user) - return { - "token": token, - "user": user, - "info": await get_user_info(user.slug), - } - - @query.field("getUsersBySlugs") async def get_users_by_slugs(_, _info, slugs): with local_session() as session: diff --git a/resolvers/reactions.py b/resolvers/reactions.py index 5bcba0f7..8fc4aa53 100644 --- a/resolvers/reactions.py +++ b/resolvers/reactions.py @@ -10,6 +10,16 @@ from orm.shout import ShoutReactionsFollower from orm.user import User from services.auth.users import UserStorage from services.stat.reacted import ReactedStorage +from services.stat.viewed import ViewedStorage + + +async def get_reaction_stat(reaction_id): + return { + "viewed": await ViewedStorage.get_reaction(reaction_id), + "reacted": len(await ReactedStorage.get_reaction(reaction_id)), + "rating": await ReactedStorage.get_reaction_rating(reaction_id), + "commented": len(await ReactedStorage.get_reaction_comments(reaction_id)), + } def reactions_follow(user, slug, auto=False): @@ -61,6 +71,8 @@ async def create_reaction(_, info, inp): except Exception as e: print(f"[resolvers.reactions] error on reactions autofollowing: {e}") + reaction.stat = await get_reaction_stat(reaction.id) + return {"reaction": reaction} @@ -86,6 +98,8 @@ async def update_reaction(_, info, inp): reaction.range = inp.get("range") session.commit() + reaction.stat = await get_reaction_stat(reaction.id) + return {"reaction": reaction} @@ -118,6 +132,7 @@ async def get_shout_reactions(_, info, slug, offset, limit): .all() ) for r in reactions: + r.stat = await get_reaction_stat(r.id) r.createdBy = await UserStorage.get_user(r.createdBy or "discours") return reactions @@ -137,6 +152,7 @@ async def get_reactions_for_shouts(_, info, shouts, offset, limit): .all() ) for r in reactions: + r.stat = await get_reaction_stat(r.id) r.createdBy = await UserStorage.get_user(r.createdBy or "discours") return reactions @@ -152,5 +168,6 @@ async def get_reactions_by_author(_, info, slug, limit=50, offset=0): .offset(offset) ) for r in reactions: + r.stat = await get_reaction_stat(r.id) r.createdBy = await UserStorage.get_user(r.createdBy or "discours") return reactions diff --git a/resolvers/topics.py b/resolvers/topics.py index de7693db..d4f26879 100644 --- a/resolvers/topics.py +++ b/resolvers/topics.py @@ -6,16 +6,30 @@ from auth.authenticate import login_required from base.orm import local_session from base.resolvers import mutation, query from orm.topic import Topic, TopicFollower -from services.stat.topicstat import TopicStat from services.zine.shoutscache import ShoutsCache from services.zine.topics import TopicStorage +from services.stat.reacted import ReactedStorage +from services.stat.topicstat import TopicStat +from services.stat.viewed import ViewedStorage + + +async def get_topic_stat(slug): + return { + "shouts": len(TopicStat.shouts_by_topic.get(slug, [])), + "authors": len(TopicStat.authors_by_topic.get(slug, [])), + "followers": len(TopicStat.followers_by_topic.get(slug, [])), + "viewed": await ViewedStorage.get_topic(slug), + "reacted": len(await ReactedStorage.get_topic(slug)), + "commented": len(await ReactedStorage.get_topic_comments(slug)), + "rating": await ReactedStorage.get_topic_rating(slug) + } @query.field("topicsAll") async def topics_all(_, _info): topics = await TopicStorage.get_topics_all() for topic in topics: - topic.stat = await TopicStat.get_stat(topic.slug) + topic.stat = await get_topic_stat(topic.slug) return topics @@ -23,18 +37,19 @@ async def topics_all(_, _info): async def topics_by_community(_, info, community): topics = await TopicStorage.get_topics_by_community(community) for topic in topics: - topic.stat = await TopicStat.get_stat(topic.slug) + topic.stat = await get_topic_stat(topic.slug) return topics @query.field("topicsByAuthor") async def topics_by_author(_, _info, author): - topics = ShoutsCache.by_author.get(author) + shouts = ShoutsCache.by_author.get(author) author_topics = set() - for tpc in topics: - tpc = await TopicStorage.topics[tpc.slug] - tpc.stat = await TopicStat.get_stat(tpc.slug) - author_topics.add(tpc) + for s in shouts: + for tpc in s.topics: + tpc = await TopicStorage.topics[tpc.slug] + tpc.stat = await get_topic_stat(tpc.slug) + author_topics.add(tpc) return list(author_topics) @@ -91,11 +106,8 @@ async def topics_random(_, info, amount=12): topics = await TopicStorage.get_topics_all() normalized_topics = [] for topic in topics: - topic_stat = await TopicStat.get_stat(topic.slug) - # FIXME: expects topicstat fix - # #if topic_stat["shouts"] > 2: - # normalized_topics.append(topic) - topic.stat = topic_stat - normalized_topics.append(topic) + topic.stat = await get_topic_stat(topic.slug) + if topic.stat["shouts"] > 2: + normalized_topics.append(topic) sample_length = min(len(normalized_topics), amount) return random.sample(normalized_topics, sample_length) diff --git a/resolvers/zine.py b/resolvers/zine.py index 61d5ec83..1999f02c 100644 --- a/resolvers/zine.py +++ b/resolvers/zine.py @@ -64,12 +64,6 @@ async def recent_reacted(_, _info, offset, limit): return ShoutsCache.recent_reacted[offset : offset + limit] -@mutation.field("viewShout") -async def view_shout(_, _info, slug): - await ViewedStorage.increment(slug) - return {"error": ""} - - @query.field("getShoutBySlug") async def get_shout_by_slug(_, info, slug): all_fields = [ diff --git a/schema.graphql b/schema.graphql index e21e0aca..6705e4ee 100644 --- a/schema.graphql +++ b/schema.graphql @@ -157,8 +157,6 @@ type Mutation { createShout(input: ShoutInput!): Result! updateShout(input: ShoutInput!): Result! deleteShout(slug: String!): Result! - viewShout(slug: String!): Result! - viewReaction(reaction_id: Int!): Result! # user profile rateUser(slug: String!, value: Int!): Result! diff --git a/services/stat/reacted.py b/services/stat/reacted.py index 748834e2..3f2ab222 100644 --- a/services/stat/reacted.py +++ b/services/stat/reacted.py @@ -1,29 +1,7 @@ import asyncio -from datetime import datetime -from enum import Enum as Enumeration - -from sqlalchemy import Column, DateTime, ForeignKey, Boolean -from sqlalchemy.orm.attributes import flag_modified -from sqlalchemy.types import Enum as ColumnEnum - -from base.orm import Base, local_session -from orm.topic import ShoutTopic - - -class ReactionKind(Enumeration): - AGREE = 1 # +1 - DISAGREE = 2 # -1 - PROOF = 3 # +1 - DISPROOF = 4 # -1 - ASK = 5 # +0 bookmark - PROPOSE = 6 # +0 - QUOTE = 7 # +0 bookmark - COMMENT = 8 # +0 - ACCEPT = 9 # +1 - REJECT = 0 # -1 - LIKE = 11 # +1 - DISLIKE = 12 # -1 - # TYPE = # rating diff +from base.orm import local_session +from orm.reaction import ReactionKind, Reaction +from services.zine.topics import TopicStorage def kind_to_rate(kind) -> int: @@ -45,18 +23,6 @@ def kind_to_rate(kind) -> int: return 0 -class ReactedByDay(Base): - __tablename__ = "reacted_by_day" - - id = None # type: ignore - reaction = Column(ForeignKey("reaction.id"), primary_key=True) - shout = Column(ForeignKey("shout.slug"), primary_key=True) - replyTo = Column(ForeignKey("reaction.id"), nullable=True) - kind = Column(ColumnEnum(ReactionKind), nullable=False, comment="Reaction kind") - day = Column(DateTime, primary_key=True, default=datetime.now) - comment = Column(Boolean, default=False) - - class ReactedStorage: reacted = {"shouts": {}, "topics": {}, "reactions": {}} rating = {"shouts": {}, "topics": {}, "reactions": {}} @@ -64,6 +30,7 @@ class ReactedStorage: to_flush = [] period = 30 * 60 # sec lock = asyncio.Lock() + modified_shouts = set([]) @staticmethod async def get_shout(shout_slug): @@ -82,7 +49,7 @@ class ReactedStorage: self = ReactedStorage async with self.lock: return list( - filter(lambda r: r.comment, self.reacted["shouts"].get(shout_slug, {})) + filter(lambda r: bool(r.body), self.reacted["shouts"].get(shout_slug, {})) ) @staticmethod @@ -90,7 +57,7 @@ class ReactedStorage: self = ReactedStorage async with self.lock: return list( - filter(lambda r: r.comment, self.reacted["topics"].get(topic_slug, [])) + filter(lambda r: bool(r.body), self.reacted["topics"].get(topic_slug, [])) ) @staticmethod @@ -99,7 +66,7 @@ class ReactedStorage: async with self.lock: return list( filter( - lambda r: r.comment, self.reacted["reactions"].get(reaction_id, {}) + lambda r: bool(r.body), self.reacted["reactions"].get(reaction_id, {}) ) ) @@ -138,118 +105,62 @@ class ReactedStorage: @staticmethod async def react(reaction): + ReactedStorage.modified_shouts.add(reaction.shout) + + @staticmethod + async def recount(reactions): self = ReactedStorage - - async with self.lock: - reactions = {} - - # iterate sibling reactions - reactions = self.reacted["shouts"].get(reaction.shout, {}) - for r in reactions.values(): - reaction = ReactedByDay.create({ - "day": datetime.now().replace( - hour=0, minute=0, second=0, microsecond=0 - ), - "reaction": r.id, - "kind": r.kind, - "shout": r.shout, - "comment": bool(r.body), - "replyTo": r.replyTo - }) - # renew sorted by shouts store - self.reacted["shouts"][reaction.shout] = self.reacted["shouts"].get(reaction.shout, []) - self.reacted["shouts"][reaction.shout].append(reaction) - if reaction.replyTo: - self.reacted["reaction"][reaction.replyTo] = self.reacted[ - "reactions" - ].get(reaction.shout, []) - self.reacted["reaction"][reaction.replyTo].append(reaction) - self.rating["reactions"][reaction.replyTo] = self.rating[ - "reactions" - ].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind) - else: - # rate only by root reactions on shout - self.rating["shouts"][reaction.replyTo] = self.rating["shouts"].get( - reaction.shout, 0 - ) + kind_to_rate(reaction.kind) - - flag_modified(reaction, "value") + for r in reactions: + # renew shout counters + self.reacted["shouts"][r.shout] = self.reacted["shouts"].get(r.shout, []) + self.reacted["shouts"][r.shout].append(r) + # renew topics counters + shout_topics = await TopicStorage.get_topics_by_slugs([r.shout, ]) + for t in shout_topics: + self.reacted["topics"][t] = self.reacted["topics"].get(t, []) + self.reacted["topics"][t].append(r) + self.rating["topics"][t] = \ + self.rating["topics"].get(t, 0) + kind_to_rate(r.kind) + if r.replyTo: + # renew reaction counters + self.reacted["reactions"][r.replyTo] = \ + self.reacted["reactions"].get(r.replyTo, []) + self.reacted["reactions"][r.replyTo].append(r) + self.rating["reactions"][r.replyTo] = \ + self.rating["reactions"].get(r.replyTo, 0) + kind_to_rate(r.kind) + else: + # renew shout rating + self.rating["shouts"][r.shout] = \ + self.rating["shouts"].get(r.shout, 0) + kind_to_rate(r.kind) @staticmethod def init(session): self = ReactedStorage - all_reactions = session.query(ReactedByDay).all() - print("[stat.reacted] %d reactions total" % len(all_reactions)) - for reaction in all_reactions: - shout = reaction.shout - topics = ( - session.query(ShoutTopic.topic).where(ShoutTopic.shout == shout).all() - ) - kind = reaction.kind - self.reacted["shouts"][shout] = self.reacted["shouts"].get(shout, []) - self.reacted["shouts"][shout].append(reaction) - self.rating["shouts"][shout] = self.rating["shouts"].get( - shout, 0 - ) + kind_to_rate(kind) - - for t in topics: - self.reacted["topics"][t] = self.reacted["topics"].get(t, []) - self.reacted["topics"][t].append(reaction) - self.rating["topics"][t] = self.rating["topics"].get( - t, 0 - ) + kind_to_rate( - kind - ) # rating - - if reaction.replyTo: - self.reacted["reactions"][reaction.replyTo] = self.reacted[ - "reactions" - ].get(reaction.replyTo, []) - self.reacted["reactions"][reaction.replyTo].append(reaction) - self.rating["reactions"][reaction.replyTo] = self.rating[ - "reactions" - ].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind) - ttt = self.reacted["topics"].values() - print("[stat.reacted] %d topics reacted" % len(ttt)) - print("[stat.reacted] %d shouts reacted" % len(self.reacted["shouts"])) - print("[stat.reacted] %d reactions reacted" % len(self.reacted["reactions"])) + all_reactions = session.query(Reaction).all() + self.modified_shouts = set([r.shout for r in all_reactions]) + print("[stat.reacted] %d shouts with reactions updates" % len(self.modified_shouts)) @staticmethod - async def flush_changes(session): + async def recount_changed(session): self = ReactedStorage async with self.lock: - for slug in dict(self.reacted["shouts"]).keys(): - topics = ( - session.query(ShoutTopic.topic) - .where(ShoutTopic.shout == slug) - .all() - ) - reactions = self.reacted["shouts"].get(slug, []) - # print('[stat.reacted] shout {' + str(slug) + "}: " + str(len(reactions))) - for ts in list(topics): - tslug = ts[0] - topic_reactions = self.reacted["topics"].get(tslug, []) - topic_reactions += reactions - # print('[stat.reacted] topic {' + str(tslug) + "}: " + str(len(topic_reactions))) - reactions += list(self.reacted["reactions"].values()) - for reaction in reactions: - if getattr(reaction, "modified", False): - session.add(reaction) - flag_modified(reaction, "value") - reaction.modified = False - # print('flushing') - for reaction in self.to_flush: - session.add(reaction) - self.to_flush.clear() - session.commit() + print('[stat.reacted] recounting...') + for slug in list(self.modified_shouts): + siblings = session.query(Reaction).where(Reaction.shout == slug).all() + await self.recount(siblings) + + print("[stat.reacted] %d shouts with reactions updates" % len(self.modified_shouts)) + print("[stat.reacted] %d topics reacted" % len(self.reacted["topics"].values())) + print("[stat.reacted] %d shouts reacted" % len(self.reacted["shouts"])) + print("[stat.reacted] %d reactions reacted" % len(self.reacted["reactions"])) + self.modified_shouts = set([]) @staticmethod async def worker(): while True: try: with local_session() as session: - await ReactedStorage().flush_changes(session) - print("[stat.reacted] periodical flush") + await ReactedStorage.recount_changed(session) except Exception as err: - print("[stat.reacted] errror: %s" % (err)) + print("[stat.reacted] recount error %s" % (err)) await asyncio.sleep(ReactedStorage.period) diff --git a/services/stat/topicstat.py b/services/stat/topicstat.py index e1ef7c59..ae6091ea 100644 --- a/services/stat/topicstat.py +++ b/services/stat/topicstat.py @@ -1,17 +1,15 @@ import asyncio from base.orm import local_session -from orm.shout import Shout -from orm.topic import ShoutTopic, TopicFollower -from services.stat.reacted import ReactedStorage -from services.stat.viewed import ViewedStorage +from orm.shout import Shout, ShoutTopic +from orm.topic import TopicFollower from services.zine.shoutauthor import ShoutAuthorStorage class TopicStat: - shouts_by_topic = {} - authors_by_topic = {} - followers_by_topic = {} + shouts_by_topic = {} # Shout object stored + authors_by_topic = {} # User + followers_by_topic = {} # User lock = asyncio.Lock() period = 30 * 60 # sec @@ -19,38 +17,33 @@ class TopicStat: async def load_stat(session): self = TopicStat shout_topics = session.query(ShoutTopic).all() - print("[stat.topics] shout topics amount", len(shout_topics)) + print("[stat.topics] shouts linked %d times" % len(shout_topics)) for shout_topic in shout_topics: - + tpc = shout_topic.topic # shouts by topics - topic = shout_topic.topic - shout = shout_topic.shout - sss = set(self.shouts_by_topic.get(topic, [])) - shout = session.query(Shout).where(Shout.slug == shout).first() - sss.union( - [ - shout, - ] - ) - self.shouts_by_topic[topic] = list(sss) + shout = session.query(Shout).where(Shout.slug == shout_topic.shout).first() + self.shouts_by_topic[tpc] = self.shouts_by_topic.get(tpc, []) + if shout not in self.shouts_by_topic[tpc]: + self.shouts_by_topic[tpc].append(shout) # authors by topics - authors = await ShoutAuthorStorage.get_authors(shout) - aaa = set(self.authors_by_topic.get(topic, [])) - aaa.union(authors) - self.authors_by_topic[topic] = list(aaa) + authors = await ShoutAuthorStorage.get_authors(shout.slug) + self.authors_by_topic[tpc] = self.authors_by_topic.get(tpc, []) + for a in authors: + if a not in self.authors_by_topic[tpc]: + self.authors_by_topic[tpc].append(a) - print("[stat.topics] authors sorted") - print("[stat.topics] shouts sorted") + print("[stat.topics] shouts indexed by %d topics" % len(self.shouts_by_topic.keys())) + print("[stat.topics] authors indexed by %d topics" % len(self.authors_by_topic.keys())) self.followers_by_topic = {} - followings = session.query(TopicFollower) + followings = session.query(TopicFollower).all() for flw in followings: topic = flw.topic user = flw.follower - if topic not in self.followers_by_topic: - self.followers_by_topic[topic] = [] - self.followers_by_topic[topic].append(user) + self.followers_by_topic[topic] = self.followers_by_topic.get(topic, []) + if user not in self.followers_by_topic[topic]: + self.followers_by_topic[topic].append(user) print("[stat.topics] followers sorted") @staticmethod @@ -59,23 +52,6 @@ class TopicStat: async with self.lock: return self.shouts_by_topic.get(topic, []) - @staticmethod - async def get_stat(topic): - self = TopicStat - async with self.lock: - shouts = self.shouts_by_topic.get(topic, []) - followers = self.followers_by_topic.get(topic, []) - authors = self.authors_by_topic.get(topic, []) - return { - "shouts": len(shouts), - "authors": len(authors), - "followers": len(followers), - "viewed": await ViewedStorage.get_topic(topic), - "reacted": len(await ReactedStorage.get_topic(topic)), - "commented": len(await ReactedStorage.get_topic_comments(topic)), - "rating": await ReactedStorage.get_topic_rating(topic), - } - @staticmethod async def worker(): self = TopicStat @@ -84,7 +60,6 @@ class TopicStat: with local_session() as session: async with self.lock: await self.load_stat(session) - print("[stat.topics] periodical update") except Exception as err: - print("[stat.topics] errror: %s" % (err)) + raise Exception(err) await asyncio.sleep(self.period) diff --git a/services/stat/viewed.py b/services/stat/viewed.py index 59eba9fc..70f1fc30 100644 --- a/services/stat/viewed.py +++ b/services/stat/viewed.py @@ -1,20 +1,12 @@ import asyncio from datetime import datetime -from sqlalchemy import Column, DateTime, ForeignKey, Integer +from base.orm import local_session + from sqlalchemy.orm.attributes import flag_modified -from base.orm import Base, local_session -from orm.topic import ShoutTopic - - -class ViewedByDay(Base): - __tablename__ = "viewed_by_day" - - id = None - shout = Column(ForeignKey("shout.slug"), primary_key=True) - day = Column(DateTime, primary_key=True, default=datetime.now) - value = Column(Integer) +from orm.shout import ShoutTopic +from orm.viewed import ViewedByDay class ViewedStorage: @@ -47,7 +39,7 @@ class ViewedStorage: if this_day_view.day < view.day: self.this_day_views[shout] = view - print("[stat.viewed] %d shouts viewed" % len(views)) + print("[stat.viewed] %d shouts viewed" % len(self.viewed['shouts'])) @staticmethod async def get_shout(shout_slug): @@ -68,7 +60,7 @@ class ViewedStorage: return self.viewed["reactions"].get(reaction_id, 0) @staticmethod - async def increment(shout_slug): + async def increment(shout_slug, amount=1): self = ViewedStorage async with self.lock: this_day_view = self.this_day_views.get(shout_slug) @@ -79,11 +71,9 @@ class ViewedStorage: this_day_view = ViewedByDay.create(shout=shout_slug, value=1) self.this_day_views[shout_slug] = this_day_view else: - this_day_view.value = this_day_view.value + 1 + this_day_view.value = this_day_view.value + amount this_day_view.modified = True - self.viewed["shouts"][shout_slug] = ( - self.viewed["shouts"].get(shout_slug, 0) + 1 - ) + self.viewed["shouts"][shout_slug] = (self.viewed["shouts"].get(shout_slug, 0) + amount) with local_session() as session: topics = ( session.query(ShoutTopic.topic) @@ -91,7 +81,7 @@ class ViewedStorage: .all() ) for t in topics: - self.viewed["topics"][t] = self.viewed["topics"].get(t, 0) + 1 + self.viewed["topics"][t] = self.viewed["topics"].get(t, 0) + amount flag_modified(this_day_view, "value") @staticmethod diff --git a/services/zine/shoutscache.py b/services/zine/shoutscache.py index eda6b8e3..2ebc20b4 100644 --- a/services/zine/shoutscache.py +++ b/services/zine/shoutscache.py @@ -7,13 +7,23 @@ from sqlalchemy.orm import selectinload from base.orm import local_session from orm.reaction import Reaction from orm.shout import Shout, ShoutAuthor, ShoutTopic -from services.stat.viewed import ViewedByDay +from services.stat.viewed import ViewedByDay, ViewedStorage +from services.stat.reacted import ReactedStorage + + +async def get_shout_stat(slug): + return { + "viewed": await ViewedStorage.get_shout(slug), + "reacted": len(await ReactedStorage.get_shout(slug)), + "commented": len(await ReactedStorage.get_comments(slug)), + "rating": await ReactedStorage.get_rating(slug), + } async def prepare_shouts(session, stmt): shouts = [] for s in list(map(lambda r: r.Shout, session.execute(stmt))): - s.stats = await s.stat + s.stat = await get_shout_stat(s.slug) shouts.append(s) return shouts @@ -41,10 +51,14 @@ class ShoutsCache: session, ( select(Shout) - .options(selectinload(Shout.authors), selectinload(Shout.topics)) + .options( + selectinload(Shout.authors), + selectinload(Shout.topics) + ) .where(bool(Shout.publishedAt)) + .filter(not bool(Shout.deletedAt)) .group_by(Shout.slug) - .order_by(desc("publishedAt")) + .order_by(desc(Shout.publishedAt)) .limit(ShoutsCache.limit) ), ) @@ -59,10 +73,13 @@ class ShoutsCache: session, ( select(Shout) - .options(selectinload(Shout.authors), selectinload(Shout.topics)) - .where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt))) + .options( + selectinload(Shout.authors), + selectinload(Shout.topics) + ) + .filter(not bool(Shout.deletedAt)) .group_by(Shout.slug) - .order_by(desc("createdAt")) + .order_by(desc(Shout.createdAt)) .limit(ShoutsCache.limit) ), ) @@ -73,22 +90,24 @@ class ShoutsCache: @staticmethod async def prepare_recent_reacted(): with local_session() as session: + reactions = session.query(Reaction).order_by(Reaction.createdAt).limit(ShoutsCache.limit) + reacted_slugs = set([]) + for r in reactions: + reacted_slugs.add(r.shout) shouts = await prepare_shouts( session, ( - select( - Shout, func.max(Reaction.createdAt).label("reactionCreatedAt") - ) + select(Shout) .options( selectinload(Shout.authors), selectinload(Shout.topics), ) - .join(Reaction, Reaction.shout == Shout.slug) - .where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt))) + .where(and_(bool(Shout.publishedAt), Shout.slug.in_(list(reacted_slugs)))) + .filter(not bool(Shout.deletedAt)) .group_by(Shout.slug) - .order_by(desc("reactionCreatedAt")) + .order_by(Shout.publishedAt) .limit(ShoutsCache.limit) - ), + ) ) async with ShoutsCache.lock: ShoutsCache.recent_reacted = shouts @@ -114,7 +133,7 @@ class ShoutsCache: .limit(ShoutsCache.limit) ), ) - shouts.sort(key=lambda s: s.stats["rating"], reverse=True) + shouts.sort(key=lambda s: s.stat["rating"], reverse=True) async with ShoutsCache.lock: print("[zine.cache] %d top shouts " % len(shouts)) ShoutsCache.top_overall = shouts @@ -135,7 +154,7 @@ class ShoutsCache: .limit(ShoutsCache.limit) ), ) - shouts.sort(key=lambda s: s.stats["rating"], reverse=True) + shouts.sort(key=lambda s: s.stat["rating"], reverse=True) async with ShoutsCache.lock: print("[zine.cache] %d top month shouts " % len(shouts)) ShoutsCache.top_month = shouts @@ -156,7 +175,7 @@ class ShoutsCache: .limit(ShoutsCache.limit) ), ) - shouts.sort(key=lambda s: s.stats["commented"], reverse=True) + shouts.sort(key=lambda s: s.stat["commented"], reverse=True) async with ShoutsCache.lock: print("[zine.cache] %d top commented shouts " % len(shouts)) ShoutsCache.top_viewed = shouts @@ -177,7 +196,7 @@ class ShoutsCache: .limit(ShoutsCache.limit) ), ) - shouts.sort(key=lambda s: s.stats["viewed"], reverse=True) + shouts.sort(key=lambda s: s.stat["viewed"], reverse=True) async with ShoutsCache.lock: print("[zine.cache] %d top viewed shouts " % len(shouts)) ShoutsCache.top_viewed = shouts @@ -186,14 +205,10 @@ class ShoutsCache: async def prepare_by_author(): shouts_by_author = {} with local_session() as session: - for a in session.query(ShoutAuthor).all(): - shout = session.query(Shout).filter(Shout.slug == a.shout).first() - - if not shouts_by_author.get(a.user): - shouts_by_author[a.user] = [] - + shout.stat = await get_shout_stat(shout.slug) + shouts_by_author[a.user] = shouts_by_author.get(a.user, []) if shout not in shouts_by_author[a.user]: shouts_by_author[a.user].append(shout) async with ShoutsCache.lock: @@ -204,17 +219,12 @@ class ShoutsCache: async def prepare_by_topic(): shouts_by_topic = {} with local_session() as session: - - for t in session.query(ShoutTopic).all(): - - shout = session.query(Shout).filter(Shout.slug == t.shout).first() - - if not shouts_by_topic.get(t.topic): - shouts_by_topic[t.topic] = [] - - if shout not in shouts_by_topic[t.topic]: - shouts_by_topic[t.topic].append(shout) - + for a in session.query(ShoutTopic).all(): + shout = session.query(Shout).filter(Shout.slug == a.shout).first() + shout.stat = await get_shout_stat(shout.slug) + shouts_by_topic[a.topic] = shouts_by_topic.get(a.topic, []) + if shout not in shouts_by_topic[a.topic]: + shouts_by_topic[a.topic].append(shout) async with ShoutsCache.lock: print("[zine.cache] indexed by %d topics " % len(shouts_by_topic.keys())) ShoutsCache.by_topic = shouts_by_topic