stats refactored

migration/__init__.py

@@ -81,6 +81,7 @@ async def shouts_handle(storage, args):
     """migrating content items one by one"""
     counter = 0
     discours_author = 0
+    anonymous_author = 0
     pub_counter = 0
     topics_dataset_bodies = []
     topics_dataset_tlist = []
@@ -104,6 +105,8 @@ async def shouts_handle(storage, args):
         author: str = shout["authors"][0].dict()
         if author["slug"] == "discours":
             discours_author += 1
+        if author["slug"] == "anonymous":
+            anonymous_author += 1
         # print('[migration] ' + shout['slug'] + ' with author ' + author)
 
         if entry.get("published"):
@@ -128,6 +131,7 @@ async def shouts_handle(storage, args):
     print("[migration] " + str(counter) + " content items were migrated")
     print("[migration] " + str(pub_counter) + " have been published")
     print("[migration] " + str(discours_author) + " authored by @discours")
+    print("[migration] " + str(anonymous_author) + " authored by @anonymous")
 
 
 async def comments_handle(storage):

migration/tables/__init__.py

@@ -1 +1 @@
-__all__ = (["users", "tags", "content_items", "comments"],)
+__all__ = (["users", "topics", "content_items", "comments"],)

migration/tables/content_items.py

@@ -8,10 +8,11 @@ from base.orm import local_session
 from migration.extract import prepare_html_body
 from orm.community import Community
 from orm.reaction import Reaction, ReactionKind
-from orm.shout import Shout, ShoutTopic, User, ShoutReactionsFollower
+from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower
+from orm.user import User
 from orm.topic import TopicFollower
 from services.stat.reacted import ReactedStorage
-from services.stat.viewed import ViewedByDay
+from services.stat.viewed import ViewedStorage
 from services.zine.topics import TopicStorage
 
 OLD_DATE = "2016-03-05 22:22:00.350000"
@@ -137,8 +138,7 @@ async def migrate(entry, storage):
     if userdata:
         userslug = userdata.get('slug')
     else:
-        userslug = "discours"  # bad old id slug is used here to change later
-        print('DISCOURS AUTHORED: ' + oid)
+        userslug = "anonymous"  # bad old id slug was found
     r["authors"] = [userslug, ]
 
     # slug
@@ -336,7 +336,7 @@ async def migrate(entry, storage):
         raise Exception("[migration] content_item.ratings error: \n%r" % content_rating)
 
     # shout views
-    ViewedByDay.create(shout=shout_dict["slug"], value=entry.get("views", 1))
+    ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1))
     # del shout_dict['ratings']
     shout_dict["oid"] = entry.get("_id")
     storage["shouts"]["by_oid"][entry["_id"]] = shout_dict

migration/tables/replacements.json

@@ -1,12 +1,8 @@
 {
+    "207": "207",
+    "90-e": "90s",
     "1990-e": "90s",
     "2000-e": "2000s",
-    "90-e": "90s",
-    "207": "207",
-    "kartochki-rubinshteyna": "rubinstein-cards",
-    "Georgia": "georgia",
-    "Japan": "japan",
-    "Sweden": "sweden",
     "abstraktsiya": "abstract",
     "absurdism": "absurdism",
     "acclimatization": "acclimatisation",
@@ -14,8 +10,8 @@
     "adolf-gitler": "adolf-hitler",
     "afrika": "africa",
     "agata-kristi": "agatha-christie",
-    "agressiya": "agression",
     "agressivnoe-povedenie": "agression",
+    "agressiya": "agression",
     "aktsii": "actions",
     "aktsionizm": "actionism",
     "alber-kamyu": "albert-kamus",
@@ -40,6 +36,7 @@
     "andrey-tarkovskiy": "andrey-tarkovsky",
     "angliyskie-istorii": "english-stories",
     "angliyskiy-yazyk": "english-langugae",
+    "ango": "ango",
     "animation": "animation",
     "animatsiya": "animation",
     "anime": "anime",
@@ -57,12 +54,13 @@
     "aristotel": "aristotle",
     "arktika": "arctic",
     "armiya": "army",
+    "armiya-1": "army",
     "art": "art",
     "art-is": "art-is",
     "artists": "artists",
     "ateizm": "atheism",
-    "audiopoeziya": "audio-poetry",
     "audio-poetry": "audio-poetry",
+    "audiopoeziya": "audio-poetry",
     "audiospektakl": "audio-spectacles",
     "auktsyon": "auktsyon",
     "avangard": "avantgarde",
@@ -74,6 +72,7 @@
     "bannye-chteniya": "sauna-reading",
     "bardsongs": "bardsongs",
     "bdsm": "bdsm",
+    "beecake": "beecake",
     "belarus": "belarus",
     "belgiya": "belgium",
     "bertold-breht": "berttold-brecht",
@@ -85,6 +84,7 @@
     "biznes": "business",
     "blizhniy-vostok": "middle-east",
     "blizost": "closeness",
+    "blocked-in-russia": "blocked-in-russia",
     "blokada": "blockade",
     "bob-dilan": "bob-dylan",
     "bog": "god",
@@ -152,30 +152,40 @@
     "demonstrations": "demonstrations",
     "depression": "depression",
     "derevnya": "village",
+    "derrida": "derrida",
     "design": "design",
     "detskie-doma": "orphanages",
     "detstvo": "childhood",
+    "devyanostye": "90s",
+    "dialog": "dialogue",
     "digital": "digital",
     "digital-art": "digital-art",
+    "dinozavry": "dinosaurs",
     "directing": "directing",
     "diskurs": "discours",
     "diskurs-1": "discourse",
+    "diskurs-analiz": "discourse-analytics",
     "dissidenty": "dissidents",
     "diy": "diy",
     "dmitriy-donskoy": "dmitriy-donskoy",
     "dmitriy-prigov": "dmitriy-prigov",
+    "dnevnik-1": "dairy",
     "dnevniki": "dairies",
     "documentary": "documentary",
+    "dokumentalnaya-poeziya": "documentary-poetry",
     "dokumenty": "doсuments",
     "domashnee-nasilie": "home-terror",
     "donald-tramp": "donald-trump",
     "donbass": "donbass",
+    "donbass-diary": "donbass-diary",
     "donorstvo": "donation",
+    "dozhd": "rain",
     "drama": "drama",
     "dramaturgy": "dramaturgy",
     "drawing": "drawing",
     "drevo-zhizni": "tree-of-life",
     "drugs": "drugs",
+    "duh": "spirit",
     "dzhaz": "jazz",
     "dzhek-keruak": "jack-keruak",
     "dzhim-morrison": "jim-morrison",
@@ -194,6 +204,7 @@
     "ekspressionizm": "expressionism",
     "ekstremizm": "extremism",
     "ekzistentsializm-1": "existentialism",
+    "ekzistentsiya": "existence",
     "elections": "elections",
     "electronic": "electronics",
     "electronics": "electronics",
@@ -248,10 +259,12 @@
     "futuristy": "futurists",
     "futurizm": "futurism",
     "galereya": "gallery",
+    "galereya-anna-nova": "gallery-anna-nova",
     "gdr": "gdr",
     "gender": "gender",
     "gendernyy-diskurs": "gender",
     "gennadiy-aygi": "gennadiy-aygi",
+    "Georgia": "georgia",
     "gerhard-rihter": "gerhard-rihter",
     "germaniya": "germany",
     "germenevtika": "hermeneutics",
@@ -268,8 +281,11 @@
     "gravyura": "engraving",
     "grazhdanskaya-oborona": "grazhdanskaya-oborona",
     "gretsiya": "greece",
+    "griby": "mushrooms",
+    "gruziya-2": "georgia",
     "gulag": "gulag",
     "han-batyy": "khan-batyy",
+    "hayku": "haiku",
     "health": "health",
     "himiya": "chemistry",
     "hip-hop": "hip-hop",
@@ -286,6 +302,7 @@
     "idm": "idm",
     "igil": "isis",
     "igor-pomerantsev": "igor-pomerantsev",
+    "igra": "game",
     "igra-prestolov": "game-of-throne",
     "igry": "games",
     "iisus-hristos": "jesus-christ",
@@ -312,6 +329,7 @@
     "iskusstvennyy-intellekt": "artificial-intelligence",
     "islam": "islam",
     "istoriya-moskvy": "moscow-history",
+    "istoriya-nauki": "history-of-sceince",
     "istoriya-teatra": "theatre-history",
     "italiya": "italy",
     "italyanskiy-yazyk": "italian-language",
@@ -322,6 +340,7 @@
     "ivan-krylov": "ivan-krylov",
     "izobreteniya": "inventions",
     "izrail-1": "israel",
+    "Japan": "japan",
     "jazz": "jazz",
     "john-lennon": "john-lennon",
     "journalism": "journalism",
@@ -329,12 +348,15 @@
     "k-pop": "k-pop",
     "kalligrafiya": "calligraphy",
     "karikatura": "caricatures",
+    "kartochki-rubinshteyna": "rubinstein-cards",
     "katrin-nenasheva": "katrin-nenasheva",
+    "kavarga": "kavarga",
     "kavkaz": "caucasus",
     "kazan": "kazan",
     "kiberbezopasnost": "cybersecurity",
     "kinoklub": "cinema-club",
     "kirill-serebrennikov": "kirill-serebrennikov",
+    "kladbische": "cemetery",
     "klassika": "classic",
     "kollektivnoe-bessoznatelnoe": "сollective-unconscious",
     "komediya": "comedy",
@@ -342,12 +364,14 @@
     "kommunizm": "communism",
     "kommuny": "communes",
     "kompyuternye-igry": "computer-games",
+    "konets-vesny": "end-of-spring",
     "konservatizm": "conservatism",
     "kontrkultura": "counter-culture",
     "kontseptualizm": "conceptualism",
     "korotkometrazhka": "cinema-shorts",
     "kosmos": "cosmos",
     "kraudfanding": "crowdfunding",
+    "kriptovalyuty": "cryptocurrencies",
     "krizis": "crisis",
     "krov": "blood",
     "krym": "crimea",
@@ -373,12 +397,15 @@
     "lirika": "lirics",
     "literary-studies": "literary-studies",
     "literature": "literature",
+    "literaturnyykaver": "literature-cover",
     "lo-fi": "lo-fi",
+    "lomonosov": "lomonosov",
     "love": "love",
     "luzha-goluboy-krovi": "luzha-goluboy-krovi",
     "lyudvig-vitgenshteyn": "ludwig-wittgenstein",
     "lzhedmitriy": "false-dmitry",
     "lzhenauka": "pseudoscience",
+    "magiya": "magic",
     "maks-veber": "max-weber",
     "manifests": "manifests",
     "manipulyatsii-soznaniem": "mind-manipulation",
@@ -388,13 +415,12 @@
     "marsel-dyushan": "marchel-duchamp",
     "martin-haydegger": "martin-hidegger",
     "matematika": "maths",
-    "vladimir-mayakovskiy": "vladimir-mayakovsky",
     "mayakovskiy": "vladimir-mayakovsky",
-    "ekzistentsiya": "existence",
     "media": "media",
     "medicine": "medicine",
     "memuary": "memoirs",
     "menedzhment": "management",
+    "menty": "police",
     "merab-mamardashvili": "merab-mamardashvili",
     "mest": "revenge",
     "metamodernizm": "metamodern",
@@ -417,6 +443,7 @@
     "moda": "fashion",
     "modernizm": "modernism",
     "mokyumentari": "mockumentary",
+    "molodezh": "youth",
     "moloko-plus": "moloko-plus",
     "money": "money",
     "monologs": "monologues",
@@ -436,6 +463,7 @@
     "muzhchiny": "man",
     "myshlenie": "thinking",
     "nagornyy-karabah": "nagorno-karabakh",
+    "nasilie-1": "violence",
     "natsionalizm": "nationalism",
     "natsionalnaya-ideya": "national-idea",
     "natsizm": "nazism",
@@ -500,9 +528,12 @@
     "poetry": "poetry",
     "poetry-of-squares": "poetry-of-squares",
     "poetry-slam": "poetry-slam",
+    "pokoy": "peace",
     "police": "police",
     "politics": "politics",
+    "politzaklyuchennye": "political-prisoners",
     "polsha": "poland",
+    "pomosch": "help",
     "pop-art": "pop-art",
     "pop-culture": "pop-culture",
     "pornografiya": "pornography",
@@ -543,8 +574,10 @@
     "pskov": "pskov",
     "psychiatry": "psychiatry",
     "psychology": "psychology",
+    "ptitsy": "birds",
     "punk": "punk",
     "r-b": "rnb",
+    "rasizm": "racism",
     "realizm": "realism",
     "redaktura": "editorial",
     "refleksiya": "reflection",
@@ -555,6 +588,7 @@
     "renovatsiya": "renovation",
     "rep": "rap",
     "reportage": "reportage",
+    "reportazh-1": "reportage",
     "repressions": "repressions",
     "research": "research",
     "retroveyv": "retrowave",
@@ -570,13 +604,16 @@
     "ronald-reygan": "ronald-reygan",
     "roskomnadzor": "roskomnadzor",
     "rossiyskoe-kino": "russian-cinema",
+    "rouling": "rowling",
     "rozhava": "rojava",
     "rpts": "rpts",
     "rus-na-grani-sryva": "rus-na-grani-sryva",
     "russia": "russia",
     "russian-language": "russian-language",
     "russian-literature": "russian-literature",
+    "russkaya-toska": "russian-toska",
     "russkiy-mir": "russkiy-mir",
+    "salo": "lard",
     "salvador-dali": "salvador-dali",
     "samoidentifikatsiya": "self-identity",
     "samoopredelenie": "self-definition",
@@ -591,6 +628,7 @@
     "second-world-war": "second-world-war",
     "sekond-hend": "second-hand",
     "seksprosvet": "sex-education",
+    "seksualnoe-nasilie": "sexual-violence",
     "sekty": "sects",
     "semiotics": "semiotics",
     "serbiya": "serbia",
@@ -606,6 +644,7 @@
     "siriya": "siria",
     "skulptura": "sculpture",
     "slavoy-zhizhek": "slavoj-zizek",
+    "smert-1": "death",
     "smysl": "meaning",
     "sny": "dreams",
     "sobytiya": "events",
@@ -637,10 +676,12 @@
     "strah": "fear",
     "street-art": "street-art",
     "stsenarii": "scenarios",
+    "sud": "court",
     "summary": "summary",
     "supergeroi": "superheroes",
     "svetlana-aleksievich": "svetlana-aleksievich",
     "svobodu-ivanu-golunovu": "free-ivan-golunov",
+    "Sweden": "sweden",
     "syurrealizm": "surrealism",
     "tales": "tales",
     "tanets": "dance",
@@ -679,6 +720,7 @@
     "tvorchestvo": "creativity",
     "ugnetennyy-zhilischnyy-klass": "oppressed-housing-class",
     "uilyam-shekspir": "william-shakespeare",
+    "ukraina-2": "ukraine",
     "ukraine": "ukraine",
     "university": "university",
     "urban-studies": "urban-studies",
@@ -686,6 +728,7 @@
     "usa": "usa",
     "ussr": "ussr",
     "utopiya": "utopia",
+    "utrata": "loss",
     "valter-benyamin": "valter-benyamin",
     "varlam-shalamov": "varlam-shalamov",
     "vasiliy-ii-temnyy": "basil-ii-temnyy",
@@ -714,6 +757,7 @@
     "visual-culture": "visual-culture",
     "vizualnaya-poeziya": "visual-poetry",
     "vladimir-lenin": "vladimir-lenin",
+    "vladimir-mayakovskiy": "vladimir-mayakovsky",
     "vladimir-nabokov": "vladimir-nabokov",
     "vladimir-putin": "vladimir-putin",
     "vladimir-sorokin": "vladimir-sorokin",
@@ -723,6 +767,7 @@
     "vong-karvay": "wong-karwai",
     "vospominaniya": "memories",
     "vostok": "east",
+    "voyna-na-ukraine": "war-in-ukraine",
     "vremya": "time",
     "vudi-allen": "woody-allen",
     "vynuzhdennye-otnosheniya": "forced-relationship",
@@ -736,65 +781,21 @@
     "yan-vermeer": "yan-vermeer",
     "yanka-dyagileva": "yanka-dyagileva",
     "yaponskaya-literatura": "japan-literature",
+    "yazychestvo": "paganism",
     "youth": "youth",
     "yozef-rot": "yozef-rot",
     "yurgen-habermas": "jorgen-habermas",
     "za-liniey-mannergeyma": "behind-mannerheim-line",
+    "zabota": "care",
     "zahar-prilepin": "zahar-prilepin",
     "zakonodatelstvo": "laws",
     "zakony-mira": "world-laws",
     "zametki": "notes",
     "zhelanie": "wish",
-    "konets-vesny": "end-of-spring",
     "zhivotnye": "animals",
     "zhoze-saramago": "jose-saramago",
     "zigmund-freyd": "sigmund-freud",
     "zolotaya-orda": "golden-horde",
     "zombi": "zombie",
-    "zombi-simpsony": "zombie-simpsons",
-    "rouling": "rowling",
-    "diskurs-analiz": "discourse-analytics",
-    "menty": "police",
-    "ptitsy": "birds",
-    "salo": "lard",
-    "rasizm": "racism",
-    "griby": "mushrooms",
-    "politzaklyuchennye": "political-prisoners",
-    "molodezh": "youth",
-    "blocked-in-russia": "blocked-in-russia",
-    "kavarga": "kavarga",
-    "galereya-anna-nova": "gallery-anna-nova",
-    "derrida": "derrida",
-    "dinozavry": "dinosaurs",
-    "beecake": "beecake",
-    "literaturnyykaver": "literature-cover",
-    "dialog": "dialogue",
-    "dozhd": "rain",
-    "pomosch": "help",
-    "igra": "game",
-    "reportazh-1": "reportage",
-    "armiya-1": "army",
-    "ukraina-2": "ukraine",
-    "nasilie-1": "violence",
-    "smert-1": "death",
-    "dnevnik-1": "dairy",
-    "voyna-na-ukraine": "war-in-ukraine",
-    "zabota": "care",
-    "ango": "ango",
-    "hayku": "haiku",
-    "utrata": "loss",
-    "pokoy": "peace",
-    "kladbische": "cemetery",
-    "lomonosov": "lomonosov",
-    "istoriya-nauki": "history-of-sceince",
-    "sud": "court",
-    "russkaya-toska": "russian-toska",
-    "duh": "spirit",
-    "devyanostye": "90s",
-    "seksualnoe-nasilie": "sexual-violence",
-    "gruziya-2": "georgia",
-    "dokumentalnaya-poeziya": "documentary-poetry",
-    "kriptovalyuty": "cryptocurrencies",
-    "magiya": "magic",
-    "yazychestvo": "paganism"
+    "zombi-simpsony": "zombie-simpsons"
 }