restructured,inbox-removed

2023-10-05 21:46:18 +03:00
parent 6dfec6714a
commit deac939ed8
49 changed files with 886 additions and 1549 deletions
--- a/migration/tables/comments.py
+++ b/migration/tables/comments.py
@@ -2,7 +2,7 @@ from datetime import datetime, timezone

 from dateutil.parser import parse as date_parse

-from base.orm import local_session
+from services.db import local_session
 from migration.html2text import html2text
 from orm.reaction import Reaction, ReactionKind
 from orm.shout import ShoutReactionsFollower
@@ -15,34 +15,28 @@ ts = datetime.now(tz=timezone.utc)

 def auto_followers(session, topics, reaction_dict):
    # creating shout's reactions following for reaction author
-    following1 = session.query(
-        ShoutReactionsFollower
-    ).where(
-        ShoutReactionsFollower.follower == reaction_dict["createdBy"]
-    ).filter(
-        ShoutReactionsFollower.shout == reaction_dict["shout"]
-    ).first()
+    following1 = (
+        session.query(ShoutReactionsFollower)
+        .where(ShoutReactionsFollower.follower == reaction_dict["createdBy"])
+        .filter(ShoutReactionsFollower.shout == reaction_dict["shout"])
+        .first()
+    )
    if not following1:
        following1 = ShoutReactionsFollower.create(
-            follower=reaction_dict["createdBy"],
-            shout=reaction_dict["shout"],
-            auto=True
+            follower=reaction_dict["createdBy"], shout=reaction_dict["shout"], auto=True
        )
        session.add(following1)
    # creating topics followings for reaction author
    for t in topics:
-        tf = session.query(
-            TopicFollower
-        ).where(
-            TopicFollower.follower == reaction_dict["createdBy"]
-        ).filter(
-            TopicFollower.topic == t['id']
-        ).first()
+        tf = (
+            session.query(TopicFollower)
+            .where(TopicFollower.follower == reaction_dict["createdBy"])
+            .filter(TopicFollower.topic == t["id"])
+            .first()
+        )
        if not tf:
            topic_following = TopicFollower.create(
-                follower=reaction_dict["createdBy"],
-                topic=t['id'],
-                auto=True
+                follower=reaction_dict["createdBy"], topic=t["id"], auto=True
            )
            session.add(topic_following)

@@ -68,18 +62,15 @@ def migrate_ratings(session, entry, reaction_dict):
        try:
            # creating reaction from old rating
            rr = Reaction.create(**re_reaction_dict)
-            following2 = session.query(
-                ShoutReactionsFollower
-            ).where(
-                ShoutReactionsFollower.follower == re_reaction_dict['createdBy']
-            ).filter(
-                ShoutReactionsFollower.shout == rr.shout
-            ).first()
+            following2 = (
+                session.query(ShoutReactionsFollower)
+                .where(ShoutReactionsFollower.follower == re_reaction_dict["createdBy"])
+                .filter(ShoutReactionsFollower.shout == rr.shout)
+                .first()
+            )
            if not following2:
                following2 = ShoutReactionsFollower.create(
-                    follower=re_reaction_dict['createdBy'],
-                    shout=rr.shout,
-                    auto=True
+                    follower=re_reaction_dict["createdBy"], shout=rr.shout, auto=True
                )
                session.add(following2)
            session.add(rr)
@@ -150,9 +141,11 @@ async def migrate(entry, storage):
            else:
                stage = "author and old id found"
                try:
-                    shout = session.query(
-                        Shout
-                    ).where(Shout.slug == old_shout["slug"]).one()
+                    shout = (
+                        session.query(Shout)
+                        .where(Shout.slug == old_shout["slug"])
+                        .one()
+                    )
                    if shout:
                        reaction_dict["shout"] = shout.id
                        reaction_dict["createdBy"] = author.id if author else 1
@@ -178,9 +171,9 @@ async def migrate(entry, storage):


 def migrate_2stage(old_comment, idmap):
-    if old_comment.get('body'):
-        new_id = idmap.get(old_comment.get('oid'))
-        new_id = idmap.get(old_comment.get('_id'))
+    if old_comment.get("body"):
+        new_id = idmap.get(old_comment.get("oid"))
+        new_id = idmap.get(old_comment.get("_id"))
        if new_id:
            new_replyto_id = None
            old_replyto_id = old_comment.get("replyTo")
@@ -190,17 +183,22 @@ def migrate_2stage(old_comment, idmap):
                comment = session.query(Reaction).where(Reaction.id == new_id).first()
                try:
                    if new_replyto_id:
-                        new_reply = session.query(Reaction).where(Reaction.id == new_replyto_id).first()
+                        new_reply = (
+                            session.query(Reaction)
+                            .where(Reaction.id == new_replyto_id)
+                            .first()
+                        )
                        if not new_reply:
                            print(new_replyto_id)
                            raise Exception("cannot find reply by id!")
                        comment.replyTo = new_reply.id
                        session.add(comment)
-                    srf = session.query(ShoutReactionsFollower).where(
-                        ShoutReactionsFollower.shout == comment.shout
-                    ).filter(
-                        ShoutReactionsFollower.follower == comment.createdBy
-                    ).first()
+                    srf = (
+                        session.query(ShoutReactionsFollower)
+                        .where(ShoutReactionsFollower.shout == comment.shout)
+                        .filter(ShoutReactionsFollower.follower == comment.createdBy)
+                        .first()
+                    )
                    if not srf:
                        srf = ShoutReactionsFollower.create(
                            shout=comment.shout, follower=comment.createdBy, auto=True
--- a/migration/tables/content_items.py
+++ b/migration/tables/content_items.py
@@ -3,13 +3,13 @@ import json
 from dateutil.parser import parse as date_parse
 from sqlalchemy.exc import IntegrityError
 from transliterate import translit
-from base.orm import local_session
+from services.db import local_session
 from migration.extract import extract_html, extract_media
 from orm.reaction import Reaction, ReactionKind
 from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower
 from orm.user import User
 from orm.topic import TopicFollower, Topic
-from services.stat.viewed import ViewedStorage
+from services.viewed import ViewedStorage
 import re

 OLD_DATE = "2016-03-05 22:22:00.350000"
@@ -33,7 +33,7 @@ def get_shout_slug(entry):
            slug = friend.get("slug", "")
            if slug:
                break
-    slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
+    slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
    return slug


@@ -41,28 +41,30 @@ def create_author_from_app(app):
    user = None
    userdata = None
    # check if email is used
-    if app['email']:
+    if app["email"]:
        with local_session() as session:
-            user = session.query(User).where(User.email == app['email']).first()
+            user = session.query(User).where(User.email == app["email"]).first()
            if not user:
                # print('[migration] app %r' % app)
-                name = app.get('name')
+                name = app.get("name")
                if name:
                    slug = translit(name, "ru", reversed=True).lower()
-                    slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
-                    print('[migration] created slug %s' % slug)
+                    slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
+                    print("[migration] created slug %s" % slug)
                    # check if slug is used
                    if slug:
                        user = session.query(User).where(User.slug == slug).first()

                        # get slug from email
                        if user:
-                            slug = app['email'].split('@')[0]
+                            slug = app["email"].split("@")[0]
                            user = session.query(User).where(User.slug == slug).first()
                            # one more try
                            if user:
-                                slug += '-author'
-                                user = session.query(User).where(User.slug == slug).first()
+                                slug += "-author"
+                                user = (
+                                    session.query(User).where(User.slug == slug).first()
+                                )

                # create user with application data
                if not user:
@@ -80,7 +82,7 @@ def create_author_from_app(app):
                    user = User.create(**userdata)
                    session.add(user)
                    session.commit()
-                    userdata['id'] = user.id
+                    userdata["id"] = user.id

            userdata = user.dict()
        return userdata
@@ -92,13 +94,16 @@ async def create_shout(shout_dict):
    s = Shout.create(**shout_dict)
    author = s.authors[0]
    with local_session() as session:
-        srf = session.query(ShoutReactionsFollower).where(
-            ShoutReactionsFollower.shout == s.id
-        ).filter(
-            ShoutReactionsFollower.follower == author.id
-        ).first()
+        srf = (
+            session.query(ShoutReactionsFollower)
+            .where(ShoutReactionsFollower.shout == s.id)
+            .filter(ShoutReactionsFollower.follower == author.id)
+            .first()
+        )
        if not srf:
-            srf = ShoutReactionsFollower.create(shout=s.id, follower=author.id, auto=True)
+            srf = ShoutReactionsFollower.create(
+                shout=s.id, follower=author.id, auto=True
+            )
            session.add(srf)
        session.commit()
    return s
@@ -117,14 +122,14 @@ async def get_user(entry, storage):
        elif user_oid:
            userdata = storage["users"]["by_oid"].get(user_oid)
        if not userdata:
-            print('no userdata by oid, anonymous')
+            print("no userdata by oid, anonymous")
            userdata = anondict
            print(app)
    # cleanup slug
    if userdata:
        slug = userdata.get("slug", "")
        if slug:
-            slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
+            slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
            userdata["slug"] = slug
    else:
        userdata = anondict
@@ -138,23 +143,30 @@ async def migrate(entry, storage):
    r = {
        "layout": type2layout[entry["type"]],
        "title": entry["title"],
-        "authors": [author, ],
+        "authors": [
+            author,
+        ],
        "slug": get_shout_slug(entry),
        "cover": (
-            "https://assets.discours.io/unsafe/1600x/" +
-            entry["thumborId"] if entry.get("thumborId") else entry.get("image", {}).get("url")
+            "https://assets.discours.io/unsafe/1600x/" + entry["thumborId"]
+            if entry.get("thumborId")
+            else entry.get("image", {}).get("url")
        ),
        "visibility": "public" if entry.get("published") else "authors",
-        "publishedAt": date_parse(entry.get("publishedAt")) if entry.get("published") else None,
-        "deletedAt": date_parse(entry.get("deletedAt")) if entry.get("deletedAt") else None,
+        "publishedAt": date_parse(entry.get("publishedAt"))
+        if entry.get("published")
+        else None,
+        "deletedAt": date_parse(entry.get("deletedAt"))
+        if entry.get("deletedAt")
+        else None,
        "createdAt": date_parse(entry.get("createdAt", OLD_DATE)),
        "updatedAt": date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts,
        "topics": await add_topics_follower(entry, storage, author),
-        "body": extract_html(entry, cleanup=True)
+        "body": extract_html(entry, cleanup=True),
    }

    # main topic patch
-    r['mainTopic'] = r['topics'][0]
+    r["mainTopic"] = r["topics"][0]

    # published author auto-confirm
    if entry.get("published"):
@@ -177,14 +189,16 @@ async def migrate(entry, storage):
        shout_dict["oid"] = entry.get("_id", "")
        shout = await create_shout(shout_dict)
    except IntegrityError as e:
-        print('[migration] create_shout integrity error', e)
+        print("[migration] create_shout integrity error", e)
        shout = await resolve_create_shout(shout_dict)
    except Exception as e:
        raise Exception(e)

    # udpate data
    shout_dict = shout.dict()
-    shout_dict["authors"] = [author.dict(), ]
+    shout_dict["authors"] = [
+        author.dict(),
+    ]

    # shout topics aftermath
    shout_dict["topics"] = await topics_aftermath(r, storage)
@@ -193,7 +207,9 @@ async def migrate(entry, storage):
    await content_ratings_to_reactions(entry, shout_dict["slug"])

    # shout views
-    await ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1), viewer='old-discours')
+    await ViewedStorage.increment(
+        shout_dict["slug"], amount=entry.get("views", 1), viewer="old-discours"
+    )
    # del shout_dict['ratings']

    storage["shouts"]["by_oid"][entry["_id"]] = shout_dict
@@ -205,7 +221,9 @@ async def add_topics_follower(entry, storage, user):
    topics = set([])
    category = entry.get("category")
    topics_by_oid = storage["topics"]["by_oid"]
-    oids = [category, ] + entry.get("tags", [])
+    oids = [
+        category,
+    ] + entry.get("tags", [])
    for toid in oids:
        tslug = topics_by_oid.get(toid, {}).get("slug")
        if tslug:
@@ -217,23 +235,20 @@ async def add_topics_follower(entry, storage, user):
            try:
                tpc = session.query(Topic).where(Topic.slug == tpcslug).first()
                if tpc:
-                    tf = session.query(
-                        TopicFollower
-                    ).where(
-                        TopicFollower.follower == user.id
-                    ).filter(
-                        TopicFollower.topic == tpc.id
-                    ).first()
+                    tf = (
+                        session.query(TopicFollower)
+                        .where(TopicFollower.follower == user.id)
+                        .filter(TopicFollower.topic == tpc.id)
+                        .first()
+                    )
                    if not tf:
                        tf = TopicFollower.create(
-                            topic=tpc.id,
-                            follower=user.id,
-                            auto=True
+                            topic=tpc.id, follower=user.id, auto=True
                        )
                        session.add(tf)
                        session.commit()
            except IntegrityError:
-                print('[migration.shout] hidden by topic ' + tpc.slug)
+                print("[migration.shout] hidden by topic " + tpc.slug)
    # main topic
    maintopic = storage["replacements"].get(topics_by_oid.get(category, {}).get("slug"))
    if maintopic in ttt:
@@ -254,7 +269,7 @@ async def process_user(userdata, storage, oid):
        if not user:
            try:
                slug = userdata["slug"].lower().strip()
-                slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
+                slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
                userdata["slug"] = slug
                user = User.create(**userdata)
                session.add(user)
@@ -263,7 +278,9 @@ async def process_user(userdata, storage, oid):
                print(f"[migration] user creating with slug {userdata['slug']}")
                print("[migration] from userdata")
                print(userdata)
-                raise Exception("[migration] cannot create user in content_items.get_user()")
+                raise Exception(
+                    "[migration] cannot create user in content_items.get_user()"
+                )
        if user.id == 946:
            print("[migration] ***************** ALPINA")
        if user.id == 2:
@@ -282,9 +299,9 @@ async def resolve_create_shout(shout_dict):
        s = session.query(Shout).filter(Shout.slug == shout_dict["slug"]).first()
        bump = False
        if s:
-            if s.createdAt != shout_dict['createdAt']:
+            if s.createdAt != shout_dict["createdAt"]:
                # create new with different slug
-                shout_dict["slug"] += '-' + shout_dict["layout"]
+                shout_dict["slug"] += "-" + shout_dict["layout"]
                try:
                    await create_shout(shout_dict)
                except IntegrityError as e:
@@ -375,7 +392,7 @@ async def content_ratings_to_reactions(entry, slug):
                    if content_rating["value"] > 0
                    else ReactionKind.DISLIKE,
                    "createdBy": rater.id,
-                    "shout": shout.id
+                    "shout": shout.id,
                }
                reaction = (
                    session.query(Reaction)
@@ -385,7 +402,11 @@ async def content_ratings_to_reactions(entry, slug):
                    .first()
                )
                if reaction:
-                    k = ReactionKind.AGREE if content_rating["value"] > 0 else ReactionKind.DISAGREE
+                    k = (
+                        ReactionKind.AGREE
+                        if content_rating["value"] > 0
+                        else ReactionKind.DISAGREE
+                    )
                    reaction_dict["kind"] = k
                    reaction.update(reaction_dict)
                    session.add(reaction)
--- a/migration/tables/remarks.py
+++ b/migration/tables/remarks.py
@@ -1,38 +1,30 @@
-from base.orm import local_session
+from services.db import local_session
 from migration.extract import extract_md
 from migration.html2text import html2text
 from orm.reaction import Reaction, ReactionKind


 def migrate(entry, storage):
-    post_oid = entry['contentItem']
+    post_oid = entry["contentItem"]
    print(post_oid)
-    shout_dict = storage['shouts']['by_oid'].get(post_oid)
+    shout_dict = storage["shouts"]["by_oid"].get(post_oid)
    if shout_dict:
-        print(shout_dict['body'])
+        print(shout_dict["body"])
        remark = {
-            "shout": shout_dict['id'],
-            "body": extract_md(
-                html2text(entry['body']),
-                shout_dict
-            ),
-            "kind": ReactionKind.REMARK
+            "shout": shout_dict["id"],
+            "body": extract_md(html2text(entry["body"]), shout_dict),
+            "kind": ReactionKind.REMARK,
        }

-        if entry.get('textBefore'):
-            remark['range'] = str(
-                    shout_dict['body']
-                        .index(
-                            entry['textBefore'] or ''
-                        )
-                ) + ':' + str(
-                    shout_dict['body']
-                        .index(
-                            entry['textAfter'] or ''
-                        ) + len(
-                            entry['textAfter'] or ''
-                        )
+        if entry.get("textBefore"):
+            remark["range"] = (
+                str(shout_dict["body"].index(entry["textBefore"] or ""))
+                + ":"
+                + str(
+                    shout_dict["body"].index(entry["textAfter"] or "")
+                    + len(entry["textAfter"] or "")
                )
+            )

        with local_session() as session:
            rmrk = Reaction.create(**remark)
--- a/migration/tables/topics.py
+++ b/migration/tables/topics.py
@@ -1,4 +1,4 @@
-from base.orm import local_session
+from services.db import local_session
 from migration.extract import extract_md
 from migration.html2text import html2text
 from orm import Topic
@@ -10,7 +10,7 @@ def migrate(entry):
        "slug": entry["slug"],
        "oid": entry["_id"],
        "title": entry["title"].replace("&nbsp;", " "),
-        "body": extract_md(html2text(body_orig))
+        "body": extract_md(html2text(body_orig)),
    }

    with local_session() as session:
--- a/migration/tables/users.py
+++ b/migration/tables/users.py
@@ -4,7 +4,7 @@ from bs4 import BeautifulSoup
 from dateutil.parser import parse
 from sqlalchemy.exc import IntegrityError

-from base.orm import local_session
+from services.db import local_session
 from orm.user import AuthorFollower, User, UserRating


@@ -19,12 +19,13 @@ def migrate(entry):
        "username": email,
        "email": email,
        "createdAt": parse(entry["createdAt"]),
-        "emailConfirmed": ("@discours.io" in email) or bool(entry["emails"][0]["verified"]),
+        "emailConfirmed": ("@discours.io" in email)
+        or bool(entry["emails"][0]["verified"]),
        "muted": False,  # amnesty
        "bio": entry["profile"].get("bio", ""),
        "links": [],
        "name": "anonymous",
-        "password": entry["services"]["password"].get("bcrypt")
+        "password": entry["services"]["password"].get("bcrypt"),
    }

    if "updatedAt" in entry:
@@ -34,9 +35,13 @@ def migrate(entry):
    if entry.get("profile"):
        # slug
        slug = entry["profile"].get("path").lower()
-        slug = re.sub('[^0-9a-zA-Z]+', '-', slug).strip()
+        slug = re.sub("[^0-9a-zA-Z]+", "-", slug).strip()
        user_dict["slug"] = slug
-        bio = (entry.get("profile", {"bio": ""}).get("bio") or "").replace('\(', '(').replace('\)', ')')
+        bio = (
+            (entry.get("profile", {"bio": ""}).get("bio") or "")
+            .replace("\(", "(")
+            .replace("\)", ")")
+        )
        bio_text = BeautifulSoup(bio, features="lxml").text

        if len(bio_text) > 120:
@@ -115,7 +120,7 @@ def post_migrate():
        "slug": "old-discours",
        "username": "old-discours",
        "email": "old@discours.io",
-        "name": "Просмотры на старой версии сайта"
+        "name": "Просмотры на старой версии сайта",
    }

    with local_session() as session:
@@ -148,11 +153,9 @@ def migrate_2stage(entry, id_map):
                }

                user_rating = UserRating.create(**user_rating_dict)
-                if user_rating_dict['value'] > 0:
+                if user_rating_dict["value"] > 0:
                    af = AuthorFollower.create(
-                        author=user.id,
-                        follower=rater.id,
-                        auto=True
+                        author=user.id, follower=rater.id, auto=True
                    )
                    session.add(af)
                session.add(user_rating)