git-dep3
This commit is contained in:
@@ -1 +0,0 @@
|
||||
__all__ = (["users", "topics", "content_items", "comments"],)
|
||||
@@ -1,209 +0,0 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from dateutil.parser import parse as date_parse
|
||||
|
||||
from services.db import local_session
|
||||
from migration.html2text import html2text
|
||||
from orm.reaction import Reaction, ReactionKind
|
||||
from orm.shout import ShoutReactionsFollower
|
||||
from orm.topic import TopicFollower
|
||||
from orm.user import User
|
||||
from orm.shout import Shout
|
||||
|
||||
ts = datetime.now(tz=timezone.utc)
|
||||
|
||||
|
||||
def auto_followers(session, topics, reaction_dict):
|
||||
# creating shout's reactions following for reaction author
|
||||
following1 = (
|
||||
session.query(ShoutReactionsFollower)
|
||||
.where(ShoutReactionsFollower.follower == reaction_dict["createdBy"])
|
||||
.filter(ShoutReactionsFollower.shout == reaction_dict["shout"])
|
||||
.first()
|
||||
)
|
||||
if not following1:
|
||||
following1 = ShoutReactionsFollower.create(
|
||||
follower=reaction_dict["createdBy"], shout=reaction_dict["shout"], auto=True
|
||||
)
|
||||
session.add(following1)
|
||||
# creating topics followings for reaction author
|
||||
for t in topics:
|
||||
tf = (
|
||||
session.query(TopicFollower)
|
||||
.where(TopicFollower.follower == reaction_dict["createdBy"])
|
||||
.filter(TopicFollower.topic == t["id"])
|
||||
.first()
|
||||
)
|
||||
if not tf:
|
||||
topic_following = TopicFollower.create(
|
||||
follower=reaction_dict["createdBy"], topic=t["id"], auto=True
|
||||
)
|
||||
session.add(topic_following)
|
||||
|
||||
|
||||
def migrate_ratings(session, entry, reaction_dict):
|
||||
for comment_rating_old in entry.get("ratings", []):
|
||||
rater = (
|
||||
session.query(User)
|
||||
.filter(User.oid == comment_rating_old["createdBy"])
|
||||
.first()
|
||||
)
|
||||
re_reaction_dict = {
|
||||
"shout": reaction_dict["shout"],
|
||||
"replyTo": reaction_dict["id"],
|
||||
"kind": ReactionKind.LIKE
|
||||
if comment_rating_old["value"] > 0
|
||||
else ReactionKind.DISLIKE,
|
||||
"createdBy": rater.id if rater else 1,
|
||||
}
|
||||
cts = comment_rating_old.get("createdAt")
|
||||
if cts:
|
||||
re_reaction_dict["createdAt"] = date_parse(cts)
|
||||
try:
|
||||
# creating reaction from old rating
|
||||
rr = Reaction.create(**re_reaction_dict)
|
||||
following2 = (
|
||||
session.query(ShoutReactionsFollower)
|
||||
.where(ShoutReactionsFollower.follower == re_reaction_dict["createdBy"])
|
||||
.filter(ShoutReactionsFollower.shout == rr.shout)
|
||||
.first()
|
||||
)
|
||||
if not following2:
|
||||
following2 = ShoutReactionsFollower.create(
|
||||
follower=re_reaction_dict["createdBy"], shout=rr.shout, auto=True
|
||||
)
|
||||
session.add(following2)
|
||||
session.add(rr)
|
||||
|
||||
except Exception as e:
|
||||
print("[migration] comment rating error: %r" % re_reaction_dict)
|
||||
raise e
|
||||
session.commit()
|
||||
|
||||
|
||||
async def migrate(entry, storage):
|
||||
"""
|
||||
{
|
||||
"_id": "hdtwS8fSyFLxXCgSC",
|
||||
"body": "<p>",
|
||||
"contentItem": "mnK8KsJHPRi8DrybQ",
|
||||
"createdBy": "bMFPuyNg6qAD2mhXe",
|
||||
"thread": "01/",
|
||||
"createdAt": "2016-04-19 04:33:53+00:00",
|
||||
"ratings": [
|
||||
{ "createdBy": "AqmRukvRiExNpAe8C", "value": 1 },
|
||||
{ "createdBy": "YdE76Wth3yqymKEu5", "value": 1 }
|
||||
],
|
||||
"rating": 2,
|
||||
"updatedAt": "2020-05-27 19:22:57.091000+00:00",
|
||||
"updatedBy": "0"
|
||||
}
|
||||
->
|
||||
type Reaction {
|
||||
id: Int!
|
||||
shout: Shout!
|
||||
createdAt: DateTime!
|
||||
createdBy: User!
|
||||
updatedAt: DateTime
|
||||
deletedAt: DateTime
|
||||
deletedBy: User
|
||||
range: String # full / 0:2340
|
||||
kind: ReactionKind!
|
||||
body: String
|
||||
replyTo: Reaction
|
||||
stat: Stat
|
||||
old_id: String
|
||||
old_thread: String
|
||||
}
|
||||
"""
|
||||
old_ts = entry.get("createdAt")
|
||||
reaction_dict = {
|
||||
"createdAt": (ts if not old_ts else date_parse(old_ts)),
|
||||
"body": html2text(entry.get("body", "")),
|
||||
"oid": entry["_id"],
|
||||
}
|
||||
shout_oid = entry.get("contentItem")
|
||||
if shout_oid not in storage["shouts"]["by_oid"]:
|
||||
if len(storage["shouts"]["by_oid"]) > 0:
|
||||
return shout_oid
|
||||
else:
|
||||
print("[migration] no shouts migrated yet")
|
||||
raise Exception
|
||||
return
|
||||
else:
|
||||
stage = "started"
|
||||
reaction = None
|
||||
with local_session() as session:
|
||||
author = session.query(User).filter(User.oid == entry["createdBy"]).first()
|
||||
old_shout = storage["shouts"]["by_oid"].get(shout_oid)
|
||||
if not old_shout:
|
||||
raise Exception("no old shout in storage")
|
||||
else:
|
||||
stage = "author and old id found"
|
||||
try:
|
||||
shout = (
|
||||
session.query(Shout)
|
||||
.where(Shout.slug == old_shout["slug"])
|
||||
.one()
|
||||
)
|
||||
if shout:
|
||||
reaction_dict["shout"] = shout.id
|
||||
reaction_dict["createdBy"] = author.id if author else 1
|
||||
reaction_dict["kind"] = ReactionKind.COMMENT
|
||||
|
||||
# creating reaction from old comment
|
||||
reaction = Reaction.create(**reaction_dict)
|
||||
session.add(reaction)
|
||||
# session.commit()
|
||||
stage = "new reaction commited"
|
||||
reaction_dict = reaction.dict()
|
||||
topics = [t.dict() for t in shout.topics]
|
||||
auto_followers(session, topics, reaction_dict)
|
||||
|
||||
migrate_ratings(session, entry, reaction_dict)
|
||||
|
||||
return reaction
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(reaction)
|
||||
raise Exception(stage)
|
||||
return
|
||||
|
||||
|
||||
def migrate_2stage(old_comment, idmap):
|
||||
if old_comment.get("body"):
|
||||
new_id = idmap.get(old_comment.get("oid"))
|
||||
new_id = idmap.get(old_comment.get("_id"))
|
||||
if new_id:
|
||||
new_replyto_id = None
|
||||
old_replyto_id = old_comment.get("replyTo")
|
||||
if old_replyto_id:
|
||||
new_replyto_id = int(idmap.get(old_replyto_id, "0"))
|
||||
with local_session() as session:
|
||||
comment = session.query(Reaction).where(Reaction.id == new_id).first()
|
||||
try:
|
||||
if new_replyto_id:
|
||||
new_reply = (
|
||||
session.query(Reaction)
|
||||
.where(Reaction.id == new_replyto_id)
|
||||
.first()
|
||||
)
|
||||
if not new_reply:
|
||||
print(new_replyto_id)
|
||||
raise Exception("cannot find reply by id!")
|
||||
comment.replyTo = new_reply.id
|
||||
session.add(comment)
|
||||
srf = (
|
||||
session.query(ShoutReactionsFollower)
|
||||
.where(ShoutReactionsFollower.shout == comment.shout)
|
||||
.filter(ShoutReactionsFollower.follower == comment.createdBy)
|
||||
.first()
|
||||
)
|
||||
if not srf:
|
||||
srf = ShoutReactionsFollower.create(
|
||||
shout=comment.shout, follower=comment.createdBy, auto=True
|
||||
)
|
||||
session.add(srf)
|
||||
session.commit()
|
||||
except Exception:
|
||||
raise Exception("cannot find a comment by oldid")
|
||||
@@ -1,420 +0,0 @@
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from dateutil.parser import parse as date_parse
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from transliterate import translit
|
||||
from services.db import local_session
|
||||
from migration.extract import extract_html, extract_media
|
||||
from orm.reaction import Reaction, ReactionKind
|
||||
from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower
|
||||
from orm.user import User
|
||||
from orm.topic import TopicFollower, Topic
|
||||
from services.viewed import ViewedStorage
|
||||
import re
|
||||
|
||||
OLD_DATE = "2016-03-05 22:22:00.350000"
|
||||
ts = datetime.now(tz=timezone.utc)
|
||||
type2layout = {
|
||||
"Article": "article",
|
||||
"Literature": "literature",
|
||||
"Music": "audio",
|
||||
"Video": "video",
|
||||
"Image": "image",
|
||||
}
|
||||
|
||||
anondict = {"slug": "anonymous", "id": 1, "name": "Аноним"}
|
||||
discours = {"slug": "discours", "id": 2, "name": "Дискурс"}
|
||||
|
||||
|
||||
def get_shout_slug(entry):
|
||||
slug = entry.get("slug", "")
|
||||
if not slug:
|
||||
for friend in entry.get("friendlySlugs", []):
|
||||
slug = friend.get("slug", "")
|
||||
if slug:
|
||||
break
|
||||
slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
|
||||
return slug
|
||||
|
||||
|
||||
def create_author_from_app(app):
|
||||
user = None
|
||||
userdata = None
|
||||
# check if email is used
|
||||
if app["email"]:
|
||||
with local_session() as session:
|
||||
user = session.query(User).where(User.email == app["email"]).first()
|
||||
if not user:
|
||||
# print('[migration] app %r' % app)
|
||||
name = app.get("name")
|
||||
if name:
|
||||
slug = translit(name, "ru", reversed=True).lower()
|
||||
slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
|
||||
print("[migration] created slug %s" % slug)
|
||||
# check if slug is used
|
||||
if slug:
|
||||
user = session.query(User).where(User.slug == slug).first()
|
||||
|
||||
# get slug from email
|
||||
if user:
|
||||
slug = app["email"].split("@")[0]
|
||||
user = session.query(User).where(User.slug == slug).first()
|
||||
# one more try
|
||||
if user:
|
||||
slug += "-author"
|
||||
user = (
|
||||
session.query(User).where(User.slug == slug).first()
|
||||
)
|
||||
|
||||
# create user with application data
|
||||
if not user:
|
||||
userdata = {
|
||||
"username": app["email"],
|
||||
"email": app["email"],
|
||||
"name": app.get("name", ""),
|
||||
"bio": app.get("bio", ""),
|
||||
"emailConfirmed": False,
|
||||
"slug": slug,
|
||||
"createdAt": ts,
|
||||
"lastSeen": ts,
|
||||
}
|
||||
# print('[migration] userdata %r' % userdata)
|
||||
user = User.create(**userdata)
|
||||
session.add(user)
|
||||
session.commit()
|
||||
userdata["id"] = user.id
|
||||
|
||||
userdata = user.dict()
|
||||
return userdata
|
||||
else:
|
||||
raise Exception("app is not ok", app)
|
||||
|
||||
|
||||
async def create_shout(shout_dict):
|
||||
s = Shout.create(**shout_dict)
|
||||
author = s.authors[0]
|
||||
with local_session() as session:
|
||||
srf = (
|
||||
session.query(ShoutReactionsFollower)
|
||||
.where(ShoutReactionsFollower.shout == s.id)
|
||||
.filter(ShoutReactionsFollower.follower == author.id)
|
||||
.first()
|
||||
)
|
||||
if not srf:
|
||||
srf = ShoutReactionsFollower.create(
|
||||
shout=s.id, follower=author.id, auto=True
|
||||
)
|
||||
session.add(srf)
|
||||
session.commit()
|
||||
return s
|
||||
|
||||
|
||||
async def get_user(entry, storage):
|
||||
app = entry.get("application")
|
||||
userdata = None
|
||||
user_oid = None
|
||||
if app:
|
||||
userdata = create_author_from_app(app)
|
||||
else:
|
||||
user_oid = entry.get("createdBy")
|
||||
if user_oid == "0":
|
||||
userdata = discours
|
||||
elif user_oid:
|
||||
userdata = storage["users"]["by_oid"].get(user_oid)
|
||||
if not userdata:
|
||||
print("no userdata by oid, anonymous")
|
||||
userdata = anondict
|
||||
print(app)
|
||||
# cleanup slug
|
||||
if userdata:
|
||||
slug = userdata.get("slug", "")
|
||||
if slug:
|
||||
slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
|
||||
userdata["slug"] = slug
|
||||
else:
|
||||
userdata = anondict
|
||||
|
||||
user = await process_user(userdata, storage, user_oid)
|
||||
return user, user_oid
|
||||
|
||||
|
||||
async def migrate(entry, storage):
|
||||
author, user_oid = await get_user(entry, storage)
|
||||
r = {
|
||||
"layout": type2layout[entry["type"]],
|
||||
"title": entry["title"],
|
||||
"authors": [
|
||||
author,
|
||||
],
|
||||
"slug": get_shout_slug(entry),
|
||||
"cover": (
|
||||
"https://assets.discours.io/unsafe/1600x/" + entry["thumborId"]
|
||||
if entry.get("thumborId")
|
||||
else entry.get("image", {}).get("url")
|
||||
),
|
||||
"visibility": "public" if entry.get("published") else "authors",
|
||||
"publishedAt": date_parse(entry.get("publishedAt"))
|
||||
if entry.get("published")
|
||||
else None,
|
||||
"deletedAt": date_parse(entry.get("deletedAt"))
|
||||
if entry.get("deletedAt")
|
||||
else None,
|
||||
"createdAt": date_parse(entry.get("createdAt", OLD_DATE)),
|
||||
"updatedAt": date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts,
|
||||
"topics": await add_topics_follower(entry, storage, author),
|
||||
"body": extract_html(entry, cleanup=True),
|
||||
}
|
||||
|
||||
# main topic patch
|
||||
r["mainTopic"] = r["topics"][0]
|
||||
|
||||
# published author auto-confirm
|
||||
if entry.get("published"):
|
||||
with local_session() as session:
|
||||
# update user.emailConfirmed if published
|
||||
author.emailConfirmed = True
|
||||
session.add(author)
|
||||
session.commit()
|
||||
|
||||
# media
|
||||
media = extract_media(entry)
|
||||
r["media"] = json.dumps(media, ensure_ascii=True) if media else None
|
||||
|
||||
# ----------------------------------- copy
|
||||
shout_dict = r.copy()
|
||||
del shout_dict["topics"]
|
||||
|
||||
try:
|
||||
# save shout to db
|
||||
shout_dict["oid"] = entry.get("_id", "")
|
||||
shout = await create_shout(shout_dict)
|
||||
except IntegrityError as e:
|
||||
print("[migration] create_shout integrity error", e)
|
||||
shout = await resolve_create_shout(shout_dict)
|
||||
except Exception as e:
|
||||
raise Exception(e)
|
||||
|
||||
# udpate data
|
||||
shout_dict = shout.dict()
|
||||
shout_dict["authors"] = [
|
||||
author.dict(),
|
||||
]
|
||||
|
||||
# shout topics aftermath
|
||||
shout_dict["topics"] = await topics_aftermath(r, storage)
|
||||
|
||||
# content_item ratings to reactions
|
||||
await content_ratings_to_reactions(entry, shout_dict["slug"])
|
||||
|
||||
# shout views
|
||||
await ViewedStorage.increment(
|
||||
shout_dict["slug"], amount=entry.get("views", 1), viewer="old-discours"
|
||||
)
|
||||
# del shout_dict['ratings']
|
||||
|
||||
storage["shouts"]["by_oid"][entry["_id"]] = shout_dict
|
||||
storage["shouts"]["by_slug"][shout_dict["slug"]] = shout_dict
|
||||
return shout_dict
|
||||
|
||||
|
||||
async def add_topics_follower(entry, storage, user):
|
||||
topics = set([])
|
||||
category = entry.get("category")
|
||||
topics_by_oid = storage["topics"]["by_oid"]
|
||||
oids = [
|
||||
category,
|
||||
] + entry.get("tags", [])
|
||||
for toid in oids:
|
||||
tslug = topics_by_oid.get(toid, {}).get("slug")
|
||||
if tslug:
|
||||
topics.add(tslug)
|
||||
ttt = list(topics)
|
||||
# add author as TopicFollower
|
||||
with local_session() as session:
|
||||
for tpcslug in topics:
|
||||
try:
|
||||
tpc = session.query(Topic).where(Topic.slug == tpcslug).first()
|
||||
if tpc:
|
||||
tf = (
|
||||
session.query(TopicFollower)
|
||||
.where(TopicFollower.follower == user.id)
|
||||
.filter(TopicFollower.topic == tpc.id)
|
||||
.first()
|
||||
)
|
||||
if not tf:
|
||||
tf = TopicFollower.create(
|
||||
topic=tpc.id, follower=user.id, auto=True
|
||||
)
|
||||
session.add(tf)
|
||||
session.commit()
|
||||
except IntegrityError:
|
||||
print("[migration.shout] hidden by topic " + tpc.slug)
|
||||
# main topic
|
||||
maintopic = storage["replacements"].get(topics_by_oid.get(category, {}).get("slug"))
|
||||
if maintopic in ttt:
|
||||
ttt.remove(maintopic)
|
||||
ttt.insert(0, maintopic)
|
||||
return ttt
|
||||
|
||||
|
||||
async def process_user(userdata, storage, oid):
|
||||
with local_session() as session:
|
||||
uid = userdata.get("id") # anonymous as
|
||||
if not uid:
|
||||
print(userdata)
|
||||
print("has no id field, set it @anonymous")
|
||||
userdata = anondict
|
||||
uid = 1
|
||||
user = session.query(User).filter(User.id == uid).first()
|
||||
if not user:
|
||||
try:
|
||||
slug = userdata["slug"].lower().strip()
|
||||
slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
|
||||
userdata["slug"] = slug
|
||||
user = User.create(**userdata)
|
||||
session.add(user)
|
||||
session.commit()
|
||||
except IntegrityError:
|
||||
print(f"[migration] user creating with slug {userdata['slug']}")
|
||||
print("[migration] from userdata")
|
||||
print(userdata)
|
||||
raise Exception(
|
||||
"[migration] cannot create user in content_items.get_user()"
|
||||
)
|
||||
if user.id == 946:
|
||||
print("[migration] ***************** ALPINA")
|
||||
if user.id == 2:
|
||||
print("[migration] +++++++++++++++++ DISCOURS")
|
||||
userdata["id"] = user.id
|
||||
userdata["createdAt"] = user.createdAt
|
||||
storage["users"]["by_slug"][userdata["slug"]] = userdata
|
||||
storage["users"]["by_oid"][oid] = userdata
|
||||
if not user:
|
||||
raise Exception("could not get a user")
|
||||
return user
|
||||
|
||||
|
||||
async def resolve_create_shout(shout_dict):
|
||||
with local_session() as session:
|
||||
s = session.query(Shout).filter(Shout.slug == shout_dict["slug"]).first()
|
||||
bump = False
|
||||
if s:
|
||||
if s.createdAt != shout_dict["createdAt"]:
|
||||
# create new with different slug
|
||||
shout_dict["slug"] += "-" + shout_dict["layout"]
|
||||
try:
|
||||
await create_shout(shout_dict)
|
||||
except IntegrityError as e:
|
||||
print(e)
|
||||
bump = True
|
||||
else:
|
||||
# update old
|
||||
for key in shout_dict:
|
||||
if key in s.__dict__:
|
||||
if s.__dict__[key] != shout_dict[key]:
|
||||
print(
|
||||
"[migration] shout already exists, but differs in %s"
|
||||
% key
|
||||
)
|
||||
bump = True
|
||||
else:
|
||||
print("[migration] shout already exists, but lacks %s" % key)
|
||||
bump = True
|
||||
if bump:
|
||||
s.update(shout_dict)
|
||||
else:
|
||||
print("[migration] something went wrong with shout: \n%r" % shout_dict)
|
||||
raise Exception("")
|
||||
session.commit()
|
||||
return s
|
||||
|
||||
|
||||
async def topics_aftermath(entry, storage):
|
||||
r = []
|
||||
for tpc in filter(lambda x: bool(x), entry["topics"]):
|
||||
oldslug = tpc
|
||||
newslug = storage["replacements"].get(oldslug, oldslug)
|
||||
|
||||
if newslug:
|
||||
with local_session() as session:
|
||||
shout = session.query(Shout).where(Shout.slug == entry["slug"]).first()
|
||||
new_topic = session.query(Topic).where(Topic.slug == newslug).first()
|
||||
|
||||
shout_topic_old = (
|
||||
session.query(ShoutTopic)
|
||||
.join(Shout)
|
||||
.join(Topic)
|
||||
.filter(Shout.slug == entry["slug"])
|
||||
.filter(Topic.slug == oldslug)
|
||||
.first()
|
||||
)
|
||||
if shout_topic_old:
|
||||
shout_topic_old.update({"topic": new_topic.id})
|
||||
else:
|
||||
shout_topic_new = (
|
||||
session.query(ShoutTopic)
|
||||
.join(Shout)
|
||||
.join(Topic)
|
||||
.filter(Shout.slug == entry["slug"])
|
||||
.filter(Topic.slug == newslug)
|
||||
.first()
|
||||
)
|
||||
if not shout_topic_new:
|
||||
try:
|
||||
ShoutTopic.create(
|
||||
**{"shout": shout.id, "topic": new_topic.id}
|
||||
)
|
||||
except Exception:
|
||||
print("[migration] shout topic error: " + newslug)
|
||||
session.commit()
|
||||
if newslug not in r:
|
||||
r.append(newslug)
|
||||
else:
|
||||
print("[migration] ignored topic slug: \n%r" % tpc["slug"])
|
||||
# raise Exception
|
||||
return r
|
||||
|
||||
|
||||
async def content_ratings_to_reactions(entry, slug):
|
||||
try:
|
||||
with local_session() as session:
|
||||
for content_rating in entry.get("ratings", []):
|
||||
rater = (
|
||||
session.query(User)
|
||||
.filter(User.oid == content_rating["createdBy"])
|
||||
.first()
|
||||
) or User.default_user
|
||||
shout = session.query(Shout).where(Shout.slug == slug).first()
|
||||
cts = content_rating.get("createdAt")
|
||||
reaction_dict = {
|
||||
"createdAt": date_parse(cts) if cts else None,
|
||||
"kind": ReactionKind.LIKE
|
||||
if content_rating["value"] > 0
|
||||
else ReactionKind.DISLIKE,
|
||||
"createdBy": rater.id,
|
||||
"shout": shout.id,
|
||||
}
|
||||
reaction = (
|
||||
session.query(Reaction)
|
||||
.filter(Reaction.shout == reaction_dict["shout"])
|
||||
.filter(Reaction.createdBy == reaction_dict["createdBy"])
|
||||
.filter(Reaction.kind == reaction_dict["kind"])
|
||||
.first()
|
||||
)
|
||||
if reaction:
|
||||
k = (
|
||||
ReactionKind.AGREE
|
||||
if content_rating["value"] > 0
|
||||
else ReactionKind.DISAGREE
|
||||
)
|
||||
reaction_dict["kind"] = k
|
||||
reaction.update(reaction_dict)
|
||||
session.add(reaction)
|
||||
else:
|
||||
rea = Reaction.create(**reaction_dict)
|
||||
session.add(rea)
|
||||
# shout_dict['ratings'].append(reaction_dict)
|
||||
|
||||
session.commit()
|
||||
except Exception:
|
||||
print("[migration] content_item.ratings error: \n%r" % content_rating)
|
||||
@@ -1,34 +0,0 @@
|
||||
from services.db import local_session
|
||||
from migration.extract import extract_md
|
||||
from migration.html2text import html2text
|
||||
from orm.reaction import Reaction, ReactionKind
|
||||
|
||||
|
||||
def migrate(entry, storage):
|
||||
post_oid = entry["contentItem"]
|
||||
print(post_oid)
|
||||
shout_dict = storage["shouts"]["by_oid"].get(post_oid)
|
||||
if shout_dict:
|
||||
print(shout_dict["body"])
|
||||
remark = {
|
||||
"shout": shout_dict["id"],
|
||||
"body": extract_md(html2text(entry["body"]), shout_dict),
|
||||
"kind": ReactionKind.REMARK,
|
||||
}
|
||||
|
||||
if entry.get("textBefore"):
|
||||
remark["range"] = (
|
||||
str(shout_dict["body"].index(entry["textBefore"] or ""))
|
||||
+ ":"
|
||||
+ str(
|
||||
shout_dict["body"].index(entry["textAfter"] or "")
|
||||
+ len(entry["textAfter"] or "")
|
||||
)
|
||||
)
|
||||
|
||||
with local_session() as session:
|
||||
rmrk = Reaction.create(**remark)
|
||||
session.commit()
|
||||
del rmrk["_sa_instance_state"]
|
||||
return rmrk
|
||||
return
|
||||
@@ -1,828 +0,0 @@
|
||||
{
|
||||
"207": "207",
|
||||
"1990-e": "90s",
|
||||
"2000-e": "2000s",
|
||||
"90-e": "90s",
|
||||
"Georgia": "georgia",
|
||||
"Japan": "japan",
|
||||
"Sweden": "sweden",
|
||||
"abstraktsiya": "abstract",
|
||||
"absurdism": "absurdism",
|
||||
"acclimatization": "acclimatisation",
|
||||
"activism": "activism",
|
||||
"adolf-gitler": "adolf-hitler",
|
||||
"afrika": "africa",
|
||||
"agata-kristi": "agatha-christie",
|
||||
"agressivnoe-povedenie": "agression",
|
||||
"agressiya": "agression",
|
||||
"aktsii": "actions",
|
||||
"aktsionizm": "actionism",
|
||||
"alber-kamyu": "albert-kamus",
|
||||
"albomy": "albums",
|
||||
"aleksandr-griboedov": "aleksander-griboedov",
|
||||
"aleksandr-pushkin": "aleksander-pushkin",
|
||||
"aleksandr-solzhenitsyn": "aleksander-solzhenitsyn",
|
||||
"aleksandr-vvedenskiy": "aleksander-vvedensky",
|
||||
"aleksey-navalnyy": "alexey-navalny",
|
||||
"alfavit": "alphabet",
|
||||
"alkogol": "alcohol",
|
||||
"alternativa": "alternative",
|
||||
"alternative": "alternative",
|
||||
"alternativnaya-istoriya": "alternative-history",
|
||||
"amerika": "america",
|
||||
"anarhizm": "anarchism",
|
||||
"anatoliy-mariengof": "anatoly-mariengof",
|
||||
"ancient-russia": "ancient-russia",
|
||||
"andegraund": "underground",
|
||||
"andrey-platonov": "andrey-platonov",
|
||||
"andrey-rodionov": "andrey-rodionov",
|
||||
"andrey-tarkovskiy": "andrey-tarkovsky",
|
||||
"angliyskie-istorii": "english-stories",
|
||||
"angliyskiy-yazyk": "english-langugae",
|
||||
"ango": "ango",
|
||||
"animation": "animation",
|
||||
"animatsiya": "animation",
|
||||
"anime": "anime",
|
||||
"anri-volohonskiy": "anri-volohonsky",
|
||||
"antifashizm": "anti-faschism",
|
||||
"antiquity": "antiquity",
|
||||
"antiutopiya": "dystopia",
|
||||
"anton-dolin": "anton-dolin",
|
||||
"antropology": "antropology",
|
||||
"antropotsen": "antropocenus",
|
||||
"architecture": "architecture",
|
||||
"arheologiya": "archeology",
|
||||
"arhetipy": "archetypes",
|
||||
"arhiv": "archive",
|
||||
"aristokraty": "aristocracy",
|
||||
"aristotel": "aristotle",
|
||||
"arktika": "arctic",
|
||||
"armiya": "army",
|
||||
"armiya-1": "army",
|
||||
"art": "art",
|
||||
"art-is": "art-is",
|
||||
"artists": "artists",
|
||||
"ateizm": "atheism",
|
||||
"audio-poetry": "audio-poetry",
|
||||
"audiopoeziya": "audio-poetry",
|
||||
"audiospektakl": "audio-spectacles",
|
||||
"auktsyon": "auktsyon",
|
||||
"avangard": "avantgarde",
|
||||
"avtofikshn": "autofiction",
|
||||
"avtorskaya-pesnya": "bardsongs",
|
||||
"azbuka-immigratsii": "immigration-basics",
|
||||
"aziatskiy-kinematograf": "asian-cinema",
|
||||
"b-movie": "b-movie",
|
||||
"bannye-chteniya": "sauna-reading",
|
||||
"bardsongs": "bardsongs",
|
||||
"bdsm": "bdsm",
|
||||
"beecake": "beecake",
|
||||
"belarus": "belarus",
|
||||
"belgiya": "belgium",
|
||||
"bertold-breht": "berttold-brecht",
|
||||
"bezumie": "madness",
|
||||
"biography": "biography",
|
||||
"biologiya": "biology",
|
||||
"bipolyarnoe-rasstroystvo": "bipolar-disorder",
|
||||
"bitniki": "beatnics",
|
||||
"biznes": "business",
|
||||
"blizhniy-vostok": "middle-east",
|
||||
"blizost": "closeness",
|
||||
"blocked-in-russia": "blocked-in-russia",
|
||||
"blokada": "blockade",
|
||||
"bob-dilan": "bob-dylan",
|
||||
"bog": "god",
|
||||
"bol": "pain",
|
||||
"bolotnoe-delo": "bolotnaya-case",
|
||||
"books": "books",
|
||||
"boris-eltsin": "boris-eltsin",
|
||||
"boris-godunov": "boris-godunov",
|
||||
"boris-grebenschikov": "boris-grebenschikov",
|
||||
"boris-nemtsov": "boris-nemtsov",
|
||||
"boris-pasternak": "boris-pasternak",
|
||||
"brak": "marriage",
|
||||
"bret-iston-ellis": "bret-iston-ellis",
|
||||
"buddizm": "buddhism",
|
||||
"bullying": "bullying",
|
||||
"bunt": "riot",
|
||||
"burning-man": "burning-man",
|
||||
"bytie": "being",
|
||||
"byurokratiya": "bureaucracy",
|
||||
"capitalism": "capitalism",
|
||||
"censored-in-russia": "censored-in-russia",
|
||||
"ch-rno-beloe": "black-and-white",
|
||||
"ch-rnyy-yumor": "black-humour",
|
||||
"chapters": "chapters",
|
||||
"charity": "charity",
|
||||
"chayldfri": "childfree",
|
||||
"chechenskaya-voyna": "chechen-war",
|
||||
"chechnya": "chechnya",
|
||||
"chelovek": "male",
|
||||
"chernobyl": "chernobyl",
|
||||
"chernyy-yumor": "black-humour",
|
||||
"children": "children",
|
||||
"china": "china",
|
||||
"chinovniki": "bureaucracy",
|
||||
"chukotka": "chukotka",
|
||||
"chuma": "plague",
|
||||
"church": "church",
|
||||
"cinema": "cinema",
|
||||
"city": "city",
|
||||
"civil-position": "civil-position",
|
||||
"clips": "clips",
|
||||
"collage": "collage",
|
||||
"comics": "comics",
|
||||
"conspiracy-theory": "conspiracy-theory",
|
||||
"contemporary-art": "contemporary-art",
|
||||
"contemporary-poetry": "poetry",
|
||||
"contemporary-prose": "prose",
|
||||
"coronavirus": "coronavirus",
|
||||
"corruption": "corruption",
|
||||
"creative-writing-school": "creative-writing-school",
|
||||
"crime": "crime",
|
||||
"criticism": "criticism",
|
||||
"critiques": "reviews",
|
||||
"culture": "culture",
|
||||
"dadaizm": "dadaism",
|
||||
"daniel-defo": "daniel-defoe",
|
||||
"daniil-harms": "daniil-kharms",
|
||||
"dante-aligeri": "dante-alighieri",
|
||||
"darkveyv": "darkwave",
|
||||
"death": "death",
|
||||
"debaty": "debats",
|
||||
"delo-seti": "seti-case",
|
||||
"democracy": "democracy",
|
||||
"demografiya": "demographics",
|
||||
"demonstrations": "demonstrations",
|
||||
"depression": "depression",
|
||||
"derevnya": "village",
|
||||
"derrida": "derrida",
|
||||
"design": "design",
|
||||
"detskie-doma": "orphanages",
|
||||
"detstvo": "childhood",
|
||||
"devid-linch": "david-linch",
|
||||
"devyanostye": "90s",
|
||||
"dialog": "dialogue",
|
||||
"digital": "digital",
|
||||
"digital-art": "digital-art",
|
||||
"dinozavry": "dinosaurs",
|
||||
"directing": "directing",
|
||||
"diskurs": "discours",
|
||||
"diskurs-1": "discourse",
|
||||
"diskurs-analiz": "discourse-analytics",
|
||||
"dissidenty": "dissidents",
|
||||
"diy": "diy",
|
||||
"dmitriy-donskoy": "dmitriy-donskoy",
|
||||
"dmitriy-prigov": "dmitriy-prigov",
|
||||
"dnevnik-1": "dairy",
|
||||
"dnevniki": "dairies",
|
||||
"documentary": "documentary",
|
||||
"dokumentalnaya-poema": "documentary-poem",
|
||||
"dokumentalnaya-poeziya": "documentary-poetry",
|
||||
"dokumenty": "doсuments",
|
||||
"domashnee-nasilie": "home-terror",
|
||||
"donald-tramp": "donald-trump",
|
||||
"donbass": "donbass",
|
||||
"donbass-diary": "donbass-diary",
|
||||
"donorstvo": "donation",
|
||||
"dozhd": "rain",
|
||||
"drama": "drama",
|
||||
"dramaturgy": "dramaturgy",
|
||||
"drawing": "drawing",
|
||||
"drevo-zhizni": "tree-of-life",
|
||||
"drugs": "drugs",
|
||||
"duh": "spirit",
|
||||
"dzhaz": "jazz",
|
||||
"dzhek-keruak": "jack-keruak",
|
||||
"dzhim-morrison": "jim-morrison",
|
||||
"dzhordzh-romero": "george-romero",
|
||||
"dzhordzho-agamben": "giorgio-agamben",
|
||||
"ecology": "ecology",
|
||||
"economics": "economics",
|
||||
"eda": "food",
|
||||
"editorial-statements": "editorial-statements",
|
||||
"eduard-limonov": "eduard-limonov",
|
||||
"education": "education",
|
||||
"egor-letov": "egor-letov",
|
||||
"ekspat": "expat",
|
||||
"eksperiment": "experiments",
|
||||
"eksperimentalnaya-muzyka": "experimental-music",
|
||||
"ekspressionizm": "expressionism",
|
||||
"ekstremizm": "extremism",
|
||||
"ekzistentsializm-1": "existentialism",
|
||||
"ekzistentsiya": "existence",
|
||||
"elections": "elections",
|
||||
"electronic": "electronics",
|
||||
"electronics": "electronics",
|
||||
"elena-glinskaya": "elena-glinskaya",
|
||||
"elena-guro": "elena-guro",
|
||||
"elizaveta-mnatsakanova": "elizaveta-mnatsakanova",
|
||||
"embient": "ambient",
|
||||
"emigration": "emigration",
|
||||
"emil-dyurkgeym": "emile-durkheim",
|
||||
"emotsii": "emotions",
|
||||
"empiric": "empiric",
|
||||
"epidemiya": "pandemic",
|
||||
"erich-von-neff": "erich-von-neff",
|
||||
"erotika": "erotics",
|
||||
"essay": "essay",
|
||||
"estetika": "aestetics",
|
||||
"etika": "ethics",
|
||||
"etno": "ethno",
|
||||
"etnos": "ethnics",
|
||||
"everyday-life": "everyday-life",
|
||||
"evgeniy-onegin": "eugene-onegin",
|
||||
"evolyutsiya": "evolution",
|
||||
"exhibitions": "exhibitions",
|
||||
"experience": "experiences",
|
||||
"experimental": "experimental",
|
||||
"experimental-music": "experimental-music",
|
||||
"explanation": "explanation",
|
||||
"faktcheking": "fact-checking",
|
||||
"falsifikatsii": "falsifications",
|
||||
"family": "family",
|
||||
"fanfiki": "fan-fiction",
|
||||
"fantastika": "sci-fi",
|
||||
"fatalizm": "fatalism",
|
||||
"fedor-dostoevskiy": "fedor-dostoevsky",
|
||||
"fedor-ioannovich": "fedor-ioannovich",
|
||||
"feleton": "feuilleton",
|
||||
"feminism": "feminism",
|
||||
"fenomenologiya": "phenomenology",
|
||||
"fentezi": "fantasy",
|
||||
"festival": "festival",
|
||||
"festival-territoriya": "festival-territory",
|
||||
"folk": "folk",
|
||||
"folklor": "folklore",
|
||||
"fotoreportazh": "photoreports",
|
||||
"france": "france",
|
||||
"frants-kafka": "franz-kafka",
|
||||
"frederik-begbeder": "frederick-begbeder",
|
||||
"freedom": "freedom",
|
||||
"friendship": "friendship",
|
||||
"fsb": "fsb",
|
||||
"futbol": "footbool",
|
||||
"future": "future",
|
||||
"futuristy": "futurists",
|
||||
"futurizm": "futurism",
|
||||
"galereya": "gallery",
|
||||
"galereya-anna-nova": "gallery-anna-nova",
|
||||
"gdr": "gdr",
|
||||
"gender": "gender",
|
||||
"gendernyy-diskurs": "gender",
|
||||
"gennadiy-aygi": "gennadiy-aygi",
|
||||
"gerhard-rihter": "gerhard-rihter",
|
||||
"germaniya": "germany",
|
||||
"germenevtika": "hermeneutics",
|
||||
"geroi": "heroes",
|
||||
"girls": "girls",
|
||||
"gkchp": "gkchp",
|
||||
"glitch": "glitch",
|
||||
"globalizatsiya": "globalisation",
|
||||
"gollivud": "hollywood",
|
||||
"gonzo": "gonzo",
|
||||
"gore-ot-uma": "woe-from-wit",
|
||||
"graffiti": "graffiti",
|
||||
"graficheskaya-novella": "graphic-novell",
|
||||
"graphics": "graphics",
|
||||
"gravyura": "engraving",
|
||||
"grazhdanskaya-oborona": "grazhdanskaya-oborona",
|
||||
"gretsiya": "greece",
|
||||
"griby": "mushrooms",
|
||||
"gruziya-2": "georgia",
|
||||
"gulag": "gulag",
|
||||
"han-batyy": "khan-batyy",
|
||||
"hayku": "haiku",
|
||||
"health": "health",
|
||||
"himiya": "chemistry",
|
||||
"hip-hop": "hip-hop",
|
||||
"history": "history",
|
||||
"history-of-russia": "history-of-russia",
|
||||
"holokost": "holocaust",
|
||||
"horeografiya": "choreography",
|
||||
"horror": "horror",
|
||||
"hospis": "hospice",
|
||||
"hristianstvo": "christianity",
|
||||
"humans": "humans",
|
||||
"humour": "humour",
|
||||
"ideologiya": "ideology",
|
||||
"idm": "idm",
|
||||
"igil": "isis",
|
||||
"igor-pomerantsev": "igor-pomerantsev",
|
||||
"igra": "game",
|
||||
"igra-prestolov": "game-of-throne",
|
||||
"igry": "games",
|
||||
"iisus-hristos": "jesus-christ",
|
||||
"illness": "illness",
|
||||
"illustration-history": "illustration-history",
|
||||
"illustrations": "illustrations",
|
||||
"imazhinizm": "imagism",
|
||||
"immanuil-kant": "immanuel-kant",
|
||||
"impressionizm": "impressionism",
|
||||
"improvizatsiya": "improvisation",
|
||||
"indi": "indie",
|
||||
"individualizm": "individualism",
|
||||
"infografika": "infographics",
|
||||
"informatsiya": "information",
|
||||
"ingmar-bergman": "ingmar-bergman",
|
||||
"inklyuziya": "inclusion",
|
||||
"installyatsiya": "installation",
|
||||
"internet": "internet",
|
||||
"interview": "interview",
|
||||
"invalidnost": "disability",
|
||||
"investigations": "investigations",
|
||||
"iosif-brodskiy": "joseph-brodsky",
|
||||
"iosif-stalin": "joseph-stalin",
|
||||
"iskusstvennyy-intellekt": "artificial-intelligence",
|
||||
"islam": "islam",
|
||||
"istoriya-moskvy": "moscow-history",
|
||||
"istoriya-nauki": "history-of-sceince",
|
||||
"istoriya-o-medsestre": "nurse-story",
|
||||
"istoriya-teatra": "theatre-history",
|
||||
"italiya": "italy",
|
||||
"italyanskiy-yazyk": "italian-language",
|
||||
"iudaika": "judaica",
|
||||
"ivan-groznyy": "ivan-grozny",
|
||||
"ivan-iii-gorbatyy": "ivan-iii-gorbaty",
|
||||
"ivan-kalita": "ivan-kalita",
|
||||
"ivan-krylov": "ivan-krylov",
|
||||
"izobreteniya": "inventions",
|
||||
"izrail-1": "israel",
|
||||
"jazz": "jazz",
|
||||
"john-lennon": "john-lennon",
|
||||
"journalism": "journalism",
|
||||
"justice": "justice",
|
||||
"k-pop": "k-pop",
|
||||
"kalligrafiya": "calligraphy",
|
||||
"karikatura": "caricatures",
|
||||
"kartochki-rubinshteyna": "rubinstein-cards",
|
||||
"katrin-nenasheva": "katrin-nenasheva",
|
||||
"kavarga": "kavarga",
|
||||
"kavkaz": "caucasus",
|
||||
"kazan": "kazan",
|
||||
"kiberbezopasnost": "cybersecurity",
|
||||
"kinoklub": "cinema-club",
|
||||
"kinokritika": "film-criticism",
|
||||
"kirill-serebrennikov": "kirill-serebrennikov",
|
||||
"kladbische": "cemetery",
|
||||
"klassika": "classic",
|
||||
"kollektivnoe-bessoznatelnoe": "сollective-unconscious",
|
||||
"komediya": "comedy",
|
||||
"kommunikatsii": "communications",
|
||||
"kommunizm": "communism",
|
||||
"kommuny": "communes",
|
||||
"kompyuternye-igry": "computer-games",
|
||||
"konets-vesny": "end-of-spring",
|
||||
"konservatizm": "conservatism",
|
||||
"kontrkultura": "counter-culture",
|
||||
"kontseptualizm": "conceptualism",
|
||||
"korotkometrazhka": "cinema-shorts",
|
||||
"kosmos": "cosmos",
|
||||
"kraudfanding": "crowdfunding",
|
||||
"kriptovalyuty": "cryptocurrencies",
|
||||
"krizis": "crisis",
|
||||
"krov": "blood",
|
||||
"krym": "crimea",
|
||||
"kulturologiya": "culturology",
|
||||
"kulty": "cults",
|
||||
"kurdistan": "kurdistan",
|
||||
"kurt-kobeyn": "kurt-cobain",
|
||||
"kurt-vonnegut": "kurt-vonnegut",
|
||||
"kvir": "queer",
|
||||
"laboratoriya": "lab",
|
||||
"language": "languages",
|
||||
"lars-fon-trier": "lars-fon-trier",
|
||||
"laws": "laws",
|
||||
"lectures": "lectures",
|
||||
"leto": "summer",
|
||||
"lev-tolstoy": "leo-tolstoy",
|
||||
"lgbt": "lgbt",
|
||||
"liberalizm": "liberalism",
|
||||
"libertarianstvo": "libertarianism",
|
||||
"life": "life",
|
||||
"likbez": "likbez",
|
||||
"lingvistika": "linguistics",
|
||||
"lirika": "lirics",
|
||||
"literary-studies": "literary-studies",
|
||||
"literature": "literature",
|
||||
"literaturnyykaver": "literature-cover",
|
||||
"lo-fi": "lo-fi",
|
||||
"lomonosov": "lomonosov",
|
||||
"love": "love",
|
||||
"luzha-goluboy-krovi": "luzha-goluboy-krovi",
|
||||
"lyudvig-vitgenshteyn": "ludwig-wittgenstein",
|
||||
"lzhedmitriy": "false-dmitry",
|
||||
"lzhenauka": "pseudoscience",
|
||||
"magiya": "magic",
|
||||
"maks-veber": "max-weber",
|
||||
"manifests": "manifests",
|
||||
"manipulyatsii-soznaniem": "mind-manipulation",
|
||||
"marina-abramovich": "marina-abramovich",
|
||||
"marketing": "marketing",
|
||||
"marksizm": "marxism",
|
||||
"marsel-dyushan": "marchel-duchamp",
|
||||
"marsel-prust": "marcel-proust",
|
||||
"martin-haydegger": "martin-hidegger",
|
||||
"matematika": "maths",
|
||||
"mayakovskiy": "vladimir-mayakovsky",
|
||||
"media": "media",
|
||||
"medicine": "medicine",
|
||||
"memuary": "memoirs",
|
||||
"menedzhment": "management",
|
||||
"menty": "police",
|
||||
"merab-mamardashvili": "merab-mamardashvili",
|
||||
"mest": "revenge",
|
||||
"metamodernizm": "metamodern",
|
||||
"metavselennaya": "metaverse",
|
||||
"metro": "metro",
|
||||
"mifologiya": "mythology",
|
||||
"mify": "myth",
|
||||
"mihael-haneke": "michael-haneke",
|
||||
"mihail-baryshnikov": "mihail-baryshnikov",
|
||||
"mihail-bulgakov": "mihail-bulgakov",
|
||||
"mikrotonalnaya-muzyka": "mikrotone-muzyka",
|
||||
"minimalizm": "minimalism",
|
||||
"minkult-privet": "minkult-privet",
|
||||
"mir": "world",
|
||||
"mirovozzrenie": "mindsets",
|
||||
"mishel-fuko": "michel-foucault",
|
||||
"mistika": "mystics",
|
||||
"mitropolit-makariy": "mitropolit-makariy",
|
||||
"mlm": "mlm",
|
||||
"mobilizatsiya": "mobilisation",
|
||||
"moda": "fashion",
|
||||
"modernizm": "modernism",
|
||||
"mokyumentari": "mockumentary",
|
||||
"molodezh": "youth",
|
||||
"moloko-plus": "moloko-plus",
|
||||
"money": "money",
|
||||
"monologs": "monologues",
|
||||
"monstratsiya": "monstration",
|
||||
"moralnaya-otvetstvennost": "moral-responsibility",
|
||||
"more": "sea",
|
||||
"moscow": "moscow",
|
||||
"moshennichestvo": "frauds",
|
||||
"moskovskiy-romanticheskiy-kontseptualizm": "moscow-romantic-conceptualism",
|
||||
"moskovskoe-delo": "moscow-case",
|
||||
"movies": "movies",
|
||||
"mozg": "brain",
|
||||
"multiplikatsiya": "animation",
|
||||
"music": "music",
|
||||
"musulmanstvo": "islam",
|
||||
"muzei": "museum",
|
||||
"muzey": "museum",
|
||||
"muzhchiny": "man",
|
||||
"myshlenie": "thinking",
|
||||
"nagornyy-karabah": "nagorno-karabakh",
|
||||
"nasilie-1": "violence",
|
||||
"natsionalizm": "nationalism",
|
||||
"natsionalnaya-ideya": "national-idea",
|
||||
"natsizm": "nazism",
|
||||
"natyurmort": "nature-morte",
|
||||
"nauchpop": "pop-science",
|
||||
"nbp": "nbp",
|
||||
"nenavist": "hate",
|
||||
"neofitsialnaya-literatura": "unofficial-literature",
|
||||
"neoklassika": "neoclassic",
|
||||
"neprozrachnye-smysly": "hidden-meanings",
|
||||
"neravenstvo": "inequality",
|
||||
"net-voyne": "no-war",
|
||||
"new-year": "new-year",
|
||||
"neyronauka": "neuro-science",
|
||||
"neyroseti": "neural-networks",
|
||||
"niu-vshe": "hse",
|
||||
"nizhniy-novgorod": "nizhny-novgorod",
|
||||
"nko": "nonprofits",
|
||||
"nlo": "ufo",
|
||||
"nobelevskaya-premiya": "nobel-prize",
|
||||
"noize-mc": "noize-mc",
|
||||
"nonkonformizm": "nonconformism",
|
||||
"notforall": "notforall",
|
||||
"novaya-drama": "new-drama",
|
||||
"novosti": "news",
|
||||
"noyz": "noise",
|
||||
"nuar": "noir",
|
||||
"oberiu": "oberiu",
|
||||
"ocherk": "etudes",
|
||||
"ochevidnyy-nuar": "ochevidnyy-nuar",
|
||||
"odinochestvo": "loneliness",
|
||||
"odna-kniga-odna-istoriya": "one-book-one-story",
|
||||
"okrainy": "outskirts",
|
||||
"omon": "swat",
|
||||
"opinions": "opinions",
|
||||
"oppozitsiya": "opposition",
|
||||
"orhan-pamuk": "orhan-pamuk",
|
||||
"ornitologiya": "ornitology",
|
||||
"osen": "autumn",
|
||||
"osip-mandelshtam": "osip-mandelshtam",
|
||||
"oskar-uayld": "oscar-wilde",
|
||||
"osoznanie": "awareness",
|
||||
"otnosheniya": "relationship",
|
||||
"pablo-pikasso": "pablo-picasso",
|
||||
"painting": "painting",
|
||||
"paintings": "painting",
|
||||
"pamyat": "memory",
|
||||
"pandemiya": "pandemic",
|
||||
"parizh": "paris",
|
||||
"patriotizm": "patriotism",
|
||||
"patsifizm": "pacifism",
|
||||
"paul-tselan": "paul-tselan",
|
||||
"per-burd": "pierre-bourdieu",
|
||||
"perezhivaniya": "worries",
|
||||
"performance": "performance",
|
||||
"peyzazh": "landscape",
|
||||
"philology": "philology",
|
||||
"philosophy": "philosophy",
|
||||
"photo": "photography",
|
||||
"photography": "photography",
|
||||
"photoprojects": "photoprojects",
|
||||
"plakaty": "posters",
|
||||
"plastilin": "plasticine",
|
||||
"plays": "plays",
|
||||
"podrostki": "teenagers",
|
||||
"poema": "poem",
|
||||
"poems": "poems",
|
||||
"poeticheskaya-proza": "poetic-prose",
|
||||
"poetry": "poetry",
|
||||
"poetry-of-squares": "poetry-of-squares",
|
||||
"poetry-slam": "poetry-slam",
|
||||
"pokoy": "peace",
|
||||
"police": "police",
|
||||
"politicheskoe-fentezi": "political-fantasy",
|
||||
"politics": "politics",
|
||||
"politzaklyuchennye": "political-prisoners",
|
||||
"polsha": "poland",
|
||||
"pomosch": "help",
|
||||
"pop-art": "pop-art",
|
||||
"pop-culture": "pop-culture",
|
||||
"populyarnaya-psihologiya": "popular-psychology",
|
||||
"pornografiya": "pornography",
|
||||
"portret": "portrait",
|
||||
"poslovitsy": "proverbs",
|
||||
"post-pank": "post-punk",
|
||||
"post-rok": "post-rock",
|
||||
"postmodernism": "postmodernism",
|
||||
"povest": "novells",
|
||||
"povsednevnost": "everyday-life",
|
||||
"power": "power",
|
||||
"pravo": "right",
|
||||
"pravoslavie": "orthodox",
|
||||
"pravozaschitniki": "human-rights-activism",
|
||||
"prazdnik": "holidays",
|
||||
"predatelstvo": "betrayal",
|
||||
"predprinimatelstvo": "entrepreneurship",
|
||||
"premera": "premier",
|
||||
"premiya-oskar": "oscar-prize",
|
||||
"pribaltika-1": "baltic",
|
||||
"priroda": "nature",
|
||||
"prison": "prison",
|
||||
"pritcha": "parable",
|
||||
"privatnost": "privacy",
|
||||
"progress": "progress",
|
||||
"projects": "projects",
|
||||
"prokrastinatsiya": "procrastination",
|
||||
"propaganda": "propaganda",
|
||||
"proschenie": "forgiveness",
|
||||
"prose": "prose",
|
||||
"proshloe": "past",
|
||||
"prostitutsiya": "prostitution",
|
||||
"prosveschenie": "enlightenment",
|
||||
"protests": "protests",
|
||||
"psalmy": "psalms",
|
||||
"psihoanaliz": "psychoanalysis",
|
||||
"psihodeliki": "psychodelics",
|
||||
"pskov": "pskov",
|
||||
"psychiatry": "psychiatry",
|
||||
"psychology": "psychology",
|
||||
"ptitsy": "birds",
|
||||
"punk": "punk",
|
||||
"r-b": "rnb",
|
||||
"rasizm": "racism",
|
||||
"realizm": "realism",
|
||||
"redaktura": "editing",
|
||||
"refleksiya": "reflection",
|
||||
"reggi": "reggae",
|
||||
"religion": "religion",
|
||||
"rene-zhirar": "rene-girard",
|
||||
"renesanss": "renessance",
|
||||
"renovatsiya": "renovation",
|
||||
"rep": "rap",
|
||||
"reportage": "reportage",
|
||||
"reportazh-1": "reportage",
|
||||
"repressions": "repressions",
|
||||
"research": "research",
|
||||
"retroveyv": "retrowave",
|
||||
"review": "review",
|
||||
"revolution": "revolution",
|
||||
"rezo-gabriadze": "rezo-gabriadze",
|
||||
"risunki": "painting",
|
||||
"roboty": "robots",
|
||||
"rock": "rock",
|
||||
"roditeli": "parents",
|
||||
"romantizm": "romantism",
|
||||
"romany": "novell",
|
||||
"ronald-reygan": "ronald-reygan",
|
||||
"roskomnadzor": "roskomnadzor",
|
||||
"rossiyskoe-kino": "russian-cinema",
|
||||
"rouling": "rowling",
|
||||
"rozhava": "rojava",
|
||||
"rpts": "rpts",
|
||||
"rus-na-grani-sryva": "rus-na-grani-sryva",
|
||||
"russia": "russia",
|
||||
"russian-language": "russian-language",
|
||||
"russian-literature": "russian-literature",
|
||||
"russkaya-toska": "russian-toska",
|
||||
"russkiy-mir": "russkiy-mir",
|
||||
"salo": "lard",
|
||||
"salvador-dali": "salvador-dali",
|
||||
"samoidentifikatsiya": "self-identity",
|
||||
"samoopredelenie": "self-definition",
|
||||
"sankt-peterburg": "saint-petersburg",
|
||||
"sasha-skochilenko": "sasha-skochilenko",
|
||||
"satira": "satiric",
|
||||
"saund-art": "sound-art",
|
||||
"schaste": "happiness",
|
||||
"school": "school",
|
||||
"science": "science",
|
||||
"sculpture": "sculpture",
|
||||
"second-world-war": "second-world-war",
|
||||
"sekond-hend": "second-hand",
|
||||
"seksprosvet": "sex-education",
|
||||
"seksualizirovannoe-nasilie": "sexualized-violence",
|
||||
"seksualnoe-nasilie": "sexualized-violence",
|
||||
"sekty": "sects",
|
||||
"semi": "semi",
|
||||
"semiotics": "semiotics",
|
||||
"serbiya": "serbia",
|
||||
"sergey-bodrov-mladshiy": "sergey-bodrov-junior",
|
||||
"sergey-solov-v": "sergey-solovyov",
|
||||
"serialy": "series",
|
||||
"sever": "north",
|
||||
"severnaya-koreya": "north-korea",
|
||||
"sex": "sex",
|
||||
"shotlandiya": "scotland",
|
||||
"shugeyz": "shoegaze",
|
||||
"siloviki": "siloviki",
|
||||
"simeon-bekbulatovich": "simeon-bekbulatovich",
|
||||
"simvolizm": "simbolism",
|
||||
"siriya": "siria",
|
||||
"skulptura": "sculpture",
|
||||
"slavoy-zhizhek": "slavoj-zizek",
|
||||
"smert-1": "death",
|
||||
"smysl": "meaning",
|
||||
"sny": "dreams",
|
||||
"sobytiya": "events",
|
||||
"social": "society",
|
||||
"society": "society",
|
||||
"sociology": "sociology",
|
||||
"sofya-paleolog": "sofya-paleolog",
|
||||
"sofya-vitovtovna": "sofya-vitovtovna",
|
||||
"soobschestva": "communities",
|
||||
"soprotivlenie": "resistence",
|
||||
"sotsializm": "socialism",
|
||||
"sotsialnaya-filosofiya": "social-philosophy",
|
||||
"sotsiologiya-1": "sociology",
|
||||
"sotsseti": "social-networks",
|
||||
"sotvorenie-tretego-rima": "third-rome",
|
||||
"sovremennost": "modernity",
|
||||
"spaces": "spaces",
|
||||
"spektakl": "spectacles",
|
||||
"spetseffekty": "special-fx",
|
||||
"spetsoperatsiya": "special-operation",
|
||||
"spetssluzhby": "special-services",
|
||||
"sport": "sport",
|
||||
"srednevekove": "middle-age",
|
||||
"state": "state",
|
||||
"statistika": "statistics",
|
||||
"stendap": "stand-up",
|
||||
"stihi": "poetry",
|
||||
"stoitsizm": "stoicism",
|
||||
"stories": "stories",
|
||||
"stoyanie-na-ugre": "stoyanie-na-ugre",
|
||||
"strah": "fear",
|
||||
"street-art": "street-art",
|
||||
"stsenarii": "scenarios",
|
||||
"sud": "court",
|
||||
"summary": "summary",
|
||||
"supergeroi": "superheroes",
|
||||
"svetlana-aleksievich": "svetlana-aleksievich",
|
||||
"svobodu-ivanu-golunovu": "free-ivan-golunov",
|
||||
"syurrealizm": "surrealism",
|
||||
"tales": "tales",
|
||||
"tanets": "dance",
|
||||
"tataro-mongolskoe-igo": "mongol-tatar-yoke",
|
||||
"tatuirovki": "tattoo",
|
||||
"technology": "technology",
|
||||
"televidenie": "television",
|
||||
"telo": "body",
|
||||
"telo-kak-iskusstvo": "body-as-art",
|
||||
"terrorizm": "terrorism",
|
||||
"tests": "tests",
|
||||
"text": "texts",
|
||||
"the-beatles": "the-beatles",
|
||||
"theater": "theater",
|
||||
"theory": "theory",
|
||||
"tokio": "tokio",
|
||||
"torture": "torture",
|
||||
"totalitarizm": "totalitarism",
|
||||
"traditions": "traditions",
|
||||
"tragicomedy": "tragicomedy",
|
||||
"transgendernost": "transgender",
|
||||
"translation": "translation",
|
||||
"transport": "transport",
|
||||
"travel": "travel",
|
||||
"travma": "trauma",
|
||||
"trendy": "trends",
|
||||
"tretiy-reyh": "third-reich",
|
||||
"triller": "thriller",
|
||||
"tsar": "central-african-republic",
|
||||
"tsar-edip": "oedipus",
|
||||
"tsarevich-dmitriy": "tsarevich-dmitry",
|
||||
"tsennosti": "values",
|
||||
"tsenzura": "censorship",
|
||||
"tseremonii": "ceremonies",
|
||||
"turizm": "tourism",
|
||||
"tvorchestvo": "creativity",
|
||||
"ugnetennyy-zhilischnyy-klass": "oppressed-housing-class",
|
||||
"uilyam-shekspir": "william-shakespeare",
|
||||
"ukraina-2": "ukraine",
|
||||
"ukraine": "ukraine",
|
||||
"university": "university",
|
||||
"urban-studies": "urban-studies",
|
||||
"uroki-literatury": "literature-lessons",
|
||||
"usa": "usa",
|
||||
"ussr": "ussr",
|
||||
"utopiya": "utopia",
|
||||
"utrata": "loss",
|
||||
"valter-benyamin": "valter-benyamin",
|
||||
"varlam-shalamov": "varlam-shalamov",
|
||||
"vasiliy-ii-temnyy": "basil-ii-temnyy",
|
||||
"vasiliy-iii": "basil-iii",
|
||||
"vdnh": "vdnh",
|
||||
"vechnost": "ethernety",
|
||||
"velikobritaniya": "great-britain",
|
||||
"velimir-hlebnikov": "velimir-hlebnikov",
|
||||
"velkom-tu-greyt-britn": "welcome-to-great-britain",
|
||||
"venedikt-erofeev": "venedikt-erofeev",
|
||||
"venetsiya": "veneece",
|
||||
"vengriya": "hungary",
|
||||
"verlibry": "free-verse",
|
||||
"veschi": "things",
|
||||
"vessels": "vessels",
|
||||
"veterany": "veterans",
|
||||
"video": "video",
|
||||
"videoart": "videoart",
|
||||
"videoklip": "clips",
|
||||
"videopoeziya": "video-poetry",
|
||||
"viktor-astafev": "viktor-astafev",
|
||||
"viktor-pelevin": "viktor-pelevin",
|
||||
"vilgelm-rayh": "wilhelm-reich",
|
||||
"vinzavod": "vinzavod",
|
||||
"violence": "violence",
|
||||
"visual-culture": "visual-culture",
|
||||
"vizualnaya-poeziya": "visual-poetry",
|
||||
"vladimir-lenin": "vladimir-lenin",
|
||||
"vladimir-mayakovskiy": "vladimir-mayakovsky",
|
||||
"vladimir-nabokov": "vladimir-nabokov",
|
||||
"vladimir-putin": "vladimir-putin",
|
||||
"vladimir-sorokin": "vladimir-sorokin",
|
||||
"vladimir-voynovich": "vladimir-voynovich",
|
||||
"vnutrenniy-opyt": "inner-expirience",
|
||||
"volga": "volga",
|
||||
"volontery": "volonteurs",
|
||||
"vong-karvay": "wong-karwai",
|
||||
"vospominaniya": "memories",
|
||||
"vostok": "east",
|
||||
"voyna-na-ukraine": "war-in-ukraine",
|
||||
"voyna-v-ukraine": "war-in-ukraine",
|
||||
"vremya": "time",
|
||||
"vudi-allen": "woody-allen",
|
||||
"vynuzhdennye-otnosheniya": "forced-relationship",
|
||||
"war": "war",
|
||||
"war-in-ukraine-images": "war-in-ukrahine-images",
|
||||
"women": "women",
|
||||
"work": "work",
|
||||
"writers": "writers",
|
||||
"xx-century": "xx-century",
|
||||
"yakob-yordans": "yakob-yordans",
|
||||
"yan-vermeer": "yan-vermeer",
|
||||
"yanka-dyagileva": "yanka-dyagileva",
|
||||
"yaponskaya-literatura": "japan-literature",
|
||||
"yazychestvo": "paganism",
|
||||
"youth": "youth",
|
||||
"yozef-rot": "yozef-rot",
|
||||
"yurgen-habermas": "jorgen-habermas",
|
||||
"za-liniey-mannergeyma": "behind-mannerheim-line",
|
||||
"zabota": "care",
|
||||
"zahar-prilepin": "zahar-prilepin",
|
||||
"zakonodatelstvo": "laws",
|
||||
"zakony-mira": "world-laws",
|
||||
"zametki": "notes",
|
||||
"zhelanie": "wish",
|
||||
"zhivotnye": "animals",
|
||||
"zhoze-saramago": "jose-saramago",
|
||||
"zigmund-freyd": "sigmund-freud",
|
||||
"zolotaya-orda": "golden-horde",
|
||||
"zombi": "zombie",
|
||||
"zombi-simpsony": "zombie-simpsons"
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
from services.db import local_session
|
||||
from migration.extract import extract_md
|
||||
from migration.html2text import html2text
|
||||
from orm import Topic
|
||||
|
||||
|
||||
def migrate(entry):
|
||||
body_orig = entry.get("description", "").replace(" ", " ")
|
||||
topic_dict = {
|
||||
"slug": entry["slug"],
|
||||
"oid": entry["_id"],
|
||||
"title": entry["title"].replace(" ", " "),
|
||||
"body": extract_md(html2text(body_orig)),
|
||||
}
|
||||
|
||||
with local_session() as session:
|
||||
slug = topic_dict["slug"]
|
||||
topic = session.query(Topic).filter(Topic.slug == slug).first() or Topic.create(
|
||||
**topic_dict
|
||||
)
|
||||
if not topic:
|
||||
raise Exception("no topic!")
|
||||
if topic:
|
||||
if len(topic.title) > len(topic_dict["title"]):
|
||||
Topic.update(topic, {"title": topic_dict["title"]})
|
||||
if len(topic.body) < len(topic_dict["body"]):
|
||||
Topic.update(topic, {"body": topic_dict["body"]})
|
||||
session.commit()
|
||||
# print(topic.__dict__)
|
||||
rt = topic.__dict__.copy()
|
||||
del rt["_sa_instance_state"]
|
||||
return rt
|
||||
@@ -1,167 +0,0 @@
|
||||
import re
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from dateutil.parser import parse
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from services.db import local_session
|
||||
from orm.user import AuthorFollower, User, UserRating
|
||||
|
||||
|
||||
def migrate(entry):
|
||||
if "subscribedTo" in entry:
|
||||
del entry["subscribedTo"]
|
||||
email = entry["emails"][0]["address"]
|
||||
user_dict = {
|
||||
"oid": entry["_id"],
|
||||
"roles": [],
|
||||
"ratings": [],
|
||||
"username": email,
|
||||
"email": email,
|
||||
"createdAt": parse(entry["createdAt"]),
|
||||
"emailConfirmed": ("@discours.io" in email)
|
||||
or bool(entry["emails"][0]["verified"]),
|
||||
"muted": False, # amnesty
|
||||
"bio": entry["profile"].get("bio", ""),
|
||||
"links": [],
|
||||
"name": "anonymous",
|
||||
"password": entry["services"]["password"].get("bcrypt"),
|
||||
}
|
||||
|
||||
if "updatedAt" in entry:
|
||||
user_dict["updatedAt"] = parse(entry["updatedAt"])
|
||||
if "wasOnineAt" in entry:
|
||||
user_dict["lastSeen"] = parse(entry["wasOnlineAt"])
|
||||
if entry.get("profile"):
|
||||
# slug
|
||||
slug = entry["profile"].get("path").lower()
|
||||
slug = re.sub("[^0-9a-zA-Z]+", "-", slug).strip()
|
||||
user_dict["slug"] = slug
|
||||
bio = (
|
||||
(entry.get("profile", {"bio": ""}).get("bio") or "")
|
||||
.replace("\(", "(")
|
||||
.replace("\)", ")")
|
||||
)
|
||||
bio_text = BeautifulSoup(bio, features="lxml").text
|
||||
|
||||
if len(bio_text) > 120:
|
||||
user_dict["about"] = bio_text
|
||||
else:
|
||||
user_dict["bio"] = bio_text
|
||||
|
||||
# userpic
|
||||
try:
|
||||
user_dict["userpic"] = (
|
||||
"https://assets.discours.io/unsafe/100x/"
|
||||
+ entry["profile"]["thumborId"]
|
||||
)
|
||||
except KeyError:
|
||||
try:
|
||||
user_dict["userpic"] = entry["profile"]["image"]["url"]
|
||||
except KeyError:
|
||||
user_dict["userpic"] = ""
|
||||
|
||||
# name
|
||||
fn = entry["profile"].get("firstName", "")
|
||||
ln = entry["profile"].get("lastName", "")
|
||||
name = fn if fn else ""
|
||||
name = (name + " " + ln) if ln else name
|
||||
if not name:
|
||||
name = slug if slug else "anonymous"
|
||||
name = (
|
||||
entry["profile"]["path"].lower().strip().replace(" ", "-")
|
||||
if len(name) < 2
|
||||
else name
|
||||
)
|
||||
user_dict["name"] = name
|
||||
|
||||
# links
|
||||
fb = entry["profile"].get("facebook", False)
|
||||
if fb:
|
||||
user_dict["links"].append(fb)
|
||||
vk = entry["profile"].get("vkontakte", False)
|
||||
if vk:
|
||||
user_dict["links"].append(vk)
|
||||
tr = entry["profile"].get("twitter", False)
|
||||
if tr:
|
||||
user_dict["links"].append(tr)
|
||||
ws = entry["profile"].get("website", False)
|
||||
if ws:
|
||||
user_dict["links"].append(ws)
|
||||
|
||||
# some checks
|
||||
if not user_dict["slug"] and len(user_dict["links"]) > 0:
|
||||
user_dict["slug"] = user_dict["links"][0].split("/")[-1]
|
||||
|
||||
user_dict["slug"] = user_dict.get("slug", user_dict["email"].split("@")[0])
|
||||
oid = user_dict["oid"]
|
||||
user_dict["slug"] = user_dict["slug"].lower().strip().replace(" ", "-")
|
||||
try:
|
||||
user = User.create(**user_dict.copy())
|
||||
except IntegrityError:
|
||||
print("[migration] cannot create user " + user_dict["slug"])
|
||||
with local_session() as session:
|
||||
old_user = (
|
||||
session.query(User).filter(User.slug == user_dict["slug"]).first()
|
||||
)
|
||||
old_user.oid = oid
|
||||
old_user.password = user_dict["password"]
|
||||
session.commit()
|
||||
user = old_user
|
||||
if not user:
|
||||
print("[migration] ERROR: cannot find user " + user_dict["slug"])
|
||||
raise Exception
|
||||
user_dict["id"] = user.id
|
||||
return user_dict
|
||||
|
||||
|
||||
def post_migrate():
|
||||
old_discours_dict = {
|
||||
"slug": "old-discours",
|
||||
"username": "old-discours",
|
||||
"email": "old@discours.io",
|
||||
"name": "Просмотры на старой версии сайта",
|
||||
}
|
||||
|
||||
with local_session() as session:
|
||||
old_discours_user = User.create(**old_discours_dict)
|
||||
session.add(old_discours_user)
|
||||
session.commit()
|
||||
|
||||
|
||||
def migrate_2stage(entry, id_map):
|
||||
ce = 0
|
||||
for rating_entry in entry.get("ratings", []):
|
||||
rater_oid = rating_entry["createdBy"]
|
||||
rater_slug = id_map.get(rater_oid)
|
||||
if not rater_slug:
|
||||
ce += 1
|
||||
# print(rating_entry)
|
||||
continue
|
||||
oid = entry["_id"]
|
||||
author_slug = id_map.get(oid)
|
||||
|
||||
with local_session() as session:
|
||||
try:
|
||||
rater = session.query(User).where(User.slug == rater_slug).one()
|
||||
user = session.query(User).where(User.slug == author_slug).one()
|
||||
|
||||
user_rating_dict = {
|
||||
"value": rating_entry["value"],
|
||||
"rater": rater.id,
|
||||
"user": user.id,
|
||||
}
|
||||
|
||||
user_rating = UserRating.create(**user_rating_dict)
|
||||
if user_rating_dict["value"] > 0:
|
||||
af = AuthorFollower.create(
|
||||
author=user.id, follower=rater.id, auto=True
|
||||
)
|
||||
session.add(af)
|
||||
session.add(user_rating)
|
||||
session.commit()
|
||||
except IntegrityError:
|
||||
print("[migration] cannot rate " + author_slug + "`s by " + rater_slug)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return ce
|
||||
Reference in New Issue
Block a user