discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

model changes

Browse Source
This commit is contained in:
Igor Lobanov
2022-11-29 13:36:46 +01:00
parent b840823fce
commit 226aeddecd
20 changed files with 254 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
migration/__init__.py
View File

@@ -119,8 +119,7 @@ async def shouts_handle(storage, args):
# print main counter
counter += 1
line = str(counter + 1) + ": " + shout["slug"] + " @" + author["slug"]
print(line)
print('[migration] shouts_handle %d: %s @%s' % ((counter + 1), shout["slug"], author["slug"]))
b = bs4.BeautifulSoup(shout["body"], "html.parser")
texts = [shout["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")]

67
migration/tables/comments.py
View File

@@ -5,8 +5,8 @@ from dateutil.parser import parse as date_parse
from base.orm import local_session
from migration.html2text import html2text
from orm.reaction import Reaction, ReactionKind
from orm.shout import ShoutReactionsFollower
from orm.topic import TopicFollower
from orm.shout import ShoutReactionsFollower, Shout
from orm.topic import TopicFollower, Topic
from orm.user import User
ts = datetime.now(tz=timezone.utc)
@@ -69,8 +69,13 @@ async def migrate(entry, storage):
author = session.query(User).filter(User.oid == entry["createdBy"]).first()
shout_dict = storage["shouts"]["by_oid"][shout_oid]
if shout_dict:
reaction_dict["shout"] = shout_dict["slug"]
reaction_dict["createdBy"] = author.slug if author else "discours"
shout = session.query(
Shout
).where(Shout.slug == shout_dict["slug"]).one()
reaction_dict["shout_id"] = shout.id
reaction_dict["createdBy"] = author.id if author else 1
reaction_dict["kind"] = ReactionKind.COMMENT
# creating reaction from old comment
@@ -80,15 +85,20 @@ async def migrate(entry, storage):
# creating shout's reactions following for reaction author
following1 = session.query(
ShoutReactionsFollower
).join(
User
).join(
Shout
).where(
ShoutReactionsFollower.follower == reaction_dict["createdBy"]
User.id == reaction_dict["createdBy"]
).filter(
ShoutReactionsFollower.shout == reaction.shout
ShoutReactionsFollower.shout_id == reaction.shout_id
).first()
if not following1:
following1 = ShoutReactionsFollower.create(
follower=reaction_dict["createdBy"],
shout=reaction.shout,
follower_id=reaction_dict["createdBy"],
shout_id=reaction.shout_id,
auto=True
)
session.add(following1)
@@ -97,15 +107,22 @@ async def migrate(entry, storage):
for t in shout_dict["topics"]:
tf = session.query(
TopicFollower
).join(
Topic
).where(
TopicFollower.follower == reaction_dict["createdBy"]
TopicFollower.follower_id == reaction_dict["createdBy"]
).filter(
TopicFollower.topic == t
Topic.slug == t
).first()
if not tf:
topic = session.query(
Topic
).where(Topic.slug == t).one()
topic_following = TopicFollower.create(
follower=reaction_dict["createdBy"],
topic=t,
follower_id=reaction_dict["createdBy"],
topic_id=topic.id,
auto=True
)
session.add(topic_following)
@@ -114,16 +131,16 @@ async def migrate(entry, storage):
for comment_rating_old in entry.get("ratings", []):
rater = (
session.query(User)
.filter(User.oid == comment_rating_old["createdBy"])
.first()
.filter(User.oid == comment_rating_old["createdBy"])
.first()
)
re_reaction_dict = {
"shout": reaction_dict["shout"],
"shout_id": reaction_dict["shout_id"],
"replyTo": reaction.id,
"kind": ReactionKind.LIKE
if comment_rating_old["value"] > 0
else ReactionKind.DISLIKE,
"createdBy": rater.slug if rater else "discours",
"createdBy": rater.id if rater else 1,
}
cts = comment_rating_old.get("createdAt")
if cts:
@@ -134,14 +151,14 @@ async def migrate(entry, storage):
following2 = session.query(
ShoutReactionsFollower
).where(
ShoutReactionsFollower.follower == re_reaction_dict['createdBy']
ShoutReactionsFollower.follower_id == re_reaction_dict['createdBy']
).filter(
ShoutReactionsFollower.shout == rr.shout
ShoutReactionsFollower.shout_id == rr.shout_id
).first()
if not following2:
following2 = ShoutReactionsFollower.create(
follower=re_reaction_dict['createdBy'],
shout=rr.shout,
follower_id=re_reaction_dict['createdBy'],
shout_id=rr.shout_id,
auto=True
)
session.add(following2)
@@ -163,22 +180,26 @@ def migrate_2stage(rr, old_new_id):
reply_oid = rr.get("replyTo")
if not reply_oid:
return
new_id = old_new_id.get(rr.get("oid"))
if not new_id:
return
with local_session() as session:
comment = session.query(Reaction).filter(Reaction.id == new_id).first()
comment.replyTo = old_new_id.get(reply_oid)
session.add(comment)
srf = session.query(ShoutReactionsFollower).where(
ShoutReactionsFollower.shout == comment.shout
ShoutReactionsFollower.shout_id == comment.shout_id
).filter(
ShoutReactionsFollower.follower == comment.createdBy
ShoutReactionsFollower.follower_id == comment.createdBy
).first()
if not srf:
srf = ShoutReactionsFollower.create(shout=comment.shout, follower=comment.createdBy, auto=True)
srf = ShoutReactionsFollower.create(shout_id=comment.shout_id, follower_id=comment.createdBy, auto=True)
session.add(srf)
session.commit()
if not rr["body"]:
raise Exception(rr)

84
migration/tables/content_items.py
View File

@@ -8,7 +8,7 @@ from migration.extract import extract_html, extract_media
from orm.reaction import Reaction, ReactionKind
from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower
from orm.user import User
from orm.topic import TopicFollower
from orm.topic import TopicFollower, Topic
from services.stat.viewed import ViewedStorage
OLD_DATE = "2016-03-05 22:22:00.350000"
@@ -41,10 +41,10 @@ def create_author_from_app(app):
name = app.get('name')
slug = (
translit(name, "ru", reversed=True)
.replace(" ", "-")
.replace("'", "")
.replace(".", "-")
.lower()
.replace(" ", "-")
.replace("'", "")
.replace(".", "-")
.lower()
)
# check if nameslug is used
user = session.query(User).where(User.slug == slug).first()
@@ -84,13 +84,19 @@ def create_author_from_app(app):
async def create_shout(shout_dict, userslug):
s = Shout.create(**shout_dict)
with local_session() as session:
srf = session.query(ShoutReactionsFollower).where(
ShoutReactionsFollower.shout == s.slug
follower = session.query(User).where(User.slug == userslug).one()
srf = session.query(
ShoutReactionsFollower
).join(
User
).where(
ShoutReactionsFollower.shout_id == s.id
).filter(
ShoutReactionsFollower.follower == userslug
User.slug == userslug
).first()
if not srf:
srf = ShoutReactionsFollower.create(shout=s.slug, follower=userslug, auto=True)
srf = ShoutReactionsFollower.create(shout_id=s.id, follower_id=follower.id, auto=True)
session.add(srf)
session.commit()
@@ -214,17 +220,21 @@ async def add_topics_follower(entry, storage, userslug):
with local_session() as session:
for tpc in topics:
try:
topic = session.query(Topic).where(Topic.slug == tpc).one()
follower = session.query(User).where(User.slug == userslug).one()
tf = session.query(
TopicFollower
).where(
TopicFollower.follower == userslug
TopicFollower.follower_id == follower.id
).filter(
TopicFollower.topic == tpc
TopicFollower.topic_id == topic.id
).first()
if not tf:
tf = TopicFollower.create(
topic=tpc,
follower=userslug,
topic_id=topic.id,
follower_id=follower.id,
auto=True
)
session.add(tf)
@@ -300,27 +310,35 @@ async def topics_aftermath(entry, storage):
for tpc in filter(lambda x: bool(x), entry["topics"]):
oldslug = tpc
newslug = storage["replacements"].get(oldslug, oldslug)
if newslug:
with local_session() as session:
shout = session.query(Shout).where(Shout.slug == entry["slug"]).one()
new_topic = session.query(Topic).where(Topic.slug == newslug).one()
shout_topic_old = (
session.query(ShoutTopic)
.filter(ShoutTopic.shout == entry["slug"])
.filter(ShoutTopic.topic == oldslug)
.first()
.join(Shout)
.join(Topic)
.filter(Shout.slug == entry["slug"])
.filter(Topic.slug == oldslug)
.first()
)
if shout_topic_old:
shout_topic_old.update({"slug": newslug})
shout_topic_old.update({"topic_id": new_topic.id})
else:
shout_topic_new = (
session.query(ShoutTopic)
.filter(ShoutTopic.shout == entry["slug"])
.filter(ShoutTopic.topic == newslug)
.first()
.join(Shout)
.join(Topic)
.filter(Shout.slug == entry["slug"])
.filter(Topic.slug == newslug)
.first()
)
if not shout_topic_new:
try:
ShoutTopic.create(
**{"shout": entry["slug"], "topic": newslug}
**{"shout_id": shout.id, "topic_id": new_topic.id}
)
except Exception:
print("[migration] shout topic error: " + newslug)
@@ -339,31 +357,35 @@ async def content_ratings_to_reactions(entry, slug):
for content_rating in entry.get("ratings", []):
rater = (
session.query(User)
.filter(User.oid == content_rating["createdBy"])
.first()
.filter(User.oid == content_rating["createdBy"])
.first()
)
reactedBy = (
rater
if rater
else session.query(User).filter(User.slug == "noname").first()
else session.query(User).filter(User.slug == "anonymous").first()
)
if rater:
shout = session.query(Shout).where(Shout.slug == slug).one()
reaction_dict = {
"kind": ReactionKind.LIKE
if content_rating["value"] > 0
else ReactionKind.DISLIKE,
"createdBy": reactedBy.slug,
"shout": slug,
"createdBy": reactedBy.id,
"shout_id": shout.id,
}
cts = content_rating.get("createdAt")
if cts:
reaction_dict["createdAt"] = date_parse(cts)
reaction = (
session.query(Reaction)
.filter(Reaction.shout == reaction_dict["shout"])
.filter(Reaction.createdBy == reaction_dict["createdBy"])
.filter(Reaction.kind == reaction_dict["kind"])
.first()
session.query(Reaction).filter(
Reaction.shout_id == reaction_dict["shout_id"]
).filter(
Reaction.createdBy == reaction_dict["createdBy"]
).filter(
Reaction.kind == reaction_dict["kind"]
).first()
)
if reaction:
k = ReactionKind.AGREE if content_rating["value"] > 0 else ReactionKind.DISAGREE

19
migration/tables/users.py
View File

@@ -115,18 +115,23 @@ def migrate_2stage(entry, id_map):
continue
oid = entry["_id"]
author_slug = id_map.get(oid)
user_rating_dict = {
"value": rating_entry["value"],
"rater": rater_slug,
"user": author_slug,
}
with local_session() as session:
try:
rater = session.query(User).where(User.slug == rater_slug).one()
user = session.query(User).where(User.slug == author_slug).one()
user_rating_dict = {
"value": rating_entry["value"],
"rater_id": rater.id,
"user_id": user.id,
}
user_rating = UserRating.create(**user_rating_dict)
if user_rating_dict['value'] > 0:
af = AuthorFollower.create(
author=user_rating_dict['user'],
follower=user_rating_dict['rater'],
author_id=user.id,
follower_id=rater.id,
auto=True
)
session.add(af)