format and lint orm
This commit is contained in:
@@ -1 +1 @@
|
||||
__all__ = ["users", "tags", "content_items", "comments"],
|
||||
__all__ = (["users", "tags", "content_items", "comments"],)
|
||||
|
||||
@@ -8,104 +8,128 @@ from services.stat.reacted import ReactedStorage
|
||||
|
||||
ts = datetime.now()
|
||||
|
||||
|
||||
async def migrate(entry, storage):
|
||||
'''
|
||||
{
|
||||
"_id": "hdtwS8fSyFLxXCgSC",
|
||||
"body": "<p>",
|
||||
"contentItem": "mnK8KsJHPRi8DrybQ",
|
||||
"createdBy": "bMFPuyNg6qAD2mhXe",
|
||||
"thread": "01/",
|
||||
"createdAt": "2016-04-19 04:33:53+00:00",
|
||||
"ratings": [
|
||||
{ "createdBy": "AqmRukvRiExNpAe8C", "value": 1 },
|
||||
{ "createdBy": "YdE76Wth3yqymKEu5", "value": 1 }
|
||||
],
|
||||
"rating": 2,
|
||||
"updatedAt": "2020-05-27 19:22:57.091000+00:00",
|
||||
"updatedBy": "0"
|
||||
}
|
||||
"""
|
||||
{
|
||||
"_id": "hdtwS8fSyFLxXCgSC",
|
||||
"body": "<p>",
|
||||
"contentItem": "mnK8KsJHPRi8DrybQ",
|
||||
"createdBy": "bMFPuyNg6qAD2mhXe",
|
||||
"thread": "01/",
|
||||
"createdAt": "2016-04-19 04:33:53+00:00",
|
||||
"ratings": [
|
||||
{ "createdBy": "AqmRukvRiExNpAe8C", "value": 1 },
|
||||
{ "createdBy": "YdE76Wth3yqymKEu5", "value": 1 }
|
||||
],
|
||||
"rating": 2,
|
||||
"updatedAt": "2020-05-27 19:22:57.091000+00:00",
|
||||
"updatedBy": "0"
|
||||
}
|
||||
|
||||
->
|
||||
->
|
||||
|
||||
type Reaction {
|
||||
id: Int!
|
||||
shout: Shout!
|
||||
createdAt: DateTime!
|
||||
createdBy: User!
|
||||
updatedAt: DateTime
|
||||
deletedAt: DateTime
|
||||
deletedBy: User
|
||||
range: String # full / 0:2340
|
||||
kind: ReactionKind!
|
||||
body: String
|
||||
replyTo: Reaction
|
||||
stat: Stat
|
||||
old_id: String
|
||||
old_thread: String
|
||||
}
|
||||
'''
|
||||
reaction_dict = {}
|
||||
reaction_dict['createdAt'] = ts if not entry.get('createdAt') else date_parse(entry.get('createdAt'))
|
||||
print('[migration] reaction original date %r' % entry.get('createdAt'))
|
||||
# print('[migration] comment date %r ' % comment_dict['createdAt'])
|
||||
reaction_dict['body'] = html2text(entry.get('body', ''))
|
||||
reaction_dict['oid'] = entry['_id']
|
||||
if entry.get('createdAt'): reaction_dict['createdAt'] = date_parse(entry.get('createdAt'))
|
||||
shout_oid = entry.get('contentItem')
|
||||
if not shout_oid in storage['shouts']['by_oid']:
|
||||
if len(storage['shouts']['by_oid']) > 0:
|
||||
return shout_oid
|
||||
else:
|
||||
print('[migration] no shouts migrated yet')
|
||||
raise Exception
|
||||
return
|
||||
else:
|
||||
with local_session() as session:
|
||||
author = session.query(User).filter(User.oid == entry['createdBy']).first()
|
||||
shout_dict = storage['shouts']['by_oid'][shout_oid]
|
||||
if shout_dict:
|
||||
reaction_dict['shout'] = shout_dict['slug']
|
||||
reaction_dict['createdBy'] = author.slug if author else 'discours'
|
||||
reaction_dict['kind'] = ReactionKind.COMMENT
|
||||
type Reaction {
|
||||
id: Int!
|
||||
shout: Shout!
|
||||
createdAt: DateTime!
|
||||
createdBy: User!
|
||||
updatedAt: DateTime
|
||||
deletedAt: DateTime
|
||||
deletedBy: User
|
||||
range: String # full / 0:2340
|
||||
kind: ReactionKind!
|
||||
body: String
|
||||
replyTo: Reaction
|
||||
stat: Stat
|
||||
old_id: String
|
||||
old_thread: String
|
||||
}
|
||||
"""
|
||||
reaction_dict = {}
|
||||
reaction_dict["createdAt"] = (
|
||||
ts if not entry.get("createdAt") else date_parse(entry.get("createdAt"))
|
||||
)
|
||||
print("[migration] reaction original date %r" % entry.get("createdAt"))
|
||||
# print('[migration] comment date %r ' % comment_dict['createdAt'])
|
||||
reaction_dict["body"] = html2text(entry.get("body", ""))
|
||||
reaction_dict["oid"] = entry["_id"]
|
||||
if entry.get("createdAt"):
|
||||
reaction_dict["createdAt"] = date_parse(entry.get("createdAt"))
|
||||
shout_oid = entry.get("contentItem")
|
||||
if not shout_oid in storage["shouts"]["by_oid"]:
|
||||
if len(storage["shouts"]["by_oid"]) > 0:
|
||||
return shout_oid
|
||||
else:
|
||||
print("[migration] no shouts migrated yet")
|
||||
raise Exception
|
||||
return
|
||||
else:
|
||||
with local_session() as session:
|
||||
author = session.query(User).filter(User.oid == entry["createdBy"]).first()
|
||||
shout_dict = storage["shouts"]["by_oid"][shout_oid]
|
||||
if shout_dict:
|
||||
reaction_dict["shout"] = shout_dict["slug"]
|
||||
reaction_dict["createdBy"] = author.slug if author else "discours"
|
||||
reaction_dict["kind"] = ReactionKind.COMMENT
|
||||
|
||||
# creating reaction from old comment
|
||||
day = (reaction_dict.get("createdAt") or ts).replace(
|
||||
hour=0, minute=0, second=0, microsecond=0
|
||||
)
|
||||
reaction = Reaction.create(**reaction_dict)
|
||||
await ReactedStorage.increment(reaction)
|
||||
|
||||
reaction_dict["id"] = reaction.id
|
||||
for comment_rating_old in entry.get("ratings", []):
|
||||
rater = (
|
||||
session.query(User)
|
||||
.filter(User.oid == comment_rating_old["createdBy"])
|
||||
.first()
|
||||
)
|
||||
reactedBy = (
|
||||
rater
|
||||
if rater
|
||||
else session.query(User).filter(User.slug == "noname").first()
|
||||
)
|
||||
re_reaction_dict = {
|
||||
"shout": reaction_dict["shout"],
|
||||
"replyTo": reaction.id,
|
||||
"kind": ReactionKind.LIKE
|
||||
if comment_rating_old["value"] > 0
|
||||
else ReactionKind.DISLIKE,
|
||||
"createdBy": reactedBy.slug if reactedBy else "discours",
|
||||
}
|
||||
cts = comment_rating_old.get("createdAt")
|
||||
if cts:
|
||||
re_reaction_dict["createdAt"] = date_parse(cts)
|
||||
try:
|
||||
# creating reaction from old rating
|
||||
rr = Reaction.create(**re_reaction_dict)
|
||||
await ReactedStorage.increment(rr)
|
||||
|
||||
except Exception as e:
|
||||
print("[migration] comment rating error: %r" % re_reaction_dict)
|
||||
raise e
|
||||
else:
|
||||
print(
|
||||
"[migration] error: cannot find shout for comment %r"
|
||||
% reaction_dict
|
||||
)
|
||||
return reaction
|
||||
|
||||
# creating reaction from old comment
|
||||
day = (reaction_dict.get('createdAt') or ts).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
reaction = Reaction.create(**reaction_dict)
|
||||
await ReactedStorage.increment(reaction)
|
||||
|
||||
reaction_dict['id'] = reaction.id
|
||||
for comment_rating_old in entry.get('ratings',[]):
|
||||
rater = session.query(User).filter(User.oid == comment_rating_old['createdBy']).first()
|
||||
reactedBy = rater if rater else session.query(User).filter(User.slug == 'noname').first()
|
||||
re_reaction_dict = {
|
||||
'shout': reaction_dict['shout'],
|
||||
'replyTo': reaction.id,
|
||||
'kind': ReactionKind.LIKE if comment_rating_old['value'] > 0 else ReactionKind.DISLIKE,
|
||||
'createdBy': reactedBy.slug if reactedBy else 'discours'
|
||||
}
|
||||
cts = comment_rating_old.get('createdAt')
|
||||
if cts: re_reaction_dict['createdAt'] = date_parse(cts)
|
||||
try:
|
||||
# creating reaction from old rating
|
||||
rr = Reaction.create(**re_reaction_dict)
|
||||
await ReactedStorage.increment(rr)
|
||||
|
||||
except Exception as e:
|
||||
print('[migration] comment rating error: %r' % re_reaction_dict)
|
||||
raise e
|
||||
else:
|
||||
print('[migration] error: cannot find shout for comment %r' % reaction_dict)
|
||||
return reaction
|
||||
|
||||
def migrate_2stage(rr, old_new_id):
|
||||
reply_oid = rr.get('replyTo')
|
||||
if not reply_oid: return
|
||||
new_id = old_new_id.get(rr.get('oid'))
|
||||
if not new_id: return
|
||||
with local_session() as session:
|
||||
comment = session.query(Reaction).filter(Reaction.id == new_id).first()
|
||||
comment.replyTo = old_new_id.get(reply_oid)
|
||||
comment.save()
|
||||
session.commit()
|
||||
if not rr['body']: raise Exception(rr)
|
||||
reply_oid = rr.get("replyTo")
|
||||
if not reply_oid:
|
||||
return
|
||||
new_id = old_new_id.get(rr.get("oid"))
|
||||
if not new_id:
|
||||
return
|
||||
with local_session() as session:
|
||||
comment = session.query(Reaction).filter(Reaction.id == new_id).first()
|
||||
comment.replyTo = old_new_id.get(reply_oid)
|
||||
comment.save()
|
||||
session.commit()
|
||||
if not rr["body"]:
|
||||
raise Exception(rr)
|
||||
|
||||
@@ -10,224 +10,279 @@ from migration.extract import prepare_html_body
|
||||
from orm.community import Community
|
||||
from orm.reaction import Reaction, ReactionKind
|
||||
|
||||
OLD_DATE = '2016-03-05 22:22:00.350000'
|
||||
OLD_DATE = "2016-03-05 22:22:00.350000"
|
||||
ts = datetime.now()
|
||||
type2layout = {
|
||||
'Article': 'article',
|
||||
'Literature': 'prose',
|
||||
'Music': 'music',
|
||||
'Video': 'video',
|
||||
'Image': 'image'
|
||||
"Article": "article",
|
||||
"Literature": "prose",
|
||||
"Music": "music",
|
||||
"Video": "video",
|
||||
"Image": "image",
|
||||
}
|
||||
|
||||
|
||||
def get_shout_slug(entry):
|
||||
slug = entry.get('slug', '')
|
||||
if not slug:
|
||||
for friend in entry.get('friendlySlugs', []):
|
||||
slug = friend.get('slug', '')
|
||||
if slug: break
|
||||
return slug
|
||||
slug = entry.get("slug", "")
|
||||
if not slug:
|
||||
for friend in entry.get("friendlySlugs", []):
|
||||
slug = friend.get("slug", "")
|
||||
if slug:
|
||||
break
|
||||
return slug
|
||||
|
||||
|
||||
async def migrate(entry, storage):
|
||||
# init, set title and layout
|
||||
r = {
|
||||
'layout': type2layout[entry['type']],
|
||||
'title': entry['title'],
|
||||
'community': Community.default_community.id,
|
||||
'authors': [],
|
||||
'topics': set([]),
|
||||
# 'rating': 0,
|
||||
# 'ratings': [],
|
||||
'createdAt': []
|
||||
}
|
||||
topics_by_oid = storage['topics']['by_oid']
|
||||
users_by_oid = storage['users']['by_oid']
|
||||
# init, set title and layout
|
||||
r = {
|
||||
"layout": type2layout[entry["type"]],
|
||||
"title": entry["title"],
|
||||
"community": Community.default_community.id,
|
||||
"authors": [],
|
||||
"topics": set([]),
|
||||
# 'rating': 0,
|
||||
# 'ratings': [],
|
||||
"createdAt": [],
|
||||
}
|
||||
topics_by_oid = storage["topics"]["by_oid"]
|
||||
users_by_oid = storage["users"]["by_oid"]
|
||||
|
||||
# author
|
||||
# author
|
||||
|
||||
oid = entry.get('createdBy', entry.get('_id', entry.get('oid')))
|
||||
userdata = users_by_oid.get(oid)
|
||||
if not userdata:
|
||||
app = entry.get('application')
|
||||
if app:
|
||||
userslug = translit(app['name'], 'ru', reversed=True)\
|
||||
.replace(' ', '-')\
|
||||
.replace('\'', '')\
|
||||
.replace('.', '-').lower()
|
||||
userdata = {
|
||||
'username': app['email'],
|
||||
'email': app['email'],
|
||||
'name': app['name'],
|
||||
'bio': app.get('bio', ''),
|
||||
'emailConfirmed': False,
|
||||
'slug': userslug,
|
||||
'createdAt': ts,
|
||||
'wasOnlineAt': ts
|
||||
}
|
||||
else:
|
||||
userdata = User.default_user.dict()
|
||||
assert userdata, 'no user found for %s from ' % [oid, len(users_by_oid.keys())]
|
||||
r['authors'] = [userdata, ]
|
||||
oid = entry.get("createdBy", entry.get("_id", entry.get("oid")))
|
||||
userdata = users_by_oid.get(oid)
|
||||
if not userdata:
|
||||
app = entry.get("application")
|
||||
if app:
|
||||
userslug = (
|
||||
translit(app["name"], "ru", reversed=True)
|
||||
.replace(" ", "-")
|
||||
.replace("'", "")
|
||||
.replace(".", "-")
|
||||
.lower()
|
||||
)
|
||||
userdata = {
|
||||
"username": app["email"],
|
||||
"email": app["email"],
|
||||
"name": app["name"],
|
||||
"bio": app.get("bio", ""),
|
||||
"emailConfirmed": False,
|
||||
"slug": userslug,
|
||||
"createdAt": ts,
|
||||
"wasOnlineAt": ts,
|
||||
}
|
||||
else:
|
||||
userdata = User.default_user.dict()
|
||||
assert userdata, "no user found for %s from " % [oid, len(users_by_oid.keys())]
|
||||
r["authors"] = [
|
||||
userdata,
|
||||
]
|
||||
|
||||
# slug
|
||||
# slug
|
||||
|
||||
slug = get_shout_slug(entry)
|
||||
if slug: r['slug'] = slug
|
||||
else: raise Exception
|
||||
|
||||
# cover
|
||||
c = ''
|
||||
if entry.get('thumborId'):
|
||||
c = 'https://assets.discours.io/unsafe/1600x/' + entry['thumborId']
|
||||
else:
|
||||
c = entry.get('image', {}).get('url')
|
||||
if not c or 'cloudinary' in c: c = ''
|
||||
r['cover'] = c
|
||||
slug = get_shout_slug(entry)
|
||||
if slug:
|
||||
r["slug"] = slug
|
||||
else:
|
||||
raise Exception
|
||||
|
||||
# timestamps
|
||||
# cover
|
||||
c = ""
|
||||
if entry.get("thumborId"):
|
||||
c = "https://assets.discours.io/unsafe/1600x/" + entry["thumborId"]
|
||||
else:
|
||||
c = entry.get("image", {}).get("url")
|
||||
if not c or "cloudinary" in c:
|
||||
c = ""
|
||||
r["cover"] = c
|
||||
|
||||
r['createdAt'] = date_parse(entry.get('createdAt', OLD_DATE))
|
||||
r['updatedAt'] = date_parse(entry['updatedAt']) if 'updatedAt' in entry else ts
|
||||
if entry.get('published'):
|
||||
r['publishedAt'] = date_parse(entry.get('publishedAt', OLD_DATE))
|
||||
if 'deletedAt' in entry: r['deletedAt'] = date_parse(entry['deletedAt'])
|
||||
# timestamps
|
||||
|
||||
# topics
|
||||
category = entry['category']
|
||||
mainTopic = topics_by_oid.get(category)
|
||||
if mainTopic:
|
||||
r['mainTopic'] = storage['replacements'].get(mainTopic["slug"], mainTopic["slug"])
|
||||
topic_oids = [category, ]
|
||||
topic_oids.extend(entry.get('tags', []))
|
||||
for oid in topic_oids:
|
||||
if oid in storage['topics']['by_oid']:
|
||||
r['topics'].add(storage['topics']['by_oid'][oid]['slug'])
|
||||
else:
|
||||
print('[migration] unknown old topic id: ' + oid)
|
||||
r['topics'] = list(r['topics'])
|
||||
|
||||
entry['topics'] = r['topics']
|
||||
entry['cover'] = r['cover']
|
||||
entry['authors'] = r['authors']
|
||||
r["createdAt"] = date_parse(entry.get("createdAt", OLD_DATE))
|
||||
r["updatedAt"] = date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts
|
||||
if entry.get("published"):
|
||||
r["publishedAt"] = date_parse(entry.get("publishedAt", OLD_DATE))
|
||||
if "deletedAt" in entry:
|
||||
r["deletedAt"] = date_parse(entry["deletedAt"])
|
||||
|
||||
# body
|
||||
r['body'] = prepare_html_body(entry)
|
||||
# topics
|
||||
category = entry["category"]
|
||||
mainTopic = topics_by_oid.get(category)
|
||||
if mainTopic:
|
||||
r["mainTopic"] = storage["replacements"].get(
|
||||
mainTopic["slug"], mainTopic["slug"]
|
||||
)
|
||||
topic_oids = [
|
||||
category,
|
||||
]
|
||||
topic_oids.extend(entry.get("tags", []))
|
||||
for oid in topic_oids:
|
||||
if oid in storage["topics"]["by_oid"]:
|
||||
r["topics"].add(storage["topics"]["by_oid"][oid]["slug"])
|
||||
else:
|
||||
print("[migration] unknown old topic id: " + oid)
|
||||
r["topics"] = list(r["topics"])
|
||||
|
||||
# save shout to db
|
||||
entry["topics"] = r["topics"]
|
||||
entry["cover"] = r["cover"]
|
||||
entry["authors"] = r["authors"]
|
||||
|
||||
s = object()
|
||||
shout_dict = r.copy()
|
||||
user = None
|
||||
del shout_dict['topics'] # NOTE: AttributeError: 'str' object has no attribute '_sa_instance_state'
|
||||
#del shout_dict['rating'] # NOTE: TypeError: 'rating' is an invalid keyword argument for Shout
|
||||
#del shout_dict['ratings']
|
||||
email = userdata.get('email')
|
||||
slug = userdata.get('slug')
|
||||
if not slug: raise Exception
|
||||
with local_session() as session:
|
||||
# c = session.query(Community).all().pop()
|
||||
if email: user = session.query(User).filter(User.email == email).first()
|
||||
if not user and slug: user = session.query(User).filter(User.slug == slug).first()
|
||||
if not user and userdata:
|
||||
try:
|
||||
userdata['slug'] = userdata['slug'].lower().strip().replace(' ', '-')
|
||||
user = User.create(**userdata)
|
||||
except sqlalchemy.exc.IntegrityError:
|
||||
print('[migration] user error: ' + userdata)
|
||||
userdata['id'] = user.id
|
||||
userdata['createdAt'] = user.createdAt
|
||||
storage['users']['by_slug'][userdata['slug']] = userdata
|
||||
storage['users']['by_oid'][entry['_id']] = userdata
|
||||
assert user, 'could not get a user'
|
||||
shout_dict['authors'] = [ user, ]
|
||||
# body
|
||||
r["body"] = prepare_html_body(entry)
|
||||
|
||||
try:
|
||||
s = Shout.create(**shout_dict)
|
||||
except sqlalchemy.exc.IntegrityError as e:
|
||||
with local_session() as session:
|
||||
s = session.query(Shout).filter(Shout.slug == shout_dict['slug']).first()
|
||||
bump = False
|
||||
if s:
|
||||
for key in shout_dict:
|
||||
if key in s.__dict__:
|
||||
if s.__dict__[key] != shout_dict[key]:
|
||||
print('[migration] shout already exists, but differs in %s' % key)
|
||||
bump = True
|
||||
else:
|
||||
print('[migration] shout already exists, but lacks %s' % key)
|
||||
bump = True
|
||||
if bump:
|
||||
s.update(shout_dict)
|
||||
else:
|
||||
print('[migration] something went wrong with shout: \n%r' % shout_dict)
|
||||
raise e
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(s)
|
||||
raise Exception
|
||||
|
||||
# save shout to db
|
||||
|
||||
# shout topics aftermath
|
||||
shout_dict['topics'] = []
|
||||
for tpc in r['topics']:
|
||||
oldslug = tpc
|
||||
newslug = storage['replacements'].get(oldslug, oldslug)
|
||||
if newslug:
|
||||
with local_session() as session:
|
||||
shout_topic_old = session.query(ShoutTopic)\
|
||||
.filter(ShoutTopic.shout == shout_dict['slug'])\
|
||||
.filter(ShoutTopic.topic == oldslug).first()
|
||||
if shout_topic_old:
|
||||
shout_topic_old.update({ 'slug': newslug })
|
||||
else:
|
||||
shout_topic_new = session.query(ShoutTopic)\
|
||||
.filter(ShoutTopic.shout == shout_dict['slug'])\
|
||||
.filter(ShoutTopic.topic == newslug).first()
|
||||
if not shout_topic_new:
|
||||
try: ShoutTopic.create(**{ 'shout': shout_dict['slug'], 'topic': newslug })
|
||||
except: print('[migration] shout topic error: ' + newslug)
|
||||
session.commit()
|
||||
if newslug not in shout_dict['topics']:
|
||||
shout_dict['topics'].append(newslug)
|
||||
else:
|
||||
print('[migration] ignored topic slug: \n%r' % tpc['slug'])
|
||||
# raise Exception
|
||||
s = object()
|
||||
shout_dict = r.copy()
|
||||
user = None
|
||||
del shout_dict[
|
||||
"topics"
|
||||
] # NOTE: AttributeError: 'str' object has no attribute '_sa_instance_state'
|
||||
# del shout_dict['rating'] # NOTE: TypeError: 'rating' is an invalid keyword argument for Shout
|
||||
# del shout_dict['ratings']
|
||||
email = userdata.get("email")
|
||||
slug = userdata.get("slug")
|
||||
if not slug:
|
||||
raise Exception
|
||||
with local_session() as session:
|
||||
# c = session.query(Community).all().pop()
|
||||
if email:
|
||||
user = session.query(User).filter(User.email == email).first()
|
||||
if not user and slug:
|
||||
user = session.query(User).filter(User.slug == slug).first()
|
||||
if not user and userdata:
|
||||
try:
|
||||
userdata["slug"] = userdata["slug"].lower().strip().replace(" ", "-")
|
||||
user = User.create(**userdata)
|
||||
except sqlalchemy.exc.IntegrityError:
|
||||
print("[migration] user error: " + userdata)
|
||||
userdata["id"] = user.id
|
||||
userdata["createdAt"] = user.createdAt
|
||||
storage["users"]["by_slug"][userdata["slug"]] = userdata
|
||||
storage["users"]["by_oid"][entry["_id"]] = userdata
|
||||
assert user, "could not get a user"
|
||||
shout_dict["authors"] = [
|
||||
user,
|
||||
]
|
||||
|
||||
# content_item ratings to reactions
|
||||
try:
|
||||
for content_rating in entry.get('ratings',[]):
|
||||
with local_session() as session:
|
||||
rater = session.query(User).filter(User.oid == content_rating['createdBy']).first()
|
||||
reactedBy = rater if rater else session.query(User).filter(User.slug == 'noname').first()
|
||||
if rater:
|
||||
reaction_dict = {
|
||||
'kind': ReactionKind.LIKE if content_rating['value'] > 0 else ReactionKind.DISLIKE,
|
||||
'createdBy': reactedBy.slug,
|
||||
'shout': shout_dict['slug']
|
||||
}
|
||||
cts = content_rating.get('createdAt')
|
||||
if cts: reaction_dict['createdAt'] = date_parse(cts)
|
||||
reaction = session.query(Reaction).\
|
||||
filter(Reaction.shout == reaction_dict['shout']).\
|
||||
filter(Reaction.createdBy == reaction_dict['createdBy']).\
|
||||
filter(Reaction.kind == reaction_dict['kind']).first()
|
||||
if reaction:
|
||||
reaction_dict['kind'] = ReactionKind.AGREE if content_rating['value'] > 0 else ReactionKind.DISAGREE,
|
||||
reaction.update(reaction_dict)
|
||||
else:
|
||||
day = (reaction_dict.get('createdAt') or ts).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
rea = Reaction.create(**reaction_dict)
|
||||
await ReactedStorage.increment(rea)
|
||||
# shout_dict['ratings'].append(reaction_dict)
|
||||
except:
|
||||
print('[migration] content_item.ratings error: \n%r' % content_rating)
|
||||
raise Exception
|
||||
try:
|
||||
s = Shout.create(**shout_dict)
|
||||
except sqlalchemy.exc.IntegrityError as e:
|
||||
with local_session() as session:
|
||||
s = session.query(Shout).filter(Shout.slug == shout_dict["slug"]).first()
|
||||
bump = False
|
||||
if s:
|
||||
for key in shout_dict:
|
||||
if key in s.__dict__:
|
||||
if s.__dict__[key] != shout_dict[key]:
|
||||
print(
|
||||
"[migration] shout already exists, but differs in %s"
|
||||
% key
|
||||
)
|
||||
bump = True
|
||||
else:
|
||||
print("[migration] shout already exists, but lacks %s" % key)
|
||||
bump = True
|
||||
if bump:
|
||||
s.update(shout_dict)
|
||||
else:
|
||||
print("[migration] something went wrong with shout: \n%r" % shout_dict)
|
||||
raise e
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(s)
|
||||
raise Exception
|
||||
|
||||
# shout views
|
||||
ViewedByDay.create( shout = shout_dict['slug'], value = entry.get('views', 1) )
|
||||
# del shout_dict['ratings']
|
||||
shout_dict['oid'] = entry.get('_id')
|
||||
storage['shouts']['by_oid'][entry['_id']] = shout_dict
|
||||
storage['shouts']['by_slug'][slug] = shout_dict
|
||||
return shout_dict
|
||||
# shout topics aftermath
|
||||
shout_dict["topics"] = []
|
||||
for tpc in r["topics"]:
|
||||
oldslug = tpc
|
||||
newslug = storage["replacements"].get(oldslug, oldslug)
|
||||
if newslug:
|
||||
with local_session() as session:
|
||||
shout_topic_old = (
|
||||
session.query(ShoutTopic)
|
||||
.filter(ShoutTopic.shout == shout_dict["slug"])
|
||||
.filter(ShoutTopic.topic == oldslug)
|
||||
.first()
|
||||
)
|
||||
if shout_topic_old:
|
||||
shout_topic_old.update({"slug": newslug})
|
||||
else:
|
||||
shout_topic_new = (
|
||||
session.query(ShoutTopic)
|
||||
.filter(ShoutTopic.shout == shout_dict["slug"])
|
||||
.filter(ShoutTopic.topic == newslug)
|
||||
.first()
|
||||
)
|
||||
if not shout_topic_new:
|
||||
try:
|
||||
ShoutTopic.create(
|
||||
**{"shout": shout_dict["slug"], "topic": newslug}
|
||||
)
|
||||
except:
|
||||
print("[migration] shout topic error: " + newslug)
|
||||
session.commit()
|
||||
if newslug not in shout_dict["topics"]:
|
||||
shout_dict["topics"].append(newslug)
|
||||
else:
|
||||
print("[migration] ignored topic slug: \n%r" % tpc["slug"])
|
||||
# raise Exception
|
||||
|
||||
# content_item ratings to reactions
|
||||
try:
|
||||
for content_rating in entry.get("ratings", []):
|
||||
with local_session() as session:
|
||||
rater = (
|
||||
session.query(User)
|
||||
.filter(User.oid == content_rating["createdBy"])
|
||||
.first()
|
||||
)
|
||||
reactedBy = (
|
||||
rater
|
||||
if rater
|
||||
else session.query(User).filter(User.slug == "noname").first()
|
||||
)
|
||||
if rater:
|
||||
reaction_dict = {
|
||||
"kind": ReactionKind.LIKE
|
||||
if content_rating["value"] > 0
|
||||
else ReactionKind.DISLIKE,
|
||||
"createdBy": reactedBy.slug,
|
||||
"shout": shout_dict["slug"],
|
||||
}
|
||||
cts = content_rating.get("createdAt")
|
||||
if cts:
|
||||
reaction_dict["createdAt"] = date_parse(cts)
|
||||
reaction = (
|
||||
session.query(Reaction)
|
||||
.filter(Reaction.shout == reaction_dict["shout"])
|
||||
.filter(Reaction.createdBy == reaction_dict["createdBy"])
|
||||
.filter(Reaction.kind == reaction_dict["kind"])
|
||||
.first()
|
||||
)
|
||||
if reaction:
|
||||
reaction_dict["kind"] = (
|
||||
ReactionKind.AGREE
|
||||
if content_rating["value"] > 0
|
||||
else ReactionKind.DISAGREE,
|
||||
)
|
||||
reaction.update(reaction_dict)
|
||||
else:
|
||||
day = (reaction_dict.get("createdAt") or ts).replace(
|
||||
hour=0, minute=0, second=0, microsecond=0
|
||||
)
|
||||
rea = Reaction.create(**reaction_dict)
|
||||
await ReactedStorage.increment(rea)
|
||||
# shout_dict['ratings'].append(reaction_dict)
|
||||
except:
|
||||
print("[migration] content_item.ratings error: \n%r" % content_rating)
|
||||
raise Exception
|
||||
|
||||
# shout views
|
||||
ViewedByDay.create(shout=shout_dict["slug"], value=entry.get("views", 1))
|
||||
# del shout_dict['ratings']
|
||||
shout_dict["oid"] = entry.get("_id")
|
||||
storage["shouts"]["by_oid"][entry["_id"]] = shout_dict
|
||||
storage["shouts"]["by_slug"][slug] = shout_dict
|
||||
return shout_dict
|
||||
|
||||
@@ -4,104 +4,144 @@ from orm import User, UserRating
|
||||
from dateutil.parser import parse
|
||||
from base.orm import local_session
|
||||
|
||||
|
||||
def migrate(entry):
|
||||
if 'subscribedTo' in entry: del entry['subscribedTo']
|
||||
email = entry['emails'][0]['address']
|
||||
user_dict = {
|
||||
'oid': entry['_id'],
|
||||
'roles': [],
|
||||
'ratings': [],
|
||||
'username': email,
|
||||
'email': email,
|
||||
'password': entry['services']['password'].get('bcrypt', ''),
|
||||
'createdAt': parse(entry['createdAt']),
|
||||
'emailConfirmed': bool(entry['emails'][0]['verified']),
|
||||
'muted': False, # amnesty
|
||||
'bio': entry['profile'].get('bio', ''),
|
||||
'notifications': [],
|
||||
'createdAt': parse(entry['createdAt']),
|
||||
'roles': [], # entry['roles'] # roles by community
|
||||
'ratings': [], # entry['ratings']
|
||||
'links': [],
|
||||
'name': 'anonymous'
|
||||
}
|
||||
if 'updatedAt' in entry: user_dict['updatedAt'] = parse(entry['updatedAt'])
|
||||
if 'wasOnineAt' in entry: user_dict['wasOnlineAt'] = parse(entry['wasOnlineAt'])
|
||||
if entry.get('profile'):
|
||||
# slug
|
||||
user_dict['slug'] = entry['profile'].get('path').lower().replace(' ', '-').strip()
|
||||
user_dict['bio'] = html2text(entry.get('profile').get('bio') or '')
|
||||
if "subscribedTo" in entry:
|
||||
del entry["subscribedTo"]
|
||||
email = entry["emails"][0]["address"]
|
||||
user_dict = {
|
||||
"oid": entry["_id"],
|
||||
"roles": [],
|
||||
"ratings": [],
|
||||
"username": email,
|
||||
"email": email,
|
||||
"password": entry["services"]["password"].get("bcrypt", ""),
|
||||
"createdAt": parse(entry["createdAt"]),
|
||||
"emailConfirmed": bool(entry["emails"][0]["verified"]),
|
||||
"muted": False, # amnesty
|
||||
"bio": entry["profile"].get("bio", ""),
|
||||
"notifications": [],
|
||||
"createdAt": parse(entry["createdAt"]),
|
||||
"roles": [], # entry['roles'] # roles by community
|
||||
"ratings": [], # entry['ratings']
|
||||
"links": [],
|
||||
"name": "anonymous",
|
||||
}
|
||||
if "updatedAt" in entry:
|
||||
user_dict["updatedAt"] = parse(entry["updatedAt"])
|
||||
if "wasOnineAt" in entry:
|
||||
user_dict["wasOnlineAt"] = parse(entry["wasOnlineAt"])
|
||||
if entry.get("profile"):
|
||||
# slug
|
||||
user_dict["slug"] = (
|
||||
entry["profile"].get("path").lower().replace(" ", "-").strip()
|
||||
)
|
||||
user_dict["bio"] = html2text(entry.get("profile").get("bio") or "")
|
||||
|
||||
# userpic
|
||||
try: user_dict['userpic'] = 'https://assets.discours.io/unsafe/100x/' + entry['profile']['thumborId']
|
||||
except KeyError:
|
||||
try: user_dict['userpic'] = entry['profile']['image']['url']
|
||||
except KeyError: user_dict['userpic'] = ''
|
||||
# userpic
|
||||
try:
|
||||
user_dict["userpic"] = (
|
||||
"https://assets.discours.io/unsafe/100x/"
|
||||
+ entry["profile"]["thumborId"]
|
||||
)
|
||||
except KeyError:
|
||||
try:
|
||||
user_dict["userpic"] = entry["profile"]["image"]["url"]
|
||||
except KeyError:
|
||||
user_dict["userpic"] = ""
|
||||
|
||||
# name
|
||||
fn = entry['profile'].get('firstName', '')
|
||||
ln = entry['profile'].get('lastName', '')
|
||||
name = user_dict['slug'] if user_dict['slug'] else 'anonymous'
|
||||
name = fn if fn else name
|
||||
name = (name + ' ' + ln) if ln else name
|
||||
name = entry['profile']['path'].lower().strip().replace(' ', '-') if len(name) < 2 else name
|
||||
user_dict['name'] = name
|
||||
# name
|
||||
fn = entry["profile"].get("firstName", "")
|
||||
ln = entry["profile"].get("lastName", "")
|
||||
name = user_dict["slug"] if user_dict["slug"] else "anonymous"
|
||||
name = fn if fn else name
|
||||
name = (name + " " + ln) if ln else name
|
||||
name = (
|
||||
entry["profile"]["path"].lower().strip().replace(" ", "-")
|
||||
if len(name) < 2
|
||||
else name
|
||||
)
|
||||
user_dict["name"] = name
|
||||
|
||||
# links
|
||||
fb = entry['profile'].get('facebook', False)
|
||||
if fb: user_dict['links'].append(fb)
|
||||
vk = entry['profile'].get('vkontakte', False)
|
||||
if vk: user_dict['links'].append(vk)
|
||||
tr = entry['profile'].get('twitter', False)
|
||||
if tr: user_dict['links'].append(tr)
|
||||
ws = entry['profile'].get('website', False)
|
||||
if ws: user_dict['links'].append(ws)
|
||||
# links
|
||||
fb = entry["profile"].get("facebook", False)
|
||||
if fb:
|
||||
user_dict["links"].append(fb)
|
||||
vk = entry["profile"].get("vkontakte", False)
|
||||
if vk:
|
||||
user_dict["links"].append(vk)
|
||||
tr = entry["profile"].get("twitter", False)
|
||||
if tr:
|
||||
user_dict["links"].append(tr)
|
||||
ws = entry["profile"].get("website", False)
|
||||
if ws:
|
||||
user_dict["links"].append(ws)
|
||||
|
||||
# some checks
|
||||
if not user_dict['slug'] and len(user_dict['links']) > 0:
|
||||
user_dict['slug'] = user_dict['links'][0].split('/')[-1]
|
||||
# some checks
|
||||
if not user_dict["slug"] and len(user_dict["links"]) > 0:
|
||||
user_dict["slug"] = user_dict["links"][0].split("/")[-1]
|
||||
|
||||
user_dict["slug"] = user_dict.get("slug", user_dict["email"].split("@")[0])
|
||||
oid = user_dict["oid"]
|
||||
user_dict["slug"] = user_dict["slug"].lower().strip().replace(" ", "-")
|
||||
try:
|
||||
user = User.create(**user_dict.copy())
|
||||
except sqlalchemy.exc.IntegrityError:
|
||||
print("[migration] cannot create user " + user_dict["slug"])
|
||||
with local_session() as session:
|
||||
old_user = (
|
||||
session.query(User).filter(User.slug == user_dict["slug"]).first()
|
||||
)
|
||||
old_user.oid = oid
|
||||
user = old_user
|
||||
if not user:
|
||||
print("[migration] ERROR: cannot find user " + user_dict["slug"])
|
||||
raise Exception
|
||||
user_dict["id"] = user.id
|
||||
return user_dict
|
||||
|
||||
user_dict['slug'] = user_dict.get('slug', user_dict['email'].split('@')[0])
|
||||
oid = user_dict['oid']
|
||||
user_dict['slug'] = user_dict['slug'].lower().strip().replace(' ', '-')
|
||||
try: user = User.create(**user_dict.copy())
|
||||
except sqlalchemy.exc.IntegrityError:
|
||||
print('[migration] cannot create user ' + user_dict['slug'])
|
||||
with local_session() as session:
|
||||
old_user = session.query(User).filter(User.slug == user_dict['slug']).first()
|
||||
old_user.oid = oid
|
||||
user = old_user
|
||||
if not user:
|
||||
print('[migration] ERROR: cannot find user ' + user_dict['slug'])
|
||||
raise Exception
|
||||
user_dict['id'] = user.id
|
||||
return user_dict
|
||||
|
||||
def migrate_2stage(entry, id_map):
|
||||
ce = 0
|
||||
for rating_entry in entry.get('ratings',[]):
|
||||
rater_oid = rating_entry['createdBy']
|
||||
rater_slug = id_map.get(rater_oid)
|
||||
if not rater_slug:
|
||||
ce +=1
|
||||
# print(rating_entry)
|
||||
continue
|
||||
oid = entry['_id']
|
||||
author_slug = id_map.get(oid)
|
||||
user_rating_dict = {
|
||||
'value': rating_entry['value'],
|
||||
'rater': rater_slug,
|
||||
'user': author_slug
|
||||
}
|
||||
with local_session() as session:
|
||||
try:
|
||||
user_rating = UserRating.create(**user_rating_dict)
|
||||
except sqlalchemy.exc.IntegrityError:
|
||||
old_rating = session.query(UserRating).filter(UserRating.rater == rater_slug).first()
|
||||
print('[migration] cannot create ' + author_slug + '`s rate from ' + rater_slug)
|
||||
print('[migration] concat rating value %d+%d=%d' % (old_rating.value, rating_entry['value'], old_rating.value + rating_entry['value']))
|
||||
old_rating.update({ 'value': old_rating.value + rating_entry['value'] })
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return ce
|
||||
ce = 0
|
||||
for rating_entry in entry.get("ratings", []):
|
||||
rater_oid = rating_entry["createdBy"]
|
||||
rater_slug = id_map.get(rater_oid)
|
||||
if not rater_slug:
|
||||
ce += 1
|
||||
# print(rating_entry)
|
||||
continue
|
||||
oid = entry["_id"]
|
||||
author_slug = id_map.get(oid)
|
||||
user_rating_dict = {
|
||||
"value": rating_entry["value"],
|
||||
"rater": rater_slug,
|
||||
"user": author_slug,
|
||||
}
|
||||
with local_session() as session:
|
||||
try:
|
||||
user_rating = UserRating.create(**user_rating_dict)
|
||||
except sqlalchemy.exc.IntegrityError:
|
||||
old_rating = (
|
||||
session.query(UserRating)
|
||||
.filter(UserRating.rater == rater_slug)
|
||||
.first()
|
||||
)
|
||||
print(
|
||||
"[migration] cannot create "
|
||||
+ author_slug
|
||||
+ "`s rate from "
|
||||
+ rater_slug
|
||||
)
|
||||
print(
|
||||
"[migration] concat rating value %d+%d=%d"
|
||||
% (
|
||||
old_rating.value,
|
||||
rating_entry["value"],
|
||||
old_rating.value + rating_entry["value"],
|
||||
)
|
||||
)
|
||||
old_rating.update({"value": old_rating.value + rating_entry["value"]})
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return ce
|
||||
|
||||
Reference in New Issue
Block a user