migration-orm-fixes
This commit is contained in:
@@ -1,12 +1,13 @@
|
||||
from datetime import datetime
|
||||
from dateutil.parser import parse as date_parse
|
||||
import json
|
||||
import datetime
|
||||
from os.path import abspath
|
||||
from orm import Shout, Comment, CommentRating, User
|
||||
from orm import Comment, CommentRating, User
|
||||
from orm.base import local_session
|
||||
from migration.html2text import html2text
|
||||
from orm.shout import Shout
|
||||
|
||||
def migrate(entry, shouts_by_oid):
|
||||
ts = datetime.now()
|
||||
|
||||
def migrate(entry, storage):
|
||||
'''
|
||||
{
|
||||
"_id": "hdtwS8fSyFLxXCgSC",
|
||||
@@ -28,60 +29,70 @@ def migrate(entry, shouts_by_oid):
|
||||
|
||||
type Comment {
|
||||
id: Int!
|
||||
author: Int!
|
||||
createdBy: User!
|
||||
body: String!
|
||||
replyTo: Int!
|
||||
replyTo: Comment!
|
||||
createdAt: DateTime!
|
||||
updatedAt: DateTime
|
||||
shout: Int!
|
||||
shout: Shout!
|
||||
deletedAt: DateTime
|
||||
deletedBy: Int
|
||||
deletedBy: User
|
||||
ratings: [CommentRating]
|
||||
views: Int
|
||||
}
|
||||
'''
|
||||
if entry.get('deleted'): return
|
||||
comment_dict = {}
|
||||
# FIXME: comment_dict['createdAt'] = ts if not entry.get('createdAt') else date_parse(entry.get('createdAt'))
|
||||
# print('[migration] comment original date %r' % entry.get('createdAt'))
|
||||
# print('[migration] comment date %r ' % comment_dict['createdAt'])
|
||||
comment_dict['body'] = html2text(entry.get('body', ''))
|
||||
comment_dict['oid'] = entry['_id']
|
||||
if entry.get('createdAt'): comment_dict['createdAt'] = date_parse(entry.get('createdAt'))
|
||||
shout_oid = entry.get('contentItem')
|
||||
if not shout_oid in storage['shouts']['by_oid']:
|
||||
print('[migration] no shout for comment', entry)
|
||||
else:
|
||||
with local_session() as session:
|
||||
author = session.query(User).filter(User.oid == entry['createdBy']).first()
|
||||
shout_dict = storage['shouts']['by_oid'][shout_oid]
|
||||
if shout_dict:
|
||||
comment_dict['shout'] = shout_dict['oid']
|
||||
comment_dict['createdBy'] = author.slug if author else 'discours'
|
||||
# FIXME if entry.get('deleted'): comment_dict['deletedAt'] = date_parse(entry['updatedAt']) or ts
|
||||
# comment_dict['deletedBy'] = session.query(User).filter(User.oid == (entry.get('updatedBy') or dd['oid'])).first()
|
||||
# FIXME if entry.get('updatedAt'): comment_dict['updatedAt'] = date_parse(entry['updatedAt']) or ts
|
||||
#for [k, v] in comment_dict.items():
|
||||
# if not v: del comment_dict[f]
|
||||
# if k.endswith('At'):
|
||||
# try: comment_dict[k] = datetime(comment_dict[k])
|
||||
# except: print(k)
|
||||
# # print('[migration] comment keys:', f)
|
||||
|
||||
shout_old_id = entry['contentItem']
|
||||
if not shout_old_id in shouts_by_oid:
|
||||
return
|
||||
shout = shouts_by_oid[shout_old_id]
|
||||
|
||||
with local_session() as session:
|
||||
author = session.query(User).filter(User.old_id == entry['createdBy']).first()
|
||||
comment_dict = {
|
||||
'author': author.id if author else 0,
|
||||
'createdAt': date_parse(entry['createdAt']),
|
||||
'body': html2text(entry['body']),
|
||||
'shout': shout["slug"]
|
||||
}
|
||||
if entry.get('deleted'):
|
||||
comment_dict['deletedAt'] = date_parse(entry['updatedAt'])
|
||||
comment_dict['deletedBy'] = str(entry['updatedBy'])
|
||||
if entry.get('updatedAt'):
|
||||
comment_dict['updatedAt'] = date_parse(entry['updatedAt'])
|
||||
# comment_dict['updatedBy'] = str(entry.get('updatedBy', 0)) invalid keyword for Comment
|
||||
# print(comment_dict)
|
||||
comment = Comment.create(**comment_dict)
|
||||
comment_dict['id'] = comment.id
|
||||
comment_dict['ratings'] = []
|
||||
comment_dict['old_id'] = entry['_id']
|
||||
# print(comment)
|
||||
for comment_rating_old in entry.get('ratings',[]):
|
||||
rater = session.query(User).filter(User.old_id == comment_rating_old['createdBy']).first()
|
||||
if rater and comment:
|
||||
comment_rating_dict = {
|
||||
'value': comment_rating_old['value'],
|
||||
'createdBy': rater.slug,
|
||||
'comment_id': comment.id
|
||||
}
|
||||
cts = comment_rating_old.get('createdAt')
|
||||
if cts: comment_rating_dict['createdAt'] = date_parse(cts)
|
||||
try:
|
||||
comment_rating = CommentRating.create(**comment_rating_dict)
|
||||
comment_dict['ratings'].append(comment_rating_dict)
|
||||
except Exception as e:
|
||||
print(comment_rating_dict)
|
||||
raise e
|
||||
comment = Comment.create(**comment_dict)
|
||||
|
||||
comment_dict['id'] = comment.id
|
||||
comment_dict['ratings'] = []
|
||||
comment_dict['oid'] = entry['_id']
|
||||
# print(comment)
|
||||
for comment_rating_old in entry.get('ratings',[]):
|
||||
rater = session.query(User).filter(User.oid == comment_rating_old['createdBy']).first()
|
||||
if rater and comment:
|
||||
comment_rating_dict = {
|
||||
'value': comment_rating_old['value'],
|
||||
'createdBy': rater.slug,
|
||||
'comment_id': comment.id
|
||||
}
|
||||
cts = comment_rating_old.get('createdAt')
|
||||
if cts: comment_rating_dict['createdAt'] = date_parse(cts)
|
||||
try:
|
||||
CommentRating.create(**comment_rating_dict)
|
||||
comment_dict['ratings'].append(comment_rating_dict)
|
||||
except Exception as e:
|
||||
print('[migration] comment rating error: %r' % comment_rating_dict)
|
||||
raise e
|
||||
else:
|
||||
print('[migration] error: cannot find shout for comment %r' % comment_dict)
|
||||
return comment_dict
|
||||
|
||||
def migrate_2stage(cmt, old_new_id):
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
from orm.base import local_session
|
||||
from orm import Topic, Community
|
||||
from dateutil.parser import parse as date_parse
|
||||
import json
|
||||
from migration.html2text import html2text
|
||||
import sqlalchemy
|
||||
|
||||
def migrate(entry, topics_by_oid):
|
||||
'''
|
||||
type Topic {
|
||||
slug: String! # ID
|
||||
createdBy: Int! # User
|
||||
createdAt: DateTime!
|
||||
value: String
|
||||
children: [String] # children topic
|
||||
}
|
||||
'''
|
||||
topic_dict = {
|
||||
'slug': entry['slug'],
|
||||
'oid': entry['_id'],
|
||||
# 'createdBy': entry['createdBy'],
|
||||
# 'createdAt': date_parse(entry['createdAt']),
|
||||
'title': entry['title'].replace(' ', ' '), #.lower(),
|
||||
'children': [],
|
||||
'community' : Community.default_community.slug,
|
||||
'body' : html2text(entry.get('description', '').replace(' ', ' '))
|
||||
}
|
||||
retopics = json.loads(open('migration/tables/replacements.json').read())
|
||||
with local_session() as session:
|
||||
slug = topics_by_oid.get(topic_dict['oid'], topic_dict)['slug']
|
||||
if slug:
|
||||
slug = retopics.get(slug, slug)
|
||||
try:
|
||||
topic = session.query(Topic).filter(Topic.slug == slug).first()
|
||||
if not topic:
|
||||
del topic_dict['oid']
|
||||
topic = Topic.create(**topic_dict)
|
||||
# print('created')
|
||||
else:
|
||||
if len(topic.title) > len(topic_dict['title']) or \
|
||||
len(topic.body) < len(topic_dict['body']):
|
||||
topic.update({
|
||||
'slug': slug,
|
||||
'title': topic_dict['title'] if len(topic.title) > len(topic_dict['title']) else topic.title,
|
||||
'body': topic_dict['body'] if len(topic.body) < len(topic_dict['body']) else topic.body
|
||||
})
|
||||
except Exception as e:
|
||||
print('not found old topic: ' + slug)
|
||||
else:
|
||||
raise Exception
|
||||
topic_dict['oid'] = entry['_id']
|
||||
return topic_dict
|
||||
@@ -1,23 +1,21 @@
|
||||
from dateutil.parser import parse as date_parse
|
||||
import frontmatter
|
||||
import json
|
||||
from orm import Shout, ShoutTopic, ShoutRating, ShoutViewByDay, User, shout
|
||||
import sqlalchemy
|
||||
from orm import Shout, ShoutTopic, ShoutRating, ShoutViewByDay, User
|
||||
from transliterate import translit
|
||||
from datetime import datetime
|
||||
from orm.base import local_session
|
||||
from orm.community import Community
|
||||
from migration.extract import prepare_body
|
||||
import os
|
||||
from orm.community import Community
|
||||
|
||||
DISCOURS_USER = {
|
||||
'id': 9999999,
|
||||
'slug': 'discours',
|
||||
'name': 'Дискурс',
|
||||
'email': 'welcome@discours.io',
|
||||
'userpic': 'https://discours.io/images/logo-mini.svg',
|
||||
'createdAt': '2016-03-05 22:22:00.350000'
|
||||
}
|
||||
OLD_DATE = '2016-03-05 22:22:00.350000'
|
||||
retopics = json.loads(open('migration/tables/replacements.json').read())
|
||||
ts = datetime.now()
|
||||
type2layout = {
|
||||
'Article': 'article',
|
||||
@@ -27,18 +25,6 @@ type2layout = {
|
||||
'Image': 'image'
|
||||
}
|
||||
|
||||
def get_metadata(r):
|
||||
metadata = {}
|
||||
metadata['title'] = r.get('title', '').replace('{', '(').replace('}', ')')
|
||||
metadata['authors'] = r.get('authors')
|
||||
metadata['createdAt'] = r.get('createdAt', ts)
|
||||
metadata['layout'] = r['layout']
|
||||
metadata['topics'] = [topic['slug'] for topic in r['topics']]
|
||||
metadata['topics'].sort()
|
||||
if r.get('cover', False):
|
||||
metadata['cover'] = r.get('cover')
|
||||
return metadata
|
||||
|
||||
def get_shout_slug(entry):
|
||||
slug = entry.get('slug', '')
|
||||
if not slug:
|
||||
@@ -47,18 +33,51 @@ def get_shout_slug(entry):
|
||||
if slug: break
|
||||
return slug
|
||||
|
||||
def migrate(entry, users_by_oid, topics_by_oid):
|
||||
def migrate(entry, storage):
|
||||
# init, set title and layout
|
||||
r = {
|
||||
'layout': type2layout[entry['type']],
|
||||
'title': entry['title'],
|
||||
'community': Community.default_community.id,
|
||||
'community': 0,
|
||||
'authors': [],
|
||||
'topics': [],
|
||||
'rating': 0,
|
||||
'ratings': [],
|
||||
'createdAt': []
|
||||
}
|
||||
topics_by_oid = storage['topics']['by_oid']
|
||||
users_by_oid = storage['users']['by_oid']
|
||||
|
||||
# author
|
||||
|
||||
oid = entry.get('createdBy', entry.get('_id', entry.get('oid')))
|
||||
userdata = users_by_oid.get(oid)
|
||||
if not userdata:
|
||||
app = entry.get('application')
|
||||
if app:
|
||||
userslug = translit(app['name'], 'ru', reversed=True)\
|
||||
.replace(' ', '-')\
|
||||
.replace('\'', '')\
|
||||
.replace('.', '-').lower()
|
||||
userdata = {
|
||||
'username': app['email'],
|
||||
'email': app['email'],
|
||||
'name': app['name'],
|
||||
'bio': app.get('bio', ''),
|
||||
'emailConfirmed': False,
|
||||
'slug': userslug,
|
||||
'createdAt': ts,
|
||||
'wasOnlineAt': ts
|
||||
}
|
||||
else:
|
||||
userdata = {
|
||||
'name': 'Дискурс',
|
||||
'slug': 'discours',
|
||||
'email': 'welcome@discours.io',
|
||||
'userpic': 'https://discours.io/image/logo-mini.svg'
|
||||
}
|
||||
assert userdata, 'no user found for %s from ' % [oid, len(users_by_oid.keys())]
|
||||
r['authors'] = [userdata, ]
|
||||
|
||||
# slug
|
||||
|
||||
@@ -72,8 +91,7 @@ def migrate(entry, users_by_oid, topics_by_oid):
|
||||
c = 'https://assets.discours.io/unsafe/1600x/' + entry['thumborId']
|
||||
else:
|
||||
c = entry.get('image', {}).get('url')
|
||||
if not c or 'cloudinary' in c:
|
||||
c = ''
|
||||
if not c or 'cloudinary' in c: c = ''
|
||||
r['cover'] = c
|
||||
|
||||
# timestamps
|
||||
@@ -85,111 +103,105 @@ def migrate(entry, users_by_oid, topics_by_oid):
|
||||
if r['publishedAt'] == OLD_DATE: r['publishedAt'] = ts
|
||||
if 'deletedAt' in entry: r['deletedAt'] = date_parse(entry['deletedAt'])
|
||||
|
||||
# connected users' data
|
||||
|
||||
# r['deletedBy'] = entry.get('deletedBy', '0') # TypeError: 'deletedBy' is an invalid keyword argument for Shout
|
||||
|
||||
oid = entry.get('createdBy', '')
|
||||
userdata = users_by_oid.get(oid, {})
|
||||
if not userdata.get('slug'):
|
||||
app = entry.get('application')
|
||||
if app:
|
||||
userslug = translit(app['name'], 'ru', reversed=True).replace(' ', '-').replace('\'', '').replace('.', '-').lower()
|
||||
userdata = {
|
||||
'username': app['email'],
|
||||
'email': app['email'],
|
||||
'name': app['name'],
|
||||
'bio': app.get('bio', ''),
|
||||
'emailConfirmed': False,
|
||||
'slug': userslug,
|
||||
'createdAt': ts,
|
||||
'wasOnlineAt': ts
|
||||
}
|
||||
if userdata == {}:
|
||||
userdata = {
|
||||
'name': 'Дискурс',
|
||||
'slug': 'discours',
|
||||
'userpic': 'https://discours.io/image/logo-mini.svg'
|
||||
}
|
||||
|
||||
author = { # a short version for public listings
|
||||
'slug': userdata.get('slug', 'discours'),
|
||||
'name': userdata.get('name', 'Дискурс'),
|
||||
'userpic': userdata.get('userpic', '')
|
||||
}
|
||||
r['authors'] = [ author, ]
|
||||
# body
|
||||
|
||||
body = prepare_body(entry)
|
||||
|
||||
# save mdx for prerender if published
|
||||
|
||||
r['body'] = body
|
||||
if entry.get('published'):
|
||||
content = ''
|
||||
metadata = get_metadata(r)
|
||||
content = frontmatter.dumps(frontmatter.Post(r['body'], **metadata))
|
||||
ext = 'mdx'
|
||||
parentDir = '/'.join(os.getcwd().split('/')[:-1])
|
||||
filepath = parentDir + '/discoursio-web/content/' + r['slug']
|
||||
# print(filepath)
|
||||
bc = bytes(content,'utf-8').decode('utf-8','ignore')
|
||||
open(filepath + '.' + ext, 'w').write(bc)
|
||||
# open(filepath + '.html', 'w').write(body_orig)
|
||||
|
||||
|
||||
# topics
|
||||
|
||||
category = entry['category']
|
||||
mainTopic = topics_by_oid.get(category)
|
||||
if mainTopic:
|
||||
r['mainTopic'] = mainTopic["slug"]
|
||||
r['mainTopic'] = storage['replacements'].get(mainTopic["slug"], mainTopic["slug"])
|
||||
topic_oids = [category, ]
|
||||
topic_errors = []
|
||||
topic_oids.extend(entry.get('tags', []))
|
||||
for oid in topic_oids:
|
||||
if oid in topics_by_oid:
|
||||
r['topics'].append(topics_by_oid[oid])
|
||||
if oid in storage['topics']['by_oid']:
|
||||
r['topics'].append(storage['topics']['by_oid'][oid]['slug'])
|
||||
else:
|
||||
# print('ERROR: unknown old topic id: ' + oid)
|
||||
topic_errors.append(oid)
|
||||
|
||||
# set prepared shout data
|
||||
print('[migration] unknown old topic id: ' + oid)
|
||||
|
||||
shout_dict = r.copy()
|
||||
del shout_dict['topics'] # FIXME: AttributeError: 'str' object has no attribute '_sa_instance_state'
|
||||
del shout_dict['rating'] # FIXME: TypeError: 'rating' is an invalid keyword argument for Shout
|
||||
del shout_dict['ratings']
|
||||
entry['topics'] = r['topics']
|
||||
entry['cover'] = r['cover']
|
||||
entry['authors'] = r['authors']
|
||||
|
||||
# get author
|
||||
|
||||
user = None
|
||||
email = userdata.get('email')
|
||||
authorslug = userdata.get('slug')
|
||||
with local_session() as session:
|
||||
try:
|
||||
if email: user = session.query(User).filter(User.email == email).first()
|
||||
if not user and authorslug: user = session.query(User).filter(User.slug == authorslug).first()
|
||||
if not user and userdata: user = User.create(**userdata)
|
||||
except:
|
||||
print('[migration] shout author error: \n%r' % entry)
|
||||
raise Exception
|
||||
assert user, 'could not get a user'
|
||||
shout_dict['authors'] = [ user, ]
|
||||
# body
|
||||
r['body'] = prepare_body(entry)
|
||||
|
||||
# save shout to db
|
||||
|
||||
s = object()
|
||||
try: s = Shout.create(**shout_dict)
|
||||
except: print('[migration] shout create error: \n%r' % shout_dict)
|
||||
shout_dict = r.copy()
|
||||
user = None
|
||||
del shout_dict['topics'] # FIXME: AttributeError: 'str' object has no attribute '_sa_instance_state'
|
||||
del shout_dict['rating'] # FIXME: TypeError: 'rating' is an invalid keyword argument for Shout
|
||||
del shout_dict['ratings']
|
||||
email = userdata.get('email')
|
||||
slug = userdata.get('slug')
|
||||
with local_session() as session:
|
||||
# c = session.query(Community).all().pop()
|
||||
if email: user = session.query(User).filter(User.email == email).first()
|
||||
if not user and slug: user = session.query(User).filter(User.slug == slug).first()
|
||||
if not user and userdata:
|
||||
try: user = User.create(**userdata)
|
||||
except sqlalchemy.exc.IntegrityError:
|
||||
print('[migration] user error: ' + userdata)
|
||||
userdata['id'] = user.id
|
||||
userdata['createdAt'] = user.createdAt
|
||||
storage['users']['by_slug'][userdata['slug']] = userdata
|
||||
storage['users']['by_oid'][entry['_id']] = userdata
|
||||
assert user, 'could not get a user'
|
||||
shout_dict['authors'] = [ user, ]
|
||||
|
||||
try:
|
||||
s = Shout.create(**shout_dict)
|
||||
except sqlalchemy.exc.IntegrityError:
|
||||
with local_session() as session:
|
||||
s = session.query(Shout).filter(Shout.slug == shout_dict['slug']).first()
|
||||
bump = False
|
||||
if s:
|
||||
for key in shout_dict:
|
||||
if key in s.__dict__:
|
||||
if s.__dict__[key] != shout_dict[key]:
|
||||
print('[migration] shout already exists, but differs in %s' % key)
|
||||
bump = True
|
||||
else:
|
||||
print('[migration] shout already exists, but lacks %s' % key)
|
||||
bump = True
|
||||
if bump:
|
||||
s.update(shout_dict)
|
||||
else:
|
||||
print('[migration] something went wrong with shout: \n%r' % shout_dict)
|
||||
session.commit()
|
||||
except:
|
||||
print(s)
|
||||
raise Exception
|
||||
|
||||
|
||||
# shout topics aftermath
|
||||
shout_dict['topics'] = []
|
||||
for tpc in r['topics']:
|
||||
oldslug = tpc
|
||||
newslug = storage['replacements'].get(oldslug, oldslug)
|
||||
if newslug:
|
||||
with local_session() as session:
|
||||
shout_topic_old = session.query(ShoutTopic)\
|
||||
.filter(ShoutTopic.shout == s.slug)\
|
||||
.filter(ShoutTopic.topic == oldslug).first()
|
||||
if shout_topic_old:
|
||||
shout_topic_old.update({ 'slug': newslug })
|
||||
else:
|
||||
shout_topic_new = session.query(ShoutTopic)\
|
||||
.filter(ShoutTopic.shout == s.slug)\
|
||||
.filter(ShoutTopic.topic == newslug).first()
|
||||
if not shout_topic_new: ShoutTopic.create(**{ 'shout': s.slug, 'topic': newslug })
|
||||
session.commit()
|
||||
shout_dict['topics'].append(newslug)
|
||||
else:
|
||||
print('[migration] ignored topic slug: \n%r' % tpc['slug'])
|
||||
# raise Exception
|
||||
|
||||
# shout ratings
|
||||
try:
|
||||
shout_dict['ratings'] = []
|
||||
for shout_rating_old in entry.get('ratings',[]):
|
||||
with local_session() as session:
|
||||
rater = session.query(User).filter(User.old_id == shout_rating_old['createdBy']).first()
|
||||
rater = session.query(User).filter(User.oid == shout_rating_old['createdBy']).first()
|
||||
if rater:
|
||||
shout_rating_dict = {
|
||||
'value': shout_rating_old['value'],
|
||||
@@ -210,43 +222,10 @@ def migrate(entry, users_by_oid, topics_by_oid):
|
||||
print('[migration] shout rating error: \n%r' % shout_rating_old)
|
||||
# raise Exception
|
||||
|
||||
# shout topics
|
||||
try:
|
||||
shout_dict['topics'] = []
|
||||
for topic in r['topics']:
|
||||
tpc = topics_by_oid[topic['oid']]
|
||||
oldslug = tpc['slug']
|
||||
newslug = retopics.get(oldslug, oldslug)
|
||||
need_create_topic = False
|
||||
if newslug:
|
||||
with local_session() as session:
|
||||
shout_topic_new = session.query(ShoutTopic)\
|
||||
.filter(ShoutTopic.shout == s.slug)\
|
||||
.filter(ShoutTopic.topic == newslug).first()
|
||||
shout_topic_old = session.query(ShoutTopic)\
|
||||
.filter(ShoutTopic.shout == s.slug)\
|
||||
.filter(ShoutTopic.topic == oldslug).first()
|
||||
if not shout_topic_new:
|
||||
if shout_topic_old:
|
||||
shout_topic_old.update({ 'slug': newslug })
|
||||
else:
|
||||
need_create_topic = True
|
||||
if need_create_topic:
|
||||
ShoutTopic.create(**{ 'shout': s.slug, 'topic': newslug })
|
||||
shout_dict['topics'].append(newslug)
|
||||
except:
|
||||
print('[migration] shout topic error: \n%r' % topic)
|
||||
raise Exception
|
||||
|
||||
# shout views
|
||||
try:
|
||||
views = entry.get('views', 1)
|
||||
ShoutViewByDay.create(
|
||||
shout = s.slug,
|
||||
value = views
|
||||
)
|
||||
except:
|
||||
print('[migration] shout view error: \n%r' % entry)
|
||||
# raise Exception
|
||||
shout_dict['old_id'] = entry.get('_id')
|
||||
return shout_dict, topic_errors
|
||||
ShoutViewByDay.create( shout = s.slug, value = entry.get('views', 1) )
|
||||
del shout_dict['ratings']
|
||||
shout_dict['oid'] = entry.get('_id')
|
||||
storage['shouts']['by_oid'][entry['_id']] = shout_dict
|
||||
storage['shouts']['by_slug'][slug] = shout_dict
|
||||
return shout_dict
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
import json
|
||||
from datetime import datetime
|
||||
from orm.base import local_session
|
||||
from orm import Topic, Community
|
||||
from dateutil.parser import parse as date_parse
|
||||
|
||||
def migrate(entry, topics_by_oid):
|
||||
'''
|
||||
type Topic {
|
||||
slug: String! # ID
|
||||
createdBy: Int! # User
|
||||
createdAt: DateTime!
|
||||
title: String
|
||||
parents: [String] # NOTE: topic can have parent topics
|
||||
children: [String] # and children
|
||||
}
|
||||
'''
|
||||
if type(entry['createdAt']) == type(''):
|
||||
ts = date_parse(entry['createdAt'])
|
||||
else:
|
||||
ts = datetime.fromtimestamp(entry['createdAt']/1000)
|
||||
topic_dict = {
|
||||
'slug': entry['slug'],
|
||||
'oid': entry['_id'],
|
||||
# 'createdBy': entry['createdBy'],
|
||||
# 'createdAt': ts,
|
||||
'title': entry['title'].replace(' ', ' '), # .lower(),
|
||||
'children': [],
|
||||
'community' : Community.default_community.slug,
|
||||
'body' : entry.get('description','').replace(' ', ' ')
|
||||
}
|
||||
try:
|
||||
retopics = json.loads(open('migration/tables/replacements.json').read())
|
||||
with local_session() as session:
|
||||
slug = topics_by_oid.get(topic_dict['oid'], topic_dict)['slug']
|
||||
slug = retopics.get(slug, slug)
|
||||
if slug:
|
||||
topic = session.query(Topic).filter(Topic.slug == slug).first()
|
||||
if not topic:
|
||||
del topic_dict['oid']
|
||||
topic = Topic.create(**topic_dict)
|
||||
except Exception as e:
|
||||
# print(e)
|
||||
raise e
|
||||
topic_dict['oid'] = entry['_id']
|
||||
return topic_dict
|
||||
28
migration/tables/topics.py
Normal file
28
migration/tables/topics.py
Normal file
@@ -0,0 +1,28 @@
|
||||
from migration.extract import extract, html2text
|
||||
from orm.base import local_session
|
||||
from orm import Topic, Community
|
||||
|
||||
def migrate(entry):
|
||||
body_orig = entry.get('description', '').replace(' ', ' ')
|
||||
topic_dict = {
|
||||
'slug': entry['slug'],
|
||||
'oid': entry['_id'],
|
||||
'title': entry['title'].replace(' ', ' '), #.lower(),
|
||||
'children': [],
|
||||
'community' : Community.default_community.slug
|
||||
}
|
||||
topic_dict['body'] = extract(html2text(body_orig), entry['_id'])
|
||||
with local_session() as session:
|
||||
slug = topic_dict['slug']
|
||||
topic = session.query(Topic).filter(Topic.slug == slug).first()
|
||||
if not topic:
|
||||
topic = Topic.create(**topic_dict)
|
||||
if len(topic.title) > len(topic_dict['title']):
|
||||
topic.update({ 'title': topic_dict['title'] })
|
||||
if len(topic.body) < len(topic_dict['body']):
|
||||
topic.update({ 'body': topic_dict['body'] })
|
||||
session.commit()
|
||||
# print(topic.__dict__)
|
||||
rt = topic.__dict__.copy()
|
||||
del rt['_sa_instance_state']
|
||||
return rt
|
||||
@@ -1,124 +1,114 @@
|
||||
import sqlalchemy
|
||||
from orm import User, Role, UserRating
|
||||
from orm import User, UserRating
|
||||
from orm.user import EmailSubscription
|
||||
import frontmatter
|
||||
from dateutil.parser import parse
|
||||
from migration.html2text import html2text
|
||||
from orm.base import local_session
|
||||
|
||||
def migrate(entry):
|
||||
'''
|
||||
|
||||
type User {
|
||||
username: String! # email
|
||||
createdAt: DateTime!
|
||||
email: String
|
||||
password: String
|
||||
oauth: String # provider:token
|
||||
name: String # to display
|
||||
userpic: String
|
||||
links: [String]
|
||||
emailConfirmed: Boolean # should contain all emails too
|
||||
id: Int!
|
||||
muted: Boolean
|
||||
roles: [Role]
|
||||
updatedAt: DateTime
|
||||
wasOnlineAt: DateTime
|
||||
ratings: [Rating]
|
||||
slug: String
|
||||
bio: String
|
||||
notifications: [Int]
|
||||
|
||||
if 'subscribedTo' in entry: del entry['subscribedTo']
|
||||
email = entry['emails'][0]['address']
|
||||
user_dict = {
|
||||
'oid': entry['_id'],
|
||||
'roles': [],
|
||||
'ratings': [],
|
||||
'username': email,
|
||||
'email': email,
|
||||
'password': entry['services']['password'].get('bcrypt', ''),
|
||||
'createdAt': parse(entry['createdAt']),
|
||||
'emailConfirmed': bool(entry['emails'][0]['verified']),
|
||||
'muted': False, # amnesty
|
||||
'bio': entry['profile'].get('bio', ''),
|
||||
'notifications': [],
|
||||
'createdAt': parse(entry['createdAt']),
|
||||
'roles': [], # entry['roles'] # roles by community
|
||||
'ratings': [], # entry['ratings']
|
||||
'links': [],
|
||||
'name': 'anonymous'
|
||||
}
|
||||
|
||||
'''
|
||||
res = {}
|
||||
res['old_id'] = entry['_id']
|
||||
res['password'] = entry['services']['password'].get('bcrypt', '')
|
||||
del entry['services']
|
||||
if 'subscribedTo' in entry: #TODO: use subscribedTo
|
||||
del entry['subscribedTo']
|
||||
res['username'] = entry['emails'][0]['address']
|
||||
res['email'] = res['username']
|
||||
res['wasOnlineAt'] = parse(entry.get('loggedInAt', entry['createdAt']))
|
||||
res['emailConfirmed'] = entry['emails'][0]['verified']
|
||||
res['createdAt'] = parse(entry['createdAt'])
|
||||
res['roles'] = [] # entry['roles'] # roles by community
|
||||
res['ratings'] = [] # entry['ratings']
|
||||
res['notifications'] = []
|
||||
res['links'] = []
|
||||
res['muted'] = False
|
||||
res['name'] = 'anonymous'
|
||||
if 'updatedAt' in entry: user_dict['updatedAt'] = parse(entry['updatedAt'])
|
||||
if 'wasOnineAt' in entry: user_dict['wasOnlineAt'] = parse(entry['wasOnlineAt'])
|
||||
if entry.get('profile'):
|
||||
# slug
|
||||
res['slug'] = entry['profile'].get('path')
|
||||
res['bio'] = entry['profile'].get('bio','')
|
||||
user_dict['slug'] = entry['profile'].get('path')
|
||||
user_dict['bio'] = entry['profile'].get('bio','')
|
||||
|
||||
# userpic
|
||||
try: res['userpic'] = 'https://assets.discours.io/unsafe/100x/' + entry['profile']['thumborId']
|
||||
try: user_dict['userpic'] = 'https://assets.discours.io/unsafe/100x/' + entry['profile']['thumborId']
|
||||
except KeyError:
|
||||
try: res['userpic'] = entry['profile']['image']['url']
|
||||
except KeyError: res['userpic'] = ''
|
||||
try: user_dict['userpic'] = entry['profile']['image']['url']
|
||||
except KeyError: user_dict['userpic'] = ''
|
||||
|
||||
# name
|
||||
fn = entry['profile'].get('firstName', '')
|
||||
ln = entry['profile'].get('lastName', '')
|
||||
name = res['slug'] if res['slug'] else 'anonymous'
|
||||
name = user_dict['slug'] if user_dict['slug'] else 'anonymous'
|
||||
name = fn if fn else name
|
||||
name = (name + ' ' + ln) if ln else name
|
||||
name = entry['profile']['path'].lower().replace(' ', '-') if len(name) < 2 else name
|
||||
res['name'] = name
|
||||
user_dict['name'] = name
|
||||
|
||||
# links
|
||||
fb = entry['profile'].get('facebook', False)
|
||||
if fb:
|
||||
res['links'].append(fb)
|
||||
if fb: user_dict['links'].append(fb)
|
||||
vk = entry['profile'].get('vkontakte', False)
|
||||
if vk:
|
||||
res['links'].append(vk)
|
||||
if vk: user_dict['links'].append(vk)
|
||||
tr = entry['profile'].get('twitter', False)
|
||||
if tr:
|
||||
res['links'].append(tr)
|
||||
if tr: user_dict['links'].append(tr)
|
||||
ws = entry['profile'].get('website', False)
|
||||
if ws:
|
||||
res['links'].append(ws)
|
||||
if ws: user_dict['links'].append(ws)
|
||||
|
||||
# some checks
|
||||
if not res['slug'] and len(res['links']) > 0: res['slug'] = res['links'][0].split('/')[-1]
|
||||
if not user_dict['slug'] and len(user_dict['links']) > 0:
|
||||
user_dict['slug'] = user_dict['links'][0].split('/')[-1]
|
||||
|
||||
res['slug'] = res.get('slug', res['email'].split('@')[0])
|
||||
old = res['old_id']
|
||||
user = User.create(**res.copy())
|
||||
res['id'] = user.id
|
||||
return res
|
||||
user_dict['slug'] = user_dict.get('slug', user_dict['email'].split('@')[0])
|
||||
oid = user_dict['oid']
|
||||
try: user = User.create(**user_dict.copy())
|
||||
except sqlalchemy.exc.IntegrityError:
|
||||
print('[migration] cannot create user ' + user_dict['slug'])
|
||||
with local_session() as session:
|
||||
old_user = session.query(User).filter(User.slug == user_dict['slug']).first()
|
||||
old_user.oid = oid
|
||||
user = old_user
|
||||
if not user:
|
||||
print('[migration] ERROR: cannot find user ' + user_dict['slug'])
|
||||
raise Exception
|
||||
user_dict['id'] = user.id
|
||||
return user_dict
|
||||
|
||||
def migrate_email_subscription(entry):
|
||||
res = {}
|
||||
res["email"] = entry["email"]
|
||||
res["createdAt"] = parse(entry["createdAt"])
|
||||
subscription = EmailSubscription.create(**res)
|
||||
EmailSubscription.create(**res)
|
||||
|
||||
def migrate_2stage(entry, id_map):
|
||||
ce = 0
|
||||
for rating_entry in entry.get('ratings',[]):
|
||||
rater_old_id = rating_entry['createdBy']
|
||||
rater_slug = id_map.get(rater_old_id)
|
||||
rater_oid = rating_entry['createdBy']
|
||||
rater_slug = id_map.get(rater_oid)
|
||||
if not rater_slug:
|
||||
ce +=1
|
||||
# print(rating_entry)
|
||||
continue
|
||||
old_id = entry['_id']
|
||||
oid = entry['_id']
|
||||
author_slug = id_map.get(oid)
|
||||
user_rating_dict = {
|
||||
'value': rating_entry['value'],
|
||||
'rater': rater_slug,
|
||||
'user': id_map.get(old_id)
|
||||
'user': author_slug
|
||||
}
|
||||
with local_session() as session:
|
||||
try:
|
||||
user_rating = UserRating.create(**user_rating_dict)
|
||||
except sqlalchemy.exc.IntegrityError:
|
||||
print('[migration] duplicate rating solving for ' + rater_slug)
|
||||
old_rating = session.query(UserRating).filter(UserRating.rater == rater_slug).first()
|
||||
old_rating.value = rating_entry['value'] + old_rating.value
|
||||
print('[migration] cannot create ' + author_slug + '`s rate from ' + rater_slug)
|
||||
print('[migration] concat rating value %d+%d=%d' % (old_rating.value, rating_entry['value'], old_rating.value + rating_entry['value']))
|
||||
old_rating.update({ 'value': old_rating.value + rating_entry['value'] })
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return ce
|
||||
|
||||
Reference in New Issue
Block a user