discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

migration wip

Browse Source
This commit is contained in:
Untone 2021-10-14 08:59:42 +03:00
parent 095211b1ff
commit 7ec763391b
4 changed files with 70 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
migrate.py
View File

@ -10,6 +10,7 @@ from migration.tables.tags import migrate as migrateTag
from migration.tables.comments import migrate as migrateComment
from migration.utils import DateTimeEncoder
from orm import Community
from dateutil.parser import parse as date_parse
IMG_REGEX = r"\!\[(.*?)\]\((data\:image\/(png|jpeg|jpg);base64\,(.*?))\)"
@ -68,20 +69,19 @@ def users():
def topics():
''' topics from categories and tags '''
print('migrating topics...')
cat_data = json.loads(
open('migration/data/content_item_categories.json').read())
# tag_data = json.loads(open('migration/data/tags.json').read())
new_data = {}
old_data = {}
cats_data = json.loads(open('migration/data/content_item_categories.json').read())
cat_topics = {}
slug_topics = {}
counter = 0
try:
for cat in cat_data:
for cat in cats_data:
topic = migrateCategory(cat)
old_data[topic['old_id']] = topic
new_data[topic['slug']] = topic
cat_topics[topic['cat_id']] = topic
slug_topics[topic['slug']] = topic
counter += 1
except Exception:
except Exception as e:
print('cats exception, try to remove database first')
raise e
'''
try:
for tag in tag_data:
@ -92,17 +92,20 @@ def topics():
print('tags exception, try to remove database first')
raise Exception
'''
export_list = sorted(new_data.items(), key=lambda item: str(
export_list = sorted(slug_topics.items(), key=lambda item: str(
item[1]['createdAt']))
open('migration/data/topics.dict.json',
'w').write(json.dumps(old_data, cls=DateTimeEncoder))
open('migration/data/topics.dict.json','w').write(json.dumps(cat_topics,
cls=DateTimeEncoder,
indent=4,
sort_keys=True,
ensure_ascii=False))
open('../src/data/topics.json', 'w').write(json.dumps(dict(export_list),
cls=DateTimeEncoder,
indent=4,
sort_keys=True,
ensure_ascii=False))
print(str(counter) + ' from ' + str(len(cat_data)) + ' cats were migrated')
#' tags and ' + str(len(tag_data)) +
print(str(counter) + ' / ' + str(len(cats_data)) + ' migrated')
print(str(len(export_list)) + ' topics were exported')
@ -277,7 +280,7 @@ if __name__ == '__main__':
'name': 'Дискурс',
'pic': 'https://discours.io/images/logo-min.svg',
'createdBy': '0',
'createdAt': OLD_DATE
'createdAt': date_parse(OLD_DATE)
})
except Exception:
pass

10
migration/tables/comments.py
View File

@ -48,8 +48,7 @@ def migrate(entry):
'''
with local_session() as session:
shout = session.query(Shout).filter(Shout.old_id == entry['_id']).first()
if not shout: print(entry)
assert shout, '=== NO SHOUT IN COMMENT ERROR ==='
if not shout: shout = session.query(Shout).first()
author = session.query(User).filter(User.old_id == entry['_id']).first()
comment_dict = {
'old_id': entry['_id'],
@ -65,14 +64,17 @@ def migrate(entry):
comment_dict['deletedBy'] = entry['updatedBy']
if 'thread' in entry:
comment_dict['old_thread'] = entry['thread']
# print(entry.keys())
print(comment_dict)
comment = Comment.create(**comment_dict)
print(comment)
for comment_rating_old in entry.get('ratings',[]):
rater_id = session.query(User).filter(User.old_id == comment_rating_old['createdBy']).first()
createdTs = comment_rating_old.get('createdAt', datetime.datetime.now())
u = entry.get('updatedAt', False)
comment_rating_dict = {
'value': comment_rating_old['value'],
'createdBy': rater_id or 0,
'createdAt': comment_rating_old.get('createdAt', datetime.datetime.now()),
'createdAt': createdTs,
'comment_id': comment.id
}
try:

23
migration/tables/content_item_categories.py
View File

@ -1,3 +1,7 @@
from orm.base import local_session
from orm import Topic
# from dateutil.parser import parse as date_parse
def migrate(entry):
'''
type Topic {
@ -16,12 +20,15 @@ def migrate(entry):
'title': entry['title'].lower(),
'parents': [],
'children': [],
'old_id': entry['_id']
'cat_id': entry['_id']
}
with local_session() as session:
topic = session.query(Topic).filter(Topic.slug == topic_slug).first()
if not topic:
topic = Topic.create(**topic_dict)
topic_dict['id'] = topic.id
return topic_dict
try:
with local_session() as session:
topic = session.query(Topic).filter(Topic.slug == entry['slug']).first()
if not topic:
topic = Topic.create(**topic_dict)
topic_dict['id'] = topic.id
return topic_dict
except Exception as e:
print(e)
return {}

58
migration/tables/content_items.py
View File

@ -1,4 +1,4 @@
from dateutil.parser import parse
from dateutil.parser import parse as date_parse
from os.path import abspath
import frontmatter
import json
@ -13,10 +13,14 @@ from orm.base import local_session
users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read())
print(str(len(users_dict.items())) + ' users loaded')
topics_dict = json.loads(open(abspath('migration/data/topics.dict.json')).read()) # old_id keyed
print(str(len(topics_dict.items())) + ' topics loaded')
cats_data = json.loads(open(abspath('migration/data/content_item_categories.json')).read()) # old_id keyed
cats_dict = { x['_id']: x for x in cats_data }
print(str(len(cats_data)) + ' categories loaded')
comments_data = json.loads(open(abspath('migration/data/comments.json')).read())
print(str(len(comments_data)) + ' comments loaded')
comments_by_post = {}
for comment in comments_data:
p = comment['contentItem']
@ -102,15 +106,15 @@ def migrate(entry):
# print(entry)
raise Exception
try:
r['topics'].append(topics_dict[entry['category']]['slug'])
r['topics'].append(cats_dict[entry['category']]['slug'])
except Exception:
print(entry['category'])
print(entry['category'])
if entry.get('image') is not None:
r['cover'] = entry['image']['url']
if entry.get('thumborId') is not None:
r['cover'] = 'https://assets.discours.io/unsafe/1600x/' + entry['thumborId']
if entry.get('updatedAt') is not None:
r['updatedAt'] = parse(entry['updatedAt'])
r['updatedAt'] = date_parse(entry['updatedAt'])
if entry.get('type') == 'Literature':
media = entry.get('media', '')
# print(media[0]['literatureBody'])
@ -125,16 +129,16 @@ def migrate(entry):
else:
print(r['slug'] + ': literature has no media')
elif entry.get('type') == 'Video':
m = entry['media'][0]
yt = m.get('youtubeId', '')
vm = m.get('vimeoId', '')
video_url = 'https://www.youtube.com/watch?v=' + yt if yt else '#'
if video_url == '#':
video_url = 'https://vimeo.com/' + vm if vm else '#'
if video_url == '#':
print(entry.get('media', 'NO MEDIA!'))
# raise Exception
r['body'] = '<ShoutVideo src=\"' + video_url + \
m = entry['media'][0]
yt = m.get('youtubeId', '')
vm = m.get('vimeoId', '')
video_url = 'https://www.youtube.com/watch?v=' + yt if yt else '#'
if video_url == '#':
video_url = 'https://vimeo.com/' + vm if vm else '#'
if video_url == '#':
print(entry.get('media', 'NO MEDIA!'))
# raise Exception
r['body'] = '<ShoutVideo src=\"' + video_url + \
'\" />' + html2text(m.get('body', '')) # FIXME
elif entry.get('type') == 'Music':
r['body'] = '<ShoutMusic media={\"' + \
@ -214,7 +218,6 @@ def migrate(entry):
del shout_dict['published']
try:
topic_slugs = shout_dict['topics']
del shout_dict['topics'] # FIXME: AttributeError: 'str' object has no attribute '_sa_instance_state'
del shout_dict['views'] # FIXME: TypeError: 'views' is an invalid keyword argument for Shout
del shout_dict['rating'] # FIXME: TypeError: 'rating' is an invalid keyword argument for Shout
@ -223,8 +226,7 @@ def migrate(entry):
r['id'] = s.id
if len(entry.get('ratings', [])) > 0:
# TODO: adding shout ratings
'''
# TODO: migrate shout ratings
shout_dict['ratings'] = []
for shout_rating_old in entry['ratings']:
shout_rating = ShoutRating.create(
@ -232,16 +234,20 @@ def migrate(entry):
shout_id = s.id,
value = shout_rating_old['value']
)
shout.ratings.append(shout_rating.id)
'''
# adding topics to created shout
for topic_slug in topic_slugs:
s.ratings.append(shout_rating.id)
s.save()
# TODO: migrate topics
'''
with local_session() as session:
for topic_slug in topic_slugs:
topic = session.query(Topic).filter(Topic.slug == topic_slug).first()
if not topic:
topic_dict = topics_dict.get(topic_slug)
topic_dict = migrateCategory()
if topic_dict:
topic = Topic.create(**topic_dict)
shout.topics = [ topic, ]
shout.save()
s.topics = [ topic, ]
s.save()
'''
except Exception as e:
r['error'] = 'db error'
# pass