discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

migration wip

Browse Source
This commit is contained in:
Untone 2021-10-14 08:59:42 +03:00
parent 095211b1ff
commit 7ec763391b
4 changed files with 70 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
migrate.py
View File

@ -10,6 +10,7 @@ from migration.tables.tags import migrate as migrateTag
from migration.tables.comments import migrate as migrateComment from migration.tables.comments import migrate as migrateComment
from migration.utils import DateTimeEncoder from migration.utils import DateTimeEncoder
from orm import Community from orm import Community
from dateutil.parser import parse as date_parse
IMG_REGEX = r"\!\[(.*?)\]\((data\:image\/(png|jpeg|jpg);base64\,(.*?))\)" IMG_REGEX = r"\!\[(.*?)\]\((data\:image\/(png|jpeg|jpg);base64\,(.*?))\)"
@ -68,20 +69,19 @@ def users():
def topics(): def topics():
''' topics from categories and tags ''' ''' topics from categories and tags '''
print('migrating topics...') print('migrating topics...')
cat_data = json.loads( cats_data = json.loads(open('migration/data/content_item_categories.json').read())
open('migration/data/content_item_categories.json').read()) cat_topics = {}
# tag_data = json.loads(open('migration/data/tags.json').read()) slug_topics = {}
new_data = {}
old_data = {}
counter = 0 counter = 0
try: try:
for cat in cat_data: for cat in cats_data:
topic = migrateCategory(cat) topic = migrateCategory(cat)
old_data[topic['old_id']] = topic cat_topics[topic['cat_id']] = topic
new_data[topic['slug']] = topic slug_topics[topic['slug']] = topic
counter += 1 counter += 1
except Exception: except Exception as e:
print('cats exception, try to remove database first') print('cats exception, try to remove database first')
raise e
''' '''
try: try:
for tag in tag_data: for tag in tag_data:
@ -92,17 +92,20 @@ def topics():
print('tags exception, try to remove database first') print('tags exception, try to remove database first')
raise Exception raise Exception
''' '''
export_list = sorted(new_data.items(), key=lambda item: str( export_list = sorted(slug_topics.items(), key=lambda item: str(
item[1]['createdAt'])) item[1]['createdAt']))
open('migration/data/topics.dict.json', open('migration/data/topics.dict.json','w').write(json.dumps(cat_topics,
'w').write(json.dumps(old_data, cls=DateTimeEncoder)) cls=DateTimeEncoder,
indent=4,
sort_keys=True,
ensure_ascii=False))
open('../src/data/topics.json', 'w').write(json.dumps(dict(export_list), open('../src/data/topics.json', 'w').write(json.dumps(dict(export_list),
cls=DateTimeEncoder, cls=DateTimeEncoder,
indent=4, indent=4,
sort_keys=True, sort_keys=True,
ensure_ascii=False)) ensure_ascii=False))
print(str(counter) + ' from ' + str(len(cat_data)) + ' cats were migrated')
#' tags and ' + str(len(tag_data)) + #' tags and ' + str(len(tag_data)) +
print(str(counter) + ' / ' + str(len(cats_data)) + ' migrated')
print(str(len(export_list)) + ' topics were exported') print(str(len(export_list)) + ' topics were exported')
@ -277,7 +280,7 @@ if __name__ == '__main__':
'name': 'Дискурс', 'name': 'Дискурс',
'pic': 'https://discours.io/images/logo-min.svg', 'pic': 'https://discours.io/images/logo-min.svg',
'createdBy': '0', 'createdBy': '0',
'createdAt': OLD_DATE 'createdAt': date_parse(OLD_DATE)
}) })
except Exception: except Exception:
pass pass

10
migration/tables/comments.py
View File

@ -48,8 +48,7 @@ def migrate(entry):
''' '''
with local_session() as session: with local_session() as session:
shout = session.query(Shout).filter(Shout.old_id == entry['_id']).first() shout = session.query(Shout).filter(Shout.old_id == entry['_id']).first()
if not shout: print(entry) if not shout: shout = session.query(Shout).first()
assert shout, '=== NO SHOUT IN COMMENT ERROR ==='
author = session.query(User).filter(User.old_id == entry['_id']).first() author = session.query(User).filter(User.old_id == entry['_id']).first()
comment_dict = { comment_dict = {
'old_id': entry['_id'], 'old_id': entry['_id'],
@ -65,14 +64,17 @@ def migrate(entry):
comment_dict['deletedBy'] = entry['updatedBy'] comment_dict['deletedBy'] = entry['updatedBy']
if 'thread' in entry: if 'thread' in entry:
comment_dict['old_thread'] = entry['thread'] comment_dict['old_thread'] = entry['thread']
# print(entry.keys()) print(comment_dict)
comment = Comment.create(**comment_dict) comment = Comment.create(**comment_dict)
print(comment)
for comment_rating_old in entry.get('ratings',[]): for comment_rating_old in entry.get('ratings',[]):
rater_id = session.query(User).filter(User.old_id == comment_rating_old['createdBy']).first() rater_id = session.query(User).filter(User.old_id == comment_rating_old['createdBy']).first()
createdTs = comment_rating_old.get('createdAt', datetime.datetime.now())
u = entry.get('updatedAt', False)
comment_rating_dict = { comment_rating_dict = {
'value': comment_rating_old['value'], 'value': comment_rating_old['value'],
'createdBy': rater_id or 0, 'createdBy': rater_id or 0,
'createdAt': comment_rating_old.get('createdAt', datetime.datetime.now()), 'createdAt': createdTs,
'comment_id': comment.id 'comment_id': comment.id
} }
try: try:

13
migration/tables/content_item_categories.py
View File

@ -1,3 +1,7 @@
from orm.base import local_session
from orm import Topic
# from dateutil.parser import parse as date_parse
def migrate(entry): def migrate(entry):
''' '''
type Topic { type Topic {
@ -16,12 +20,15 @@ def migrate(entry):
'title': entry['title'].lower(), 'title': entry['title'].lower(),
'parents': [], 'parents': [],
'children': [], 'children': [],
'old_id': entry['_id'] 'cat_id': entry['_id']
} }
try:
with local_session() as session: with local_session() as session:
topic = session.query(Topic).filter(Topic.slug == topic_slug).first() topic = session.query(Topic).filter(Topic.slug == entry['slug']).first()
if not topic: if not topic:
topic = Topic.create(**topic_dict) topic = Topic.create(**topic_dict)
topic_dict['id'] = topic.id topic_dict['id'] = topic.id
return topic_dict return topic_dict
except Exception as e:
print(e)
return {}

32
migration/tables/content_items.py
View File

@ -1,4 +1,4 @@
from dateutil.parser import parse from dateutil.parser import parse as date_parse
from os.path import abspath from os.path import abspath
import frontmatter import frontmatter
import json import json
@ -13,10 +13,14 @@ from orm.base import local_session
users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read()) users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read())
print(str(len(users_dict.items())) + ' users loaded') print(str(len(users_dict.items())) + ' users loaded')
topics_dict = json.loads(open(abspath('migration/data/topics.dict.json')).read()) # old_id keyed
print(str(len(topics_dict.items())) + ' topics loaded') cats_data = json.loads(open(abspath('migration/data/content_item_categories.json')).read()) # old_id keyed
cats_dict = { x['_id']: x for x in cats_data }
print(str(len(cats_data)) + ' categories loaded')
comments_data = json.loads(open(abspath('migration/data/comments.json')).read()) comments_data = json.loads(open(abspath('migration/data/comments.json')).read())
print(str(len(comments_data)) + ' comments loaded') print(str(len(comments_data)) + ' comments loaded')
comments_by_post = {} comments_by_post = {}
for comment in comments_data: for comment in comments_data:
p = comment['contentItem'] p = comment['contentItem']
@ -102,7 +106,7 @@ def migrate(entry):
# print(entry) # print(entry)
raise Exception raise Exception
try: try:
r['topics'].append(topics_dict[entry['category']]['slug']) r['topics'].append(cats_dict[entry['category']]['slug'])
except Exception: except Exception:
print(entry['category']) print(entry['category'])
if entry.get('image') is not None: if entry.get('image') is not None:
@ -110,7 +114,7 @@ def migrate(entry):
if entry.get('thumborId') is not None: if entry.get('thumborId') is not None:
r['cover'] = 'https://assets.discours.io/unsafe/1600x/' + entry['thumborId'] r['cover'] = 'https://assets.discours.io/unsafe/1600x/' + entry['thumborId']
if entry.get('updatedAt') is not None: if entry.get('updatedAt') is not None:
r['updatedAt'] = parse(entry['updatedAt']) r['updatedAt'] = date_parse(entry['updatedAt'])
if entry.get('type') == 'Literature': if entry.get('type') == 'Literature':
media = entry.get('media', '') media = entry.get('media', '')
# print(media[0]['literatureBody']) # print(media[0]['literatureBody'])
@ -214,7 +218,6 @@ def migrate(entry):
del shout_dict['published'] del shout_dict['published']
try: try:
topic_slugs = shout_dict['topics']
del shout_dict['topics'] # FIXME: AttributeError: 'str' object has no attribute '_sa_instance_state' del shout_dict['topics'] # FIXME: AttributeError: 'str' object has no attribute '_sa_instance_state'
del shout_dict['views'] # FIXME: TypeError: 'views' is an invalid keyword argument for Shout del shout_dict['views'] # FIXME: TypeError: 'views' is an invalid keyword argument for Shout
del shout_dict['rating'] # FIXME: TypeError: 'rating' is an invalid keyword argument for Shout del shout_dict['rating'] # FIXME: TypeError: 'rating' is an invalid keyword argument for Shout
@ -223,8 +226,7 @@ def migrate(entry):
r['id'] = s.id r['id'] = s.id
if len(entry.get('ratings', [])) > 0: if len(entry.get('ratings', [])) > 0:
# TODO: adding shout ratings # TODO: migrate shout ratings
'''
shout_dict['ratings'] = [] shout_dict['ratings'] = []
for shout_rating_old in entry['ratings']: for shout_rating_old in entry['ratings']:
shout_rating = ShoutRating.create( shout_rating = ShoutRating.create(
@ -232,16 +234,20 @@ def migrate(entry):
shout_id = s.id, shout_id = s.id,
value = shout_rating_old['value'] value = shout_rating_old['value']
) )
shout.ratings.append(shout_rating.id) s.ratings.append(shout_rating.id)
s.save()
# TODO: migrate topics
''' '''
# adding topics to created shout with local_session() as session:
for topic_slug in topic_slugs: for topic_slug in topic_slugs:
topic = session.query(Topic).filter(Topic.slug == topic_slug).first()
if not topic: if not topic:
topic_dict = topics_dict.get(topic_slug) topic_dict = migrateCategory()
if topic_dict: if topic_dict:
topic = Topic.create(**topic_dict) topic = Topic.create(**topic_dict)
shout.topics = [ topic, ] s.topics = [ topic, ]
shout.save() s.save()
'''
except Exception as e: except Exception as e:
r['error'] = 'db error' r['error'] = 'db error'
# pass # pass