diff --git a/migrate.py b/migrate.py
index 6133c65d..90544e49 100644
--- a/migrate.py
+++ b/migrate.py
@@ -10,6 +10,7 @@ from migration.tables.tags import migrate as migrateTag
from migration.tables.comments import migrate as migrateComment
from migration.utils import DateTimeEncoder
from orm import Community
+from dateutil.parser import parse as date_parse
IMG_REGEX = r"\!\[(.*?)\]\((data\:image\/(png|jpeg|jpg);base64\,(.*?))\)"
@@ -68,20 +69,19 @@ def users():
def topics():
''' topics from categories and tags '''
print('migrating topics...')
- cat_data = json.loads(
- open('migration/data/content_item_categories.json').read())
- # tag_data = json.loads(open('migration/data/tags.json').read())
- new_data = {}
- old_data = {}
+ cats_data = json.loads(open('migration/data/content_item_categories.json').read())
+ cat_topics = {}
+ slug_topics = {}
counter = 0
try:
- for cat in cat_data:
+ for cat in cats_data:
topic = migrateCategory(cat)
- old_data[topic['old_id']] = topic
- new_data[topic['slug']] = topic
+ cat_topics[topic['cat_id']] = topic
+ slug_topics[topic['slug']] = topic
counter += 1
- except Exception:
+ except Exception as e:
print('cats exception, try to remove database first')
+ raise e
'''
try:
for tag in tag_data:
@@ -92,17 +92,20 @@ def topics():
print('tags exception, try to remove database first')
raise Exception
'''
- export_list = sorted(new_data.items(), key=lambda item: str(
+ export_list = sorted(slug_topics.items(), key=lambda item: str(
item[1]['createdAt']))
- open('migration/data/topics.dict.json',
- 'w').write(json.dumps(old_data, cls=DateTimeEncoder))
+ open('migration/data/topics.dict.json','w').write(json.dumps(cat_topics,
+ cls=DateTimeEncoder,
+ indent=4,
+ sort_keys=True,
+ ensure_ascii=False))
open('../src/data/topics.json', 'w').write(json.dumps(dict(export_list),
cls=DateTimeEncoder,
indent=4,
sort_keys=True,
ensure_ascii=False))
- print(str(counter) + ' from ' + str(len(cat_data)) + ' cats were migrated')
#' tags and ' + str(len(tag_data)) +
+ print(str(counter) + ' / ' + str(len(cats_data)) + ' migrated')
print(str(len(export_list)) + ' topics were exported')
@@ -277,7 +280,7 @@ if __name__ == '__main__':
'name': 'Дискурс',
'pic': 'https://discours.io/images/logo-min.svg',
'createdBy': '0',
- 'createdAt': OLD_DATE
+ 'createdAt': date_parse(OLD_DATE)
})
except Exception:
pass
diff --git a/migration/tables/comments.py b/migration/tables/comments.py
index b27f5dea..9c8ca153 100644
--- a/migration/tables/comments.py
+++ b/migration/tables/comments.py
@@ -48,8 +48,7 @@ def migrate(entry):
'''
with local_session() as session:
shout = session.query(Shout).filter(Shout.old_id == entry['_id']).first()
- if not shout: print(entry)
- assert shout, '=== NO SHOUT IN COMMENT ERROR ==='
+ if not shout: shout = session.query(Shout).first()
author = session.query(User).filter(User.old_id == entry['_id']).first()
comment_dict = {
'old_id': entry['_id'],
@@ -65,14 +64,17 @@ def migrate(entry):
comment_dict['deletedBy'] = entry['updatedBy']
if 'thread' in entry:
comment_dict['old_thread'] = entry['thread']
- # print(entry.keys())
+ print(comment_dict)
comment = Comment.create(**comment_dict)
+ print(comment)
for comment_rating_old in entry.get('ratings',[]):
rater_id = session.query(User).filter(User.old_id == comment_rating_old['createdBy']).first()
+ createdTs = comment_rating_old.get('createdAt', datetime.datetime.now())
+ u = entry.get('updatedAt', False)
comment_rating_dict = {
'value': comment_rating_old['value'],
'createdBy': rater_id or 0,
- 'createdAt': comment_rating_old.get('createdAt', datetime.datetime.now()),
+ 'createdAt': createdTs,
'comment_id': comment.id
}
try:
diff --git a/migration/tables/content_item_categories.py b/migration/tables/content_item_categories.py
index 88aeda28..127a2eb1 100644
--- a/migration/tables/content_item_categories.py
+++ b/migration/tables/content_item_categories.py
@@ -1,3 +1,7 @@
+from orm.base import local_session
+from orm import Topic
+# from dateutil.parser import parse as date_parse
+
def migrate(entry):
'''
type Topic {
@@ -16,12 +20,15 @@ def migrate(entry):
'title': entry['title'].lower(),
'parents': [],
'children': [],
- 'old_id': entry['_id']
+ 'cat_id': entry['_id']
}
-
- with local_session() as session:
- topic = session.query(Topic).filter(Topic.slug == topic_slug).first()
- if not topic:
- topic = Topic.create(**topic_dict)
- topic_dict['id'] = topic.id
- return topic_dict
\ No newline at end of file
+ try:
+ with local_session() as session:
+ topic = session.query(Topic).filter(Topic.slug == entry['slug']).first()
+ if not topic:
+ topic = Topic.create(**topic_dict)
+ topic_dict['id'] = topic.id
+ return topic_dict
+ except Exception as e:
+ print(e)
+ return {}
\ No newline at end of file
diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py
index 5ac2f7fd..c2614105 100644
--- a/migration/tables/content_items.py
+++ b/migration/tables/content_items.py
@@ -1,4 +1,4 @@
-from dateutil.parser import parse
+from dateutil.parser import parse as date_parse
from os.path import abspath
import frontmatter
import json
@@ -13,10 +13,14 @@ from orm.base import local_session
users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read())
print(str(len(users_dict.items())) + ' users loaded')
-topics_dict = json.loads(open(abspath('migration/data/topics.dict.json')).read()) # old_id keyed
-print(str(len(topics_dict.items())) + ' topics loaded')
+
+cats_data = json.loads(open(abspath('migration/data/content_item_categories.json')).read()) # old_id keyed
+cats_dict = { x['_id']: x for x in cats_data }
+print(str(len(cats_data)) + ' categories loaded')
+
comments_data = json.loads(open(abspath('migration/data/comments.json')).read())
print(str(len(comments_data)) + ' comments loaded')
+
comments_by_post = {}
for comment in comments_data:
p = comment['contentItem']
@@ -102,15 +106,15 @@ def migrate(entry):
# print(entry)
raise Exception
try:
- r['topics'].append(topics_dict[entry['category']]['slug'])
+ r['topics'].append(cats_dict[entry['category']]['slug'])
except Exception:
- print(entry['category'])
+ print(entry['category'])
if entry.get('image') is not None:
r['cover'] = entry['image']['url']
if entry.get('thumborId') is not None:
r['cover'] = 'https://assets.discours.io/unsafe/1600x/' + entry['thumborId']
if entry.get('updatedAt') is not None:
- r['updatedAt'] = parse(entry['updatedAt'])
+ r['updatedAt'] = date_parse(entry['updatedAt'])
if entry.get('type') == 'Literature':
media = entry.get('media', '')
# print(media[0]['literatureBody'])
@@ -125,16 +129,16 @@ def migrate(entry):
else:
print(r['slug'] + ': literature has no media')
elif entry.get('type') == 'Video':
- m = entry['media'][0]
- yt = m.get('youtubeId', '')
- vm = m.get('vimeoId', '')
- video_url = 'https://www.youtube.com/watch?v=' + yt if yt else '#'
- if video_url == '#':
- video_url = 'https://vimeo.com/' + vm if vm else '#'
- if video_url == '#':
- print(entry.get('media', 'NO MEDIA!'))
- # raise Exception
- r['body'] = '' + html2text(m.get('body', '')) # FIXME
elif entry.get('type') == 'Music':
r['body'] = ' 0:
- # TODO: adding shout ratings
- '''
+ # TODO: migrate shout ratings
shout_dict['ratings'] = []
for shout_rating_old in entry['ratings']:
shout_rating = ShoutRating.create(
@@ -232,16 +234,20 @@ def migrate(entry):
shout_id = s.id,
value = shout_rating_old['value']
)
- shout.ratings.append(shout_rating.id)
- '''
- # adding topics to created shout
- for topic_slug in topic_slugs:
+ s.ratings.append(shout_rating.id)
+ s.save()
+ # TODO: migrate topics
+ '''
+ with local_session() as session:
+ for topic_slug in topic_slugs:
+ topic = session.query(Topic).filter(Topic.slug == topic_slug).first()
if not topic:
- topic_dict = topics_dict.get(topic_slug)
+ topic_dict = migrateCategory()
if topic_dict:
topic = Topic.create(**topic_dict)
- shout.topics = [ topic, ]
- shout.save()
+ s.topics = [ topic, ]
+ s.save()
+ '''
except Exception as e:
r['error'] = 'db error'
# pass