diff --git a/migrate.py b/migrate.py index 006f5532..7a4b1807 100644 --- a/migrate.py +++ b/migrate.py @@ -146,14 +146,24 @@ def shouts(): def export_shouts(limit): print('reading json...') newdata = json.loads(open('migration/data/shouts.dict.json', 'r').read()) - print(str(len(newdata.keys())) + ' loaded') + print(str(len(newdata.keys())) + ' shouts loaded') + + users_old = json.loads(open('migration/data/users.dict.json').read()) + export_authors = json.loads(open('../src/data/authors.json').read()) + print(str(len(export_authors.items())) + ' pre-exported authors loaded') + users_slug = { u['slug']: u for old_id, u in users_old.items()} + print(str(len(users_slug.items())) + ' users loaded') + export_list = [i for i in newdata.items() if i[1]['layout'] == 'article' and i[1]['published']] export_list = sorted(export_list, key=lambda item: item[1]['createdAt'] or OLD_DATE, reverse=True) print(str(len(export_list)) + ' filtered') + export_list = export_list[:limit or len(export_list)] export_clean = {} for (slug, article) in export_list: if article['layout'] == 'article': + for author in article['authors']: + export_authors[author['slug']] = users_slug[author['slug']] export_clean[article['slug']] = extract_images(article) metadata = get_metadata(article) content = frontmatter.dumps(frontmatter.Post(article['body'], **metadata)) @@ -165,7 +175,13 @@ def export_shouts(limit): indent=4, sort_keys=True, ensure_ascii=False)) - print(str(len(export_clean.items())) + ' exported') + print(str(len(export_clean.items())) + ' articles exported') + open('../src/data/authors.json', 'w').write(json.dumps(export_authors, + cls=DateTimeEncoder, + indent=4, + sort_keys=True, + ensure_ascii=False)) + print(str(len(export_authors.items())) + ' total authors exported') if __name__ == '__main__': diff --git a/migration/tables/comments.py b/migration/tables/comments.py index c8f504a6..271ddc4a 100644 --- a/migration/tables/comments.py +++ b/migration/tables/comments.py @@ -1,10 +1,33 @@ -from html2text import html2text import datetime +import json +from os.path import abspath +from orm import Shout +from orm.base import local_session +from migration.html2text import html2text -# markdown = Converter() +users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read()) +topics_dict = json.loads(open(abspath('migration/data/topics.dict.json')).read()) # old_id keyed def migrate(entry): ''' + { + "_id": "hdtwS8fSyFLxXCgSC", + "body": "

", + "contentItem": "mnK8KsJHPRi8DrybQ", + "createdBy": "bMFPuyNg6qAD2mhXe", + "thread": "01/", + "createdAt": "2016-04-19 04:33:53+00:00", + "ratings": [ + { "createdBy": "AqmRukvRiExNpAe8C", "value": 1 }, + { "createdBy": "YdE76Wth3yqymKEu5", "value": 1 } + ], + "rating": 2, + "updatedAt": "2020-05-27 19:22:57.091000+00:00", + "updatedBy": "0" + } + + -> + type Comment { id: Int! author: Int! @@ -21,10 +44,16 @@ def migrate(entry): old_id: String } ''' - # TODO: implement comments migration - return { - 'slug': entry['slug'], - 'createdAt': entry['createdAt'], - 'body': html2text(entry['body']), - 'replyTo': entry[''] - } \ No newline at end of file + with local_session() as session: + shout_id = session.query(Shout).filter(Shout.old_id == entry['_id']).first() + return { + 'old_id': entry['_id'], + 'old_thread': entry['thread'], + 'createdBy': users_dict[entry['createdBy']], + 'createdAt': entry['createdAt'], + 'body': html2text(entry['body']), + 'shout': shout_id, + 'rating': entry['rating'], + 'ratings': [] # TODO: ratings in comments + } + return None diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index 8a165ba1..3418e932 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -11,6 +11,8 @@ from datetime import datetime from sqlalchemy.exc import IntegrityError from orm.base import local_session +comments_data = json.loads(open(abspath('migration/data/comments.json')).read()) +comments_dict = { x['_id']: x for x in comments_data } users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read()) topics_dict = json.loads(open(abspath('migration/data/topics.dict.json')).read()) # old_id keyed users_dict['0'] = { @@ -77,6 +79,7 @@ def migrate(entry): 'views': entry.get('views', 0), 'rating': entry.get('rating', 0), 'ratings': [], + 'comments': entry.get('comments', []), 'createdAt': entry.get('createdAt', '2016-03-05 22:22:00.350000') } r['slug'] = entry.get('slug', '') @@ -184,37 +187,60 @@ def migrate(entry): content = frontmatter.dumps(frontmatter.Post(body, **metadata)) if entry['published']: - # if r.get('old_id', None): - # ext = 'html' - # content = str(body).replace('

', '').replace('

', '') - # else: ext = 'md' open('migration/content/' + r['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content) - try: - shout_dict = r.copy() - shout_dict['authors'] = [user, ] - if entry.get('createdAt') is not None: - shout_dict['createdAt'] = parse(r.get('createdAt')) - else: - shout_dict['createdAt'] = ts - if entry.get('published'): - if entry.get('publishedAt') is not None: - shout_dict['publishedAt'] = parse(entry.get('publishedAt')) - else: - shout_dict['publishedAt'] = ts - del shout_dict['published'] - # del shout_dict['views'] - # del shout_dict['rating'] - del shout_dict['ratings'] # FIXME try: - s = Shout.create(**shout_dict) - r['id'] = s.id - except: - pass - except: - r['body'] = 'body moved' - print(r) - # print(s) - raise Exception + shout_dict = r.copy() + shout_dict['authors'] = [user, ] + if entry.get('createdAt') is not None: + shout_dict['createdAt'] = parse(r.get('createdAt')) + else: + shout_dict['createdAt'] = ts + if entry.get('published'): + if entry.get('publishedAt') is not None: + shout_dict['publishedAt'] = parse(entry.get('publishedAt')) + else: + shout_dict['publishedAt'] = ts + del shout_dict['published'] + + shout_dict['comments'] = [] + for cid in r['comments']: + comment = comments_dict[cid] + comment_ratings = [] + for cr in comment['ratings']: + comment_ratings.append({ + 'value': cr['value'], + 'createdBy': users_dict[cr['createdBy']], + 'createdAt': cr['createdAt'] or ts}) + shout_dict['comments'].append({ + 'old_id': comment['_id'], + 'old_thread': comment['thread'], # TODO: old_thread to replyTo logix + 'createdBy': users_dict[comment['createdBy']], + 'createdAt': comment['createdAt'] or ts, + 'body': html2text(comment['body']), + 'shout': shout_dict['old_id'], + 'rating': comment['rating'], + 'ratings': comment_ratings + }) + + shout_dict['ratings'] = [] + for rating in r['ratings']: + shout_dict['ratings'].append({ + 'value': rating['value'], + 'createdBy': users_dict[rating['createdBy']], + 'createdAt': r['createdAt'] or ts}) + + try: + del shout_dict['views'] # FIXME + del shout_dict['rating'] # FIXME + del shout_dict['ratings'] # FIXME + # del shout_dict['comments'] + s = Shout.create(**shout_dict) # FIXME: AttributeError: 'str' object has no attribute '_sa_instance_state' + r['id'] = s.id + except Exception as e: + pass # raise e + except Exception as e: + if not r['body']: r['body'] = 'body moved' + raise e return r