migrating comments and adding authors

migrate.py

@@ -146,14 +146,24 @@ def shouts():
 def export_shouts(limit):
     print('reading json...')
     newdata = json.loads(open('migration/data/shouts.dict.json', 'r').read())
-    print(str(len(newdata.keys())) + ' loaded')
+    print(str(len(newdata.keys())) + ' shouts loaded')
+
+    users_old = json.loads(open('migration/data/users.dict.json').read())
+    export_authors = json.loads(open('../src/data/authors.json').read())
+    print(str(len(export_authors.items())) + ' pre-exported authors loaded')
+    users_slug = { u['slug']: u for old_id, u in users_old.items()}
+    print(str(len(users_slug.items())) + ' users loaded')
+
     export_list = [i for i in newdata.items() if i[1]['layout'] == 'article' and i[1]['published']]
     export_list = sorted(export_list, key=lambda item: item[1]['createdAt'] or OLD_DATE, reverse=True)
     print(str(len(export_list)) + ' filtered')
+
     export_list = export_list[:limit or len(export_list)]
     export_clean = {}
     for (slug, article) in export_list:
         if article['layout'] == 'article':
+            for author in article['authors']:
+              export_authors[author['slug']] = users_slug[author['slug']]
             export_clean[article['slug']] = extract_images(article)
             metadata = get_metadata(article)
             content = frontmatter.dumps(frontmatter.Post(article['body'], **metadata))
@@ -165,7 +175,13 @@ def export_shouts(limit):
                                                             indent=4,
                                                             sort_keys=True,
                                                             ensure_ascii=False))
-    print(str(len(export_clean.items())) + ' exported')
+    print(str(len(export_clean.items())) + ' articles exported')
+    open('../src/data/authors.json', 'w').write(json.dumps(export_authors,
+                                                           cls=DateTimeEncoder,
+                                                           indent=4,
+                                                           sort_keys=True,
+                                                           ensure_ascii=False))
+    print(str(len(export_authors.items())) + ' total authors exported')
 
 
 if __name__ == '__main__':

migration/tables/comments.py

@@ -1,10 +1,33 @@
-from html2text import html2text
 import datetime
+import json
+from os.path import abspath
+from orm import Shout
+from orm.base import local_session
+from migration.html2text import html2text
 
-# markdown = Converter()
+users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read())
+topics_dict = json.loads(open(abspath('migration/data/topics.dict.json')).read()) # old_id keyed
 
 def migrate(entry):
     '''
+    {
+      "_id": "hdtwS8fSyFLxXCgSC",
+      "body": "<p>",
+      "contentItem": "mnK8KsJHPRi8DrybQ",
+      "createdBy": "bMFPuyNg6qAD2mhXe",
+      "thread": "01/",
+      "createdAt": "2016-04-19 04:33:53+00:00",
+      "ratings": [
+        { "createdBy": "AqmRukvRiExNpAe8C", "value": 1 },
+        { "createdBy": "YdE76Wth3yqymKEu5", "value": 1 }
+      ],
+      "rating": 2,
+      "updatedAt": "2020-05-27 19:22:57.091000+00:00",
+      "updatedBy": "0"
+    }
+
+    ->
+
     type Comment {
         id: Int!
         author: Int!
@@ -21,10 +44,16 @@ def migrate(entry):
         old_id: String
     }
     '''
-    # TODO: implement comments migration
+    with local_session() as session:
+        shout_id = session.query(Shout).filter(Shout.old_id == entry['_id']).first()
         return {
-        'slug': entry['slug'],
+            'old_id': entry['_id'],
+            'old_thread': entry['thread'],
+            'createdBy': users_dict[entry['createdBy']],
             'createdAt': entry['createdAt'],
             'body': html2text(entry['body']),
-        'replyTo': entry['']
+            'shout': shout_id,
+            'rating': entry['rating'],
+            'ratings': [] # TODO: ratings in comments
         }
+    return None

migration/tables/content_items.py

@@ -11,6 +11,8 @@ from datetime import datetime
 from sqlalchemy.exc import IntegrityError
 from orm.base import local_session
 
+comments_data = json.loads(open(abspath('migration/data/comments.json')).read())
+comments_dict = { x['_id']: x for x in comments_data }
 users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read())
 topics_dict = json.loads(open(abspath('migration/data/topics.dict.json')).read()) # old_id keyed
 users_dict['0'] = {
@@ -77,6 +79,7 @@ def migrate(entry):
         'views': entry.get('views', 0),
         'rating': entry.get('rating', 0),
         'ratings': [],
+        'comments': entry.get('comments', []),
         'createdAt': entry.get('createdAt', '2016-03-05 22:22:00.350000')
     }
     r['slug'] = entry.get('slug', '')
@@ -184,10 +187,6 @@ def migrate(entry):
     content = frontmatter.dumps(frontmatter.Post(body, **metadata))
 
     if entry['published']:
-        # if r.get('old_id', None):
-        #    ext = 'html'
-        #    content = str(body).replace('<p></p>', '').replace('<p> </p>', '')
-        # else:
         ext = 'md'
         open('migration/content/' +
              r['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content)
@@ -204,17 +203,44 @@ def migrate(entry):
                 else:
                     shout_dict['publishedAt'] = ts
             del shout_dict['published']
-        # del shout_dict['views']
+
-        # del shout_dict['rating']
+            shout_dict['comments'] = []
-        del shout_dict['ratings'] # FIXME
+            for cid in r['comments']:
+              comment = comments_dict[cid]
+              comment_ratings = []
+              for cr in comment['ratings']:
+                comment_ratings.append({
+                  'value': cr['value'],
+                  'createdBy': users_dict[cr['createdBy']],
+                  'createdAt': cr['createdAt'] or ts})
+              shout_dict['comments'].append({
+                'old_id': comment['_id'],
+                'old_thread': comment['thread'], # TODO: old_thread to replyTo logix
+                'createdBy': users_dict[comment['createdBy']],
+                'createdAt': comment['createdAt'] or ts,
+                'body': html2text(comment['body']),
+                'shout': shout_dict['old_id'],
+                'rating': comment['rating'],
+                'ratings': comment_ratings
+            })
+
+            shout_dict['ratings'] = []
+            for rating in r['ratings']:
+              shout_dict['ratings'].append({
+                'value': rating['value'],
+                'createdBy': users_dict[rating['createdBy']],
+                'createdAt': r['createdAt'] or ts})
+
             try:
-            s = Shout.create(**shout_dict)
+                del shout_dict['views'] # FIXME
+                del shout_dict['rating'] # FIXME
+                del shout_dict['ratings'] # FIXME
+                # del shout_dict['comments']
+                s = Shout.create(**shout_dict) # FIXME: AttributeError: 'str' object has no attribute '_sa_instance_state'
                 r['id'] = s.id
-        except:
+            except Exception as e:
-            pass
+              pass # raise e
-    except:
+        except Exception as e:
-        r['body'] = 'body moved'
+            if not r['body']: r['body'] = 'body moved'
-        print(r)
+            raise e
-        # print(s)
-        raise Exception
     return r