migrate shouts: fix publishedAt and topics

migrate.py

@@ -9,7 +9,7 @@ from migration.tables.content_item_categories import migrate as migrateCategory
 from migration.tables.tags import migrate as migrateTag
 from migration.tables.comments import migrate as migrateComment
 from migration.utils import DateTimeEncoder
-from orm import Community
+from orm import Community, Topic
 from dateutil.parser import parse as date_parse
 
 from orm.base import local_session
@@ -166,6 +166,13 @@ if __name__ == '__main__':
         try: limit = int(sys.argv[2]) if len(sys.argv) > 2 else len(content_data)
         except ValueError:  limit = len(content_data)
         
+        if not topics_by_cat:
+            with local_session() as session:
+                topics = session.query(Topic).all()
+            print("loaded %s topics" % len(topics))
+            for topic in topics:
+                topics_by_cat[topic.cat_id] = topic
+
         for entry in content_data[:limit]:
             try:
                 shout = migrateShout(entry, users_by_oid, topics_by_cat)
@@ -306,14 +313,17 @@ if __name__ == '__main__':
         elif cmd == "topics":
             topics(export_topics, topics_by_slug, topics_by_cat, topics_by_tag, cats_data, tags_data)
         elif cmd == "shouts":
-            Community.create(**{
+            with local_session() as session:
-                'id' : 0,
+                community = session.query(Community).filter(Community.id == 0).first()
-                'slug': 'discours.io',
+            if not community:
-                'name': 'Дискурс',
+                Community.create(**{
-                'pic': 'https://discours.io/images/logo-min.svg',
+                    'id' : 0,
-                'createdBy': '0',
+                    'slug': 'discours.io',
-                'createdAt': date_parse(OLD_DATE)
+                    'name': 'Дискурс',
-            })
+                    'pic': 'https://discours.io/images/logo-min.svg',
+                    'createdBy': '0',
+                    'createdAt': date_parse(OLD_DATE)
+                })
             shouts(content_data, shouts_by_slug, shouts_by_oid) # NOTE: listens limit
         elif cmd == "export_shouts":
             export_shouts(shouts_by_slug, export_articles, export_authors, content_dict)

migration/tables/content_items.py

@@ -3,7 +3,7 @@ import frontmatter
 import json
 import sqlite3
 import sqlalchemy
-from orm import Shout, Comment, Topic, ShoutRating, User #, TODO: CommentRating
+from orm import Shout, Comment, Topic, ShoutTopic, ShoutRating, User
 from bs4 import BeautifulSoup
 from migration.html2text import html2text
 from migration.tables.comments import migrate as migrateComment
@@ -37,7 +37,7 @@ def get_metadata(r):
     metadata['authors'] = r.get('authors')
     metadata['createdAt'] = r.get('createdAt', ts)
     metadata['layout'] = r['layout']
-    metadata['topics'] = r['topics']
+    metadata['topics'] = [topic.slug for topic in r['topics']]
     if r.get('cover', False):
         metadata['cover'] = r.get('cover')
     return metadata
@@ -66,6 +66,7 @@ def migrate(entry, users_by_oid, topics_by_oid):
         views: Int
     }
     '''
+    # print(entry)
     content = ''
     r = {
         'layout': type2layout[entry['type']],
@@ -73,7 +74,6 @@ def migrate(entry, users_by_oid, topics_by_oid):
         'community': 0,
         'authors': [],
         'topics': [],
-        'published': entry.get('published', False),
         'views': entry.get('views', 0),
         'rating': entry.get('rating', 0),
         'ratings': [],
@@ -90,9 +90,9 @@ def migrate(entry, users_by_oid, topics_by_oid):
         # print(entry)
         raise Exception
     try:
-        r['topics'].append(topics_by_oid[entry['category']]['slug'])
+        r['topics'].append(topics_by_oid[entry['category']])
     except Exception:
-        print(entry['category'])
+        print("invalid category %s" % (entry['category']))
     if entry.get('image') is not None:
         r['cover'] = entry['image']['url']
     if entry.get('thumborId') is not None:
@@ -174,13 +174,12 @@ def migrate(entry, users_by_oid, topics_by_oid):
         open('migration/content/' + r['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content)
     try:
         shout_dict['createdAt'] = date_parse(r.get('createdAt')) if entry.get('createdAt') else ts
-        shout_dict['publishedAt'] = date_parse(entry.get('publishedAt')) if entry.get('published') else ts
+        shout_dict['publishedAt'] = date_parse(entry.get('publishedAt')) if entry.get('published') else None
 
         if entry.get('deletedAt') is not None:
             shout_dict['deletedAt'] = date_parse(entry.get('deletedAt'))
             shout_dict['deletedBy'] = entry.get('deletedBy', '0')
         
-        del shout_dict['published'] # invalid keyword argument for Shout
         del shout_dict['topics'] # FIXME: AttributeError: 'str' object has no attribute '_sa_instance_state'
         del shout_dict['views'] # FIXME: TypeError: 'views' is an invalid keyword argument for Shout
         del shout_dict['rating'] # FIXME: TypeError: 'rating' is an invalid keyword argument for Shout
@@ -223,11 +222,8 @@ def migrate(entry, users_by_oid, topics_by_oid):
                     shout_dict['ratings'].append(shout_rating_dict)
             # shout topics
             shout_dict['topics'] = []
-            for topic_slug in r['topics']:
+            for topic in r['topics']:
-                topic = session.query(Topic).filter(Topic.slug == topic_slug).first()
+                ShoutTopic.create(**{ 'shout': s.id, 'topic': topic.id })
-                if not topic:
-                    try: topic = Topic.create(**{ 'slug': topic_slug, 'title': topic_slug })
-                    except Exception as e: raise e
                 shout_dict['topics'].append(topic.slug)
     except Exception as e:
         if not shout_dict['body']: r['body'] = 'body moved'