migration wip, minor fixes scheme

migration/tables/comments.py

@@ -1,10 +1,10 @@
-from html2md import Converter
+# from html2md import Converter
 import datetime
 
-markdown = Converter()
+# markdown = Converter()
 
 def migrate(entry):
-    ```
+    '''
     # is comment
     type Shout {
         org: String!
@@ -25,12 +25,12 @@ def migrate(entry):
         visibleForRoles: [String] # role ids are strings
         visibleForUsers: [Int]
     }
-    ```
+    '''
     # TODO: implement comments migration
     return {
         'org': 'discours.io',
         'slug': entry['slug'],
         'createdAt': entry['createdAt'],
-        'body': markdown.feed(entry['body']),
+        'body': html2text(entry['body']),
         'replyTo': entry['']
     }

migration/tables/content_items.py

@@ -1,14 +1,16 @@
-from migration.html2md import Converter
+# from migration.html2md import Converter
 from dateutil.parser import parse
 from os.path import abspath
 import frontmatter
 import json
 from orm import Shout
+from bs4 import BeautifulSoup
+from migration.html2text import html2text
 
 users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read())
-users_dict['0'] = {'id': 99999 }
+users_dict['0'] = {'id': 9999999, 'slug': 'discours', 'viewname': 'Дискурс' }
 
-markdown = Converter()
+# markdown = Converter()
 
 type2layout = {
     'Article': 'article',
@@ -18,7 +20,7 @@ type2layout = {
     'Image': 'image'
 }
 
-def migrate(entry):
+def migrate(entry, data=users_dict):
     '''  
     type Shout {
         org_id: Int!
@@ -43,27 +45,37 @@ def migrate(entry):
         views: Int
     }
     '''
+    try:
+        author = data[entry['createdBy']]
+    except KeyError:
+        author = data['0']
+
+    # print(author)
     r = {
             'org_id': 0,
             'layout': type2layout[entry['type']],
             'title': entry['title'],
-            'authors': [ users_dict[entry['createdBy']]['id'], ],
+            'authors': [ { 'slug': author['slug'], 'name': author['viewname'], 'pic': author.get('userpic', '') }, ],
             'topics': [],
             'published': entry['published'],
             'views': entry['views'],
             'rating': entry['rating'],
             'ratings': []
         }
-    r['slug'] = entry.get('slug')
+    r['slug'] = entry.get('slug', '')
+    body_orig = entry.get('body', '')
     if not r['slug'] and entry.get('friendlySlugs') is not None:
         r['slug'] = entry['friendlySlugs']['slug'][0]['slug']
         if(r['slug'] is None):
             r['slug'] = entry['friendlySlugs'][0]['slug']
+    if not r['slug']:
+        print('NO SLUG ERROR')
+        # print(entry)
+        raise Exception
     if entry.get('image') is not None:
         r['cover'] = entry['image']['url']
-    elif entry.get('thumborId') is not None:
-        r['cover'] = 'https://discours.io/' + entry['thumborId']
-
+    if entry.get('thumborId') is not None:
+        r['cover'] = 'https://assets.discours.io/unsafe/1600x/' + entry['thumborId']
     if entry.get('publishedAt') is not None:
         r['publishedAt'] = entry['publishedAt']
     if entry.get('createdAt') is not None:
@@ -71,20 +83,60 @@ def migrate(entry):
     if entry.get('updatedAt') is not None:
         r['updatedAt'] = entry['updatedAt']
     if entry.get('type') == 'Literature':
-        r['body'] = markdown.feed(entry['media'][0]['literatureBody'])
+        media = entry.get('media', '')
+        # print(media[0]['literatureBody'])
+        if type(media) == list:
+            body_orig = media[0].get('literatureBody', '')
+            if body_orig == '':
+                print('EMPTY BODY!')
+            else:
+                # body_html = str(BeautifulSoup(body_orig, features="html.parser"))
+                #markdown.feed(body_html)
+                body = html2text(body_orig).replace('****', '**')
+                r['body'] = body
+                # r['body2'] = markdown.md_file
+        else:
+            print(r['slug'] + ': literature has no media')
     elif entry.get('type') == 'Video':
-        r['body'] = '<ShoutVideo src=\"' + entry['media'][0]['youtubeId'] + '\" />'
+        m = entry['media'][0]
+        yt = m.get('youtubeId', '')
+        vm = m.get('vimeoId', '')
+        videoUrl = 'https://www.youtube.com/watch?v=' + yt if yt else '#'
+        if videoUrl == '#':
+            videoUrl = 'https://vimeo.com/' + vm if vm else '#'
+        if videoUrl == '#':
+            print(m)
+            # raise Exception
+        r['body'] = '<ShoutVideo src=\"' + videoUrl + '\" />' + html2text(m.get('body', '')) # FIXME
     elif entry.get('type') == 'Music':
-        r['body'] = '<ShoutMusic media={\"' + json.dumps(entry['media']) +'\"} />'
-    elif entry.get('body') is not None:
-        r['body'] = markdown.feed(entry['body'])
-    else:
-        r['body'] = '## ' + r['title']
-    body = r['body']
-    del r['body']
-    metadata = frontmatter.dumps(r)
-    open('migration/content/' + r['slug'] + '.md', 'w').write(metadata + '\n' + body)
-    r['body'] = body
-    shout = Shout.create(**r.copy())
-    r['id'] = shout['id']
+        r['body'] = '<ShoutMusic media={\"' + json.dumps(entry['media']) +'\"} />' # FIXME
+
+    if r.get('body') is None:
+        body_orig = entry.get('body', '')
+        # body_html = BeautifulSoup(body_orig, features="html.parser")
+        r['body'] = html2text(body_orig).replace('****', '**')
+        # markdown.feed(body_html)
+        # r['body2'] = markdown.md_file
+    if not r['body']:
+        r['body'] = entry.get('body')
+    metadata = {}
+    metadata['title'] = r.get('title')
+    metadata['authors'] = r.get('authors')
+    if r.get('cover', False):
+        metadata['cover'] = r.get('cover')
+    body = r.get('body')
+    post = frontmatter.Post(body, **metadata)
+    dumped = frontmatter.dumps(post)
+    # raise Exception
+
+    open('migration/content/' + entry['type'].lower() + '/' + r['slug'] + '.md', 'w').write(dumped)
+    # open('migration/content/' + entry['type'].lower() + '/' + r['slug'] + '.my.md', 'w').write(r['body2'])
+    #if body_orig:
+    #    open('migration/content/' + entry['type'].lower() + '/' + r['slug'] + '.html', 'w').write(body_orig)
+    #markdown.related_data = []
+    #markdown.md_file = ''
+    #markdown.reset()
+    r['body'] = dumped
+    # shout = Shout.create(**r.copy())
+    # r['id'] = shout['id']
     return r

migration/tables/users.py

@@ -1,10 +1,17 @@
 from orm import User, Role
 import frontmatter
 from dateutil.parser import parse
-from migration.html2md import Converter
-markdown = Converter()
+from migration.html2text import html2text
+# from migration.html2md import Converter
+# markdown = Converter()
 counter = 0
 
+
+def add(data):
+    data.emailConfirmed = False
+    user = User.create(**data)
+    return user
+
 def migrate(entry):
         '''
         
@@ -45,10 +52,10 @@ def migrate(entry):
         res['notifications'] = []
         res['links'] = []
         res['muted'] = False
-        res['bio'] = markdown.feed(entry.get('bio', ''))
+        res['bio'] = html2text(entry.get('bio', ''))
         if entry['profile']:
             res['slug'] = entry['profile'].get('path')
-            res['userpic'] = entry['profile'].get('image', {'url': ''}).get('url', '')
+            res['userpic'] = entry['profile'].get('image', {'thumborId': ''}).get('thumborId', '') # adding 'https://assets.discours.io/unsafe/1600x' in web ui
             fn = entry['profile'].get('firstName', '')
             ln = entry['profile'].get('lastName', '')
             viewname = res['slug'] if res['slug'] else 'anonymous'