from dateutil.parser import parse
from orm import User
from os.path import abspath
import frontmatter
import json
from orm import Shout
from bs4 import BeautifulSoup
from migration.html2text import html2text
from transliterate import translit
from datetime import datetime
from sqlalchemy.exc import IntegrityError
from orm.base import local_session
users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read())
topics_dict = json.loads(open(abspath('migration/data/topics.dict.json')).read()) # old_id keyed
users_dict['0'] = {
'id': 9999999,
'slug': 'discours.io',
'name': 'Дискурс',
'userpic': 'https://discours.io/images/logo-mini.svg',
'createdAt': '2016-03-05 22:22:00.350000'
}
ts = datetime.now()
type2layout = {
'Article': 'article',
'Literature': 'prose',
'Music': 'music',
'Video': 'video',
'Image': 'image'
}
def get_metadata(r):
metadata = {}
metadata['title'] = r.get('title')
metadata['authors'] = r.get('authors')
metadata['createdAt'] = r.get('createdAt', ts)
metadata['layout'] = r['layout']
if r.get('cover', False):
metadata['cover'] = r.get('cover')
return metadata
def migrate(entry):
'''
type Shout {
slug: String!
author: Int!
body: String!
createdAt: DateTime!
updatedAt: DateTime!
deletedAt: DateTime
deletedBy: Int
rating: Int
ratigns: [Rating]
published: Bool!
publishedAt: DateTime # if there is no published field - it is not published
replyTo: String # another shout
tags: [String] # actual values
topics: [String] # topic-slugs, order has matter
title: String
versionOf: String
visibleForRoles: [String] # role ids are strings
visibleForUsers: [Int]
views: Int
}
'''
content = ''
r = {
'layout': type2layout[entry['type']],
'title': entry['title'],
'community': 0,
'authors': [],
'topics': [],
'published': entry.get('published', False),
'views': entry.get('views', 0),
'rating': entry.get('rating', 0),
'ratings': [],
'createdAt': entry.get('createdAt', '2016-03-05 22:22:00.350000')
}
r['slug'] = entry.get('slug', '')
body_orig = entry.get('body', '')
if not r['slug'] and entry.get('friendlySlugs') is not None:
r['slug'] = entry['friendlySlugs']['slug'][0]['slug']
if(r['slug'] is None):
r['slug'] = entry['friendlySlugs'][0]['slug']
if not r['slug']:
print('NO SLUG ERROR')
# print(entry)
raise Exception
try:
r['topics'].append(topics_dict[entry['category']]['slug'])
except Exception:
print(entry['category'])
if entry.get('image') is not None:
r['cover'] = entry['image']['url']
if entry.get('thumborId') is not None:
r['cover'] = 'https://assets.discours.io/unsafe/1600x/' + entry['thumborId']
if entry.get('updatedAt') is not None:
r['updatedAt'] = parse(entry['updatedAt'])
if entry.get('type') == 'Literature':
media = entry.get('media', '')
# print(media[0]['literatureBody'])
if type(media) == list:
body_orig = media[0].get('literatureBody', '')
if body_orig == '':
print('EMPTY BODY!')
else:
body_html = str(BeautifulSoup(
body_orig, features="html.parser"))
r['body'] = html2text(body_html).replace('****', '**')
r['old_id'] = entry.get('_id')
else:
print(r['slug'] + ': literature has no media')
elif entry.get('type') == 'Video':
m = entry['media'][0]
yt = m.get('youtubeId', '')
vm = m.get('vimeoId', '')
video_url = 'https://www.youtube.com/watch?v=' + yt if yt else '#'
if video_url == '#':
video_url = 'https://vimeo.com/' + vm if vm else '#'
if video_url == '#':
print(entry.get('media', 'NO MEDIA!'))
# raise Exception
r['body'] = '
', '') # else: ext = 'md' open('migration/content/' + r['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content) try: shout_dict = r.copy() shout_dict['authors'] = [user, ] if entry.get('createdAt') is not None: shout_dict['createdAt'] = parse(r.get('createdAt')) else: shout_dict['createdAt'] = ts if entry.get('published'): if entry.get('publishedAt') is not None: shout_dict['publishedAt'] = parse(entry.get('publishedAt')) else: shout_dict['publishedAt'] = ts del shout_dict['published'] # del shout_dict['views'] # del shout_dict['rating'] del shout_dict['ratings'] # FIXME try: s = Shout.create(**shout_dict) r['id'] = s.id except: pass except: r['body'] = 'body moved' print(r) # print(s) raise Exception return r