discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

upgrade

Browse Source
This commit is contained in:
Untone 2021-10-08 12:58:19 +03:00
parent c3e0c5720a
commit fe28c3918c
3 changed files with 66 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
migrate.py
View File

@ -2,8 +2,9 @@
import json import json
import base64 import base64
import re import re
import frontmatter
from migration.tables.users import migrate as migrateUser from migration.tables.users import migrate as migrateUser
from migration.tables.content_items import migrate as migrateShout from migration.tables.content_items import get_metadata, migrate as migrateShout
from migration.tables.content_item_categories import migrate as migrateCategory from migration.tables.content_item_categories import migrate as migrateCategory
from migration.tables.tags import migrate as migrateTag from migration.tables.tags import migrate as migrateTag
from migration.utils import DateTimeEncoder from migration.utils import DateTimeEncoder
@ -70,16 +71,19 @@ def topics():
print('migrating topics...') print('migrating topics...')
cat_data = json.loads( cat_data = json.loads(
open('migration/data/content_item_categories.json').read()) open('migration/data/content_item_categories.json').read())
tag_data = json.loads(open('migration/data/tags.json').read()) # tag_data = json.loads(open('migration/data/tags.json').read())
newdata = {} new_data = {}
old_data = {}
counter = 0 counter = 0
try: try:
for cat in cat_data: for cat in cat_data:
topic = migrateCategory(cat) topic = migrateCategory(cat)
newdata[topic['slug']] = topic old_data[topic['old_id']] = topic
new_data[topic['slug']] = topic
counter += 1 counter += 1
except Exception: except Exception:
print('cats exception, try to remove database first') print('cats exception, try to remove database first')
'''
try: try:
for tag in tag_data: for tag in tag_data:
topic = migrateTag(tag) topic = migrateTag(tag)
@ -88,14 +92,19 @@ def topics():
except Exception: except Exception:
print('tags exception, try to remove database first') print('tags exception, try to remove database first')
raise Exception raise Exception
export_list = sorted(newdata.items(), key=lambda item: str( '''
item[1]['createdAt']))[-10:] export_list = sorted(new_data.items(), key=lambda item: str(
item[1]['createdAt']))
open('migration/data/topics.dict.json', open('migration/data/topics.dict.json',
'w').write(json.dumps(newdata, cls=DateTimeEncoder)) 'w').write(json.dumps(old_data, cls=DateTimeEncoder))
open('../src/data/topics.json', 'w').write(json.dumps(dict(export_list), open('../src/data/topics.json', 'w').write(json.dumps(dict(export_list),
cls=DateTimeEncoder, indent=4, sort_keys=True, ensure_ascii=False)) cls=DateTimeEncoder,
indent=4,
sort_keys=True,
ensure_ascii=False))
print(str(counter) + ' from ' + str(len(cat_data)) + print(str(counter) + ' from ' + str(len(cat_data)) +
' tags and ' + str(len(tag_data)) + ' cats were migrated') #' tags and ' + str(len(tag_data)) +
' cats were migrated')
print(str(len(export_list)) + ' topics were exported') print(str(len(export_list)) + ' topics were exported')
@ -110,7 +119,7 @@ def shouts():
errored = [] errored = []
for entry in content_data: for entry in content_data:
try: try:
(shout, content) = migrateShout(entry) shout = migrateShout(entry)
newdata[shout['slug']] = shout newdata[shout['slug']] = shout
author = newdata[shout['slug']]['authors'][0]['slug'] author = newdata[shout['slug']]['authors'][0]['slug']
line = str(counter+1) + ': ' + shout['slug'] + " @" + str(author) line = str(counter+1) + ': ' + shout['slug'] + " @" + str(author)
@ -127,12 +136,14 @@ def shouts():
limit = int(sys.argv[2]) if len(sys.argv) > 2 else len(content_data) limit = int(sys.argv[2]) if len(sys.argv) > 2 else len(content_data)
except ValueError: except ValueError:
limit = len(content_data) limit = len(content_data)
export_list = sorted(newdata.items( export_list = [i for i in newdata.items() if i[1]['layout'] == 'article' and i[1]['published']]
), key=lambda item: item[1]['createdAt'] if item[1]['layout'] == 'article' else OLD_DATE)[:limit] export_list = sorted(export_list, key=lambda item: item[1]['createdAt'] or OLD_DATE, reverse=True)[:limit]
export_clean = {} export_clean = {}
for slug, a in dict(export_list).items(): for (slug, a) in export_list:
export_clean[slug] = extract_images(a) export_clean[a['slug']] = extract_images(a)
open('../content/discours.io/'+slug+'.md', 'w').write(content) metadata = get_metadata(a)
content = frontmatter.dumps(frontmatter.Post(a['body'], **metadata))
open('../content/discours.io/'+a['slug']+'.md', 'w').write(content)
open('migration/data/shouts.dict.json', open('migration/data/shouts.dict.json',
'w').write(json.dumps(newdata, cls=DateTimeEncoder)) 'w').write(json.dumps(newdata, cls=DateTimeEncoder))
open('../src/data/articles.json', 'w').write(json.dumps(dict(export_clean), open('../src/data/articles.json', 'w').write(json.dumps(dict(export_clean),

8
migration/bson2json.py
View File

@ -3,10 +3,10 @@ import datetime
import json import json
import importlib import importlib
import DateTimeEncoder from utils from migration.utils import DateTimeEncoder
def json_tables(): def json_tables():
print('creating json files at data/') print('creating json files at migration/data/')
data = { data = {
"content_items": [], "content_items": [],
"content_item_categories": [], "content_item_categories": [],
@ -17,13 +17,13 @@ def json_tables():
} }
for table in data.keys(): for table in data.keys():
lc = [] lc = []
with open('data/'+table+'.bson', 'rb') as f: with open('migration/data/'+table+'.bson', 'rb') as f:
bs = f.read() bs = f.read()
base = 0 base = 0
while base < len(bs): while base < len(bs):
base, d = bson.decode_document(bs, base) base, d = bson.decode_document(bs, base)
lc.append(d) lc.append(d)
data[table] = lc data[table] = lc
open('data/'+table+'.json', 'w').write(json.dumps(lc,cls=DateTimeEncoder)) open('migration/data/'+table+'.json', 'w').write(json.dumps(lc,cls=DateTimeEncoder))
return data return data

54
migration/tables/content_items.py
View File

@ -12,6 +12,7 @@ from sqlalchemy.exc import IntegrityError
from orm.base import local_session from orm.base import local_session
users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read()) users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read())
topics_dict = json.loads(open(abspath('migration/data/topics.dict.json')).read()) # old_id keyed
users_dict['0'] = { users_dict['0'] = {
'id': 9999999, 'id': 9999999,
'slug': 'discours.io', 'slug': 'discours.io',
@ -31,6 +32,16 @@ type2layout = {
} }
def get_metadata(r):
metadata = {}
metadata['title'] = r.get('title')
metadata['authors'] = r.get('authors')
metadata['createdAt'] = r.get('createdAt', ts)
metadata['layout'] = r['layout']
if r.get('cover', False):
metadata['cover'] = r.get('cover')
return metadata
def migrate(entry): def migrate(entry):
''' '''
type Shout { type Shout {
@ -66,7 +77,7 @@ def migrate(entry):
'views': entry.get('views', 0), 'views': entry.get('views', 0),
'rating': entry.get('rating', 0), 'rating': entry.get('rating', 0),
'ratings': [], 'ratings': [],
'createdAt': '2016-03-05 22:22:00.350000' 'createdAt': entry.get('createdAt', '2016-03-05 22:22:00.350000')
} }
r['slug'] = entry.get('slug', '') r['slug'] = entry.get('slug', '')
body_orig = entry.get('body', '') body_orig = entry.get('body', '')
@ -78,6 +89,10 @@ def migrate(entry):
print('NO SLUG ERROR') print('NO SLUG ERROR')
# print(entry) # print(entry)
raise Exception raise Exception
try:
r['topics'].append(topics_dict[entry['category']]['slug'])
except Exception:
print(entry['category'])
if entry.get('image') is not None: if entry.get('image') is not None:
r['cover'] = entry['image']['url'] r['cover'] = entry['image']['url']
if entry.get('thumborId') is not None: if entry.get('thumborId') is not None:
@ -102,13 +117,13 @@ def migrate(entry):
m = entry['media'][0] m = entry['media'][0]
yt = m.get('youtubeId', '') yt = m.get('youtubeId', '')
vm = m.get('vimeoId', '') vm = m.get('vimeoId', '')
videoUrl = 'https://www.youtube.com/watch?v=' + yt if yt else '#' video_url = 'https://www.youtube.com/watch?v=' + yt if yt else '#'
if videoUrl == '#': if video_url == '#':
videoUrl = 'https://vimeo.com/' + vm if vm else '#' video_url = 'https://vimeo.com/' + vm if vm else '#'
if videoUrl == '#': if video_url == '#':
print(entry.get('media', 'NO MEDIA!')) print(entry.get('media', 'NO MEDIA!'))
# raise Exception # raise Exception
r['body'] = '<ShoutVideo src=\"' + videoUrl + \ r['body'] = '<ShoutVideo src=\"' + video_url + \
'\" />' + html2text(m.get('body', '')) # FIXME '\" />' + html2text(m.get('body', '')) # FIXME
elif entry.get('type') == 'Music': elif entry.get('type') == 'Music':
r['body'] = '<ShoutMusic media={\"' + \ r['body'] = '<ShoutMusic media={\"' + \
@ -163,15 +178,10 @@ def migrate(entry):
'pic': userpic 'pic': userpic
}) })
metadata = {} r['layout'] = type2layout[entry['type']]
metadata['title'] = r.get('title')
metadata['authors'] = r.get('authors') metadata = get_metadata(r)
metadata['createdAt'] = entry.get('createdAt', ts) content = frontmatter.dumps(frontmatter.Post(body, **metadata))
metadata['layout'] = type2layout[entry['type']]
if r.get('cover', False):
metadata['cover'] = r.get('cover')
post = frontmatter.Post(body, **metadata)
dumped = frontmatter.dumps(post)
if entry['published']: if entry['published']:
# if r.get('old_id', None): # if r.get('old_id', None):
@ -179,15 +189,13 @@ def migrate(entry):
# content = str(body).replace('<p></p>', '').replace('<p> </p>', '') # content = str(body).replace('<p></p>', '').replace('<p> </p>', '')
# else: # else:
ext = 'md' ext = 'md'
content = dumped
open('migration/content/' + open('migration/content/' +
metadata['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content) r['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content)
try: try:
shout_dict = r.copy() shout_dict = r.copy()
shout_dict['authors'] = [user, ] shout_dict['authors'] = [user, ]
if entry.get('createdAt') is not None: if entry.get('createdAt') is not None:
shout_dict['createdAt'] = parse(entry.get('createdAt')) shout_dict['createdAt'] = parse(r.get('createdAt'))
else: else:
shout_dict['createdAt'] = ts shout_dict['createdAt'] = ts
if entry.get('published'): if entry.get('published'):
@ -196,9 +204,9 @@ def migrate(entry):
else: else:
shout_dict['publishedAt'] = ts shout_dict['publishedAt'] = ts
del shout_dict['published'] del shout_dict['published']
del shout_dict['views'] # FIXME # del shout_dict['views']
del shout_dict['rating'] # FIXME # del shout_dict['rating']
del shout_dict['ratings'] del shout_dict['ratings'] # FIXME
try: try:
s = Shout.create(**shout_dict) s = Shout.create(**shout_dict)
r['id'] = s.id r['id'] = s.id
@ -209,4 +217,4 @@ def migrate(entry):
print(r) print(r)
# print(s) # print(s)
raise Exception raise Exception
return (r, content) return r