discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

upgrade migration

Browse Source
This commit is contained in:
Untone 2021-10-08 07:42:59 +03:00
parent 3efba57cf3
commit c3e0c5720a
4 changed files with 260 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

186
migrate.py
View File

@ -1,18 +1,45 @@
''' cmd managed migration '''
import json import json
import base64
import re
from migration.tables.users import migrate as migrateUser from migration.tables.users import migrate as migrateUser
from migration.tables.content_items import migrate as migrateShout from migration.tables.content_items import migrate as migrateShout
from migration.tables.content_item_categories import migrate as migrateTopic from migration.tables.content_item_categories import migrate as migrateCategory
from migration.tables.tags import migrate as migrateTag
from migration.utils import DateTimeEncoder from migration.utils import DateTimeEncoder
from orm import Community from orm import Community
def users(limit):
IMG_REGEX = r"\!\[(.*?)\]\((data\:image\/(png|jpeg|jpg);base64\,(.*?))\)"
OLD_DATE = '2016-03-05 22:22:00.350000'
def extract_images(article):
''' extract b64 encoded images from markdown in article body '''
body = article['body']
images = []
matches = re.finditer(IMG_REGEX, body, re.IGNORECASE | re.MULTILINE)
for i, match in enumerate(matches, start=1):
ext = match.group(3)
link = '/static/upload/image-' + \
article['old_id'] + str(i) + '.' + ext
img = match.group(4)
if img not in images:
open('..' + link, 'wb').write(base64.b64decode(img))
images.append(img)
body = body.replace(match.group(2), link)
print(link)
article['body'] = body
return article
def users():
''' migrating users first '''
print('migrating users...') print('migrating users...')
data = json.loads(open('migration/data/users.json').read())
newdata = {} newdata = {}
exportData = {} data = json.loads(open('migration/data/users.json').read())
counter = 0 counter = 0
# limit = 100 export_data = {}
#try:
for entry in data: for entry in data:
oid = entry['_id'] oid = entry['_id']
user = migrateUser(entry) user = migrateUser(entry)
@ -23,96 +50,127 @@ def users(limit):
del user['emailConfirmed'] del user['emailConfirmed']
del user['username'] del user['username']
del user['email'] del user['email']
exportData[user['slug']] = user export_data[user['slug']] = user
counter += 1 counter += 1
if counter > limit: export_list = sorted(export_data.items(),
break key=lambda item: item[1]['rating'])[-10:]
#except Exception: open('migration/data/users.dict.json',
# print(str(counter) + '/' + str(len(data)) + ' users entries were migrated') 'w').write(json.dumps(newdata, cls=DateTimeEncoder)) # NOTE: by old_id
# print('try to remove database first') open('../src/data/authors.json', 'w').write(json.dumps(dict(export_list),
open('migration/data/users.dict.json','w').write( json.dumps(newdata, cls=DateTimeEncoder) ) cls=DateTimeEncoder,
open('../src/data/authors.json','w').write( json.dumps(exportData, cls=DateTimeEncoder) ) indent=4,
print(str(counter) + ' users entries were migrated') sort_keys=True,
ensure_ascii=False))
print(str(len(newdata.items())) + ' user accounts were migrated')
print(str(len(export_list)) + ' authors were exported')
def topics(): def topics():
''' topics from categories and tags '''
print('migrating topics...') print('migrating topics...')
data = json.loads(open('migration/data/content_item_categories.json').read()) cat_data = json.loads(
open('migration/data/content_item_categories.json').read())
tag_data = json.loads(open('migration/data/tags.json').read())
newdata = {} newdata = {}
exportData = {}
counter = 0 counter = 0
try: try:
for entry in data: for cat in cat_data:
oid = entry['_id'] topic = migrateCategory(cat)
newdata[oid] = migrateTopic(entry) newdata[topic['slug']] = topic
exportData[entry['slug']] = newdata[oid]
counter += 1 counter += 1
except Exception: except Exception:
print(str(counter) + '/' + str(len(data)) + ' topics were migrated') print('cats exception, try to remove database first')
print('try to remove database first') try:
open('migration/data/topics.dict.json','w').write( json.dumps(newdata, cls=DateTimeEncoder) ) for tag in tag_data:
open('../src/data/topics.json','w').write( json.dumps(exportData, cls=DateTimeEncoder) ) topic = migrateTag(tag)
print(str(counter) + ' topics were migrated') newdata[topic['slug']] = topic
counter += 1
except Exception:
print('tags exception, try to remove database first')
raise Exception
export_list = sorted(newdata.items(), key=lambda item: str(
item[1]['createdAt']))[-10:]
open('migration/data/topics.dict.json',
'w').write(json.dumps(newdata, cls=DateTimeEncoder))
open('../src/data/topics.json', 'w').write(json.dumps(dict(export_list),
cls=DateTimeEncoder, indent=4, sort_keys=True, ensure_ascii=False))
print(str(counter) + ' from ' + str(len(cat_data)) +
' tags and ' + str(len(tag_data)) + ' cats were migrated')
print(str(len(export_list)) + ' topics were exported')
def shouts(limit):
def shouts():
''' migrating content items one by one '''
print('loading shouts...') print('loading shouts...')
counter = 0 counter = 0
discoursAuthor = 0 discours_author = 0
data = json.loads(open('migration/data/content_items.json').read()) content_data = json.loads(open('migration/data/content_items.json').read())
newdata = {} newdata = {}
print(str(len(data)) + ' entries loaded. now migrating...') print(str(len(content_data)) + ' entries loaded. now migrating...')
errored = [] errored = []
exportData = {} for entry in content_data:
for entry in data:
try: try:
oid = entry['_id'] (shout, content) = migrateShout(entry)
shout = migrateShout(entry) newdata[shout['slug']] = shout
newdata[oid] = shout author = newdata[shout['slug']]['authors'][0]['slug']
author = newdata[oid]['authors'][0]['slug'] line = str(counter+1) + ': ' + shout['slug'] + " @" + str(author)
line = str(counter) + ': ' + newdata[oid]['slug'] + " @" + str(author) print(line)
if shout['layout'] == 'article': counter += 1
counter += 1
exportData[shout['slug']] = shout
print(line)
# counter += 1
if author == 'discours.io': if author == 'discours.io':
discoursAuthor += 1 discours_author += 1
open('./shouts.id.log','a').write(line + '\n') open('./shouts.id.log', 'a').write(line + '\n')
if counter > limit:
break
except Exception: except Exception:
print(entry['_id']) print(entry['_id'])
errored.append(entry) errored.append(entry)
raise Exception raise Exception(" error")
try:
limit = int(sys.argv[2]) if len(sys.argv) > 2 else len(content_data)
except ValueError:
limit = len(content_data)
export_list = sorted(newdata.items(
), key=lambda item: item[1]['createdAt'] if item[1]['layout'] == 'article' else OLD_DATE)[:limit]
export_clean = {}
for slug, a in dict(export_list).items():
export_clean[slug] = extract_images(a)
open('../content/discours.io/'+slug+'.md', 'w').write(content)
open('migration/data/shouts.dict.json',
'w').write(json.dumps(newdata, cls=DateTimeEncoder))
open('../src/data/articles.json', 'w').write(json.dumps(dict(export_clean),
cls=DateTimeEncoder,
indent=4,
sort_keys=True,
ensure_ascii=False))
print(str(counter) + '/' + str(len(content_data)) +
' content items were migrated')
print(str(len(export_list)) + ' shouts were exported')
print(str(discours_author) + ' from them by @discours.io')
open('migration/data/shouts.dict.json','w').write( json.dumps(newdata, cls=DateTimeEncoder) )
open('../src/data/articles.json','w').write( json.dumps(exportData, cls=DateTimeEncoder) )
print(str(counter) + ' shouts were migrated')
print(str(discoursAuthor) + ' from them by @discours.io')
print(str(len(errored)) + ' shouts without authors')
if __name__ == '__main__': if __name__ == '__main__':
import sys import sys
if len(sys.argv) > 1: if len(sys.argv) > 1:
if sys.argv[1] == "users": if sys.argv[1] == "users":
users(668) users()
elif sys.argv[1] == "topics": elif sys.argv[1] == "topics":
topics() topics()
elif sys.argv[1] == "shouts": elif sys.argv[1] == "shouts":
Community.create(**{ try:
'slug': 'discours.io', Community.create(**{
'name': 'Дискурс', 'slug': 'discours.io',
'pic': 'https://discours.io/images/logo-min.svg', 'name': 'Дискурс',
'createdBy': '0', 'pic': 'https://discours.io/images/logo-min.svg',
'createdAt': ts 'createdBy': '0',
'createdAt': OLD_DATE
}) })
shouts(3626) except Exception:
pass
shouts()
elif sys.argv[1] == "all": elif sys.argv[1] == "all":
users()
topics() topics()
users(668) shouts()
shouts(3626)
elif sys.argv[1] == "bson": elif sys.argv[1] == "bson":
import migration.bson2json from migration import bson2json
bson2json.json_tables() bson2json.json_tables()
else: else:
print('usage: python migrate.py <all|topics|users|shouts|comments>') print('usage: python migrate.py <bson|all|topics|users|shouts>')

64
migration/tables/content_items.py
View File

@ -16,8 +16,9 @@ users_dict['0'] = {
'id': 9999999, 'id': 9999999,
'slug': 'discours.io', 'slug': 'discours.io',
'name': 'Дискурс', 'name': 'Дискурс',
'userpic': 'https://discours.io/images/logo-mini.svg' 'userpic': 'https://discours.io/images/logo-mini.svg',
} 'createdAt': '2016-03-05 22:22:00.350000'
}
ts = datetime.now() ts = datetime.now()
@ -29,7 +30,8 @@ type2layout = {
'Image': 'image' 'Image': 'image'
} }
def migrate(entry, limit=3626, start=0):
def migrate(entry):
''' '''
type Shout { type Shout {
slug: String! slug: String!
@ -53,17 +55,19 @@ def migrate(entry, limit=3626, start=0):
views: Int views: Int
} }
''' '''
content = ''
r = { r = {
'layout': type2layout[entry['type']], 'layout': type2layout[entry['type']],
'title': entry['title'], 'title': entry['title'],
'community': 0, 'community': 0,
'authors': [], 'authors': [],
'topics': [], 'topics': [],
'published': entry.get('published', False), 'published': entry.get('published', False),
'views': entry.get('views', 0), 'views': entry.get('views', 0),
'rating': entry.get('rating', 0), 'rating': entry.get('rating', 0),
'ratings': [] 'ratings': [],
} 'createdAt': '2016-03-05 22:22:00.350000'
}
r['slug'] = entry.get('slug', '') r['slug'] = entry.get('slug', '')
body_orig = entry.get('body', '') body_orig = entry.get('body', '')
if not r['slug'] and entry.get('friendlySlugs') is not None: if not r['slug'] and entry.get('friendlySlugs') is not None:
@ -88,7 +92,8 @@ def migrate(entry, limit=3626, start=0):
if body_orig == '': if body_orig == '':
print('EMPTY BODY!') print('EMPTY BODY!')
else: else:
body_html = str(BeautifulSoup(body_orig, features="html.parser")) body_html = str(BeautifulSoup(
body_orig, features="html.parser"))
r['body'] = html2text(body_html).replace('****', '**') r['body'] = html2text(body_html).replace('****', '**')
r['old_id'] = entry.get('_id') r['old_id'] = entry.get('_id')
else: else:
@ -103,20 +108,20 @@ def migrate(entry, limit=3626, start=0):
if videoUrl == '#': if videoUrl == '#':
print(entry.get('media', 'NO MEDIA!')) print(entry.get('media', 'NO MEDIA!'))
# raise Exception # raise Exception
r['body'] = '<ShoutVideo src=\"' + videoUrl + '\" />' + html2text(m.get('body', '')) # FIXME r['body'] = '<ShoutVideo src=\"' + videoUrl + \
'\" />' + html2text(m.get('body', '')) # FIXME
elif entry.get('type') == 'Music': elif entry.get('type') == 'Music':
r['body'] = '<ShoutMusic media={\"' + json.dumps(entry['media']) +'\"} />' # FIXME r['body'] = '<ShoutMusic media={\"' + \
json.dumps(entry['media']) + '\"} />' # FIXME
if r.get('body') is None: if r.get('body') is None:
body_orig = entry.get('body', '') body_orig = entry.get('body', '')
body_html = str(BeautifulSoup(body_orig, features="html.parser")) body_html = str(BeautifulSoup(body_orig, features="html.parser"))
r['body'] = html2text(body_html).replace('****', '**') r['body'] = html2text(body_html).replace('****', '**')
r['old_id'] = entry.get('_id') r['old_id'] = entry.get('_id')
body = r.get('body') body = r.get('body')
user = None user = None
try: try:
userdata = users_dict[entry['createdBy']] userdata = users_dict.get(entry['createdBy'], users_dict['0'])
slug = userdata['slug'] slug = userdata['slug']
name = userdata['name'] name = userdata['name']
userpic = userdata['userpic'] userpic = userdata['userpic']
@ -137,10 +142,11 @@ def migrate(entry, limit=3626, start=0):
user = User.create(**authordata) user = User.create(**authordata)
except IntegrityError: except IntegrityError:
with local_session() as session: with local_session() as session:
user = session.query(User).filter(User.email == authordata['email']).first() user = session.query(User).filter(
User.email == authordata['email']).first()
if user is None: if user is None:
user = session.query(User).filter(User.slug == authordata['slug']).first() user = session.query(User).filter(
User.slug == authordata['slug']).first()
slug = user['slug'] slug = user['slug']
name = user['name'] name = user['name']
userpic = user.userpic userpic = user.userpic
@ -168,14 +174,14 @@ def migrate(entry, limit=3626, start=0):
dumped = frontmatter.dumps(post) dumped = frontmatter.dumps(post)
if entry['published']: if entry['published']:
#if r.get('old_id', None): # if r.get('old_id', None):
# ext = 'html' # ext = 'html'
# content = str(body).replace('<p></p>', '').replace('<p> </p>', '') # content = str(body).replace('<p></p>', '').replace('<p> </p>', '')
#else: # else:
ext = 'md' ext = 'md'
content = dumped content = dumped
open('migration/content/' + metadata['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content) open('migration/content/' +
metadata['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content)
try: try:
shout_dict = r.copy() shout_dict = r.copy()
@ -190,8 +196,8 @@ def migrate(entry, limit=3626, start=0):
else: else:
shout_dict['publishedAt'] = ts shout_dict['publishedAt'] = ts
del shout_dict['published'] del shout_dict['published']
del shout_dict['views'] # FIXME del shout_dict['views'] # FIXME
del shout_dict['rating'] # FIXME del shout_dict['rating'] # FIXME
del shout_dict['ratings'] del shout_dict['ratings']
try: try:
s = Shout.create(**shout_dict) s = Shout.create(**shout_dict)
@ -203,4 +209,4 @@ def migrate(entry, limit=3626, start=0):
print(r) print(r)
# print(s) # print(s)
raise Exception raise Exception
return r return (r, content)

28
migration/tables/tags.py
View File

@ -1,20 +1,36 @@
import json
from os.path import abspath
from datetime import datetime
users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read())
users_dict['0'] = {
'id': 9999999,
'slug': 'discours.io',
'name': 'Дискурс',
'userpic': 'https://discours.io/images/logo-mini.svg',
'createdAt': '2016-03-05 22:22:00.350000'
}
ts = datetime.now()
def migrate(entry): def migrate(entry):
``` '''
type Topic { type Topic {
slug: String! # ID slug: String! # ID
createdBy: Int! # User createdBy: Int! # User
createdAt: DateTime! createdAt: DateTime!
value: String title: String
parents: [String] # NOTE: topic can have parent topics parents: [String] # NOTE: topic can have parent topics
children: [String] # and children children: [String] # and children
} }
``` '''
creator = get_new_user_id(entry['createdBy']) creator = users_dict.get(entry['createdBy'], users_dict['0'])
return { return {
'slug': entry['slug'], 'slug': entry['slug'],
'createdBy': creator_id, # NOTE: uses an old user id 'createdBy': creator['id'], # NOTE: uses an old user id
'createdAt': entry['createdAt'], 'createdAt': entry['createdAt'],
'title': entry['value'].lower(), 'title': entry['title'].lower(),
'parents': [], 'parents': [],
'children': [] 'children': []
} }

148
migration/tables/users.py
View File

@ -7,76 +7,82 @@ from migration.html2text import html2text
counter = 0 counter = 0
def migrate(entry, limit=668): def migrate(entry, limit=668):
''' '''
type User { type User {
username: String! # email username: String! # email
createdAt: DateTime! createdAt: DateTime!
email: String email: String
password: String password: String
oauth: String # provider:token oauth: String # provider:token
name: String # to display name: String # to display
userpic: String userpic: String
links: [String] links: [String]
emailConfirmed: Boolean # should contain all emails too emailConfirmed: Boolean # should contain all emails too
id: Int! id: Int!
muted: Boolean muted: Boolean
rating: Int rating: Int
roles: [Role] roles: [Role]
updatedAt: DateTime updatedAt: DateTime
wasOnlineAt: DateTime wasOnlineAt: DateTime
ratings: [Rating] ratings: [Rating]
slug: String slug: String
bio: String bio: String
notifications: [Int] notifications: [Int]
} }
''' '''
res = {} res = {}
res['old_id'] = entry['_id'] res['old_id'] = entry['_id']
res['password'] = entry['services']['password'].get('bcrypt', '') res['password'] = entry['services']['password'].get('bcrypt', '')
res['username'] = entry['emails'][0]['address'] res['username'] = entry['emails'][0]['address']
res['email'] = res['username'] res['email'] = res['username']
res['wasOnlineAt'] = parse(entry.get('loggedInAt', entry['createdAt'])) res['wasOnlineAt'] = parse(entry.get('loggedInAt', entry['createdAt']))
res['emailConfirmed'] = entry['emails'][0]['verified'] res['emailConfirmed'] = entry['emails'][0]['verified']
res['createdAt'] = parse(entry['createdAt']) res['createdAt'] = parse(entry['createdAt'])
res['rating'] = entry['rating'] # number res['rating'] = entry['rating'] # number
res['roles'] = [] # entry['roles'] # roles by community res['roles'] = [] # entry['roles'] # roles by community
res['ratings'] = [] # entry['ratings'] res['ratings'] = [] # entry['ratings']
res['notifications'] = [] res['notifications'] = []
res['links'] = [] res['links'] = []
res['muted'] = False res['muted'] = False
res['bio'] = html2text(entry.get('bio', '')) res['bio'] = html2text(entry.get('bio', ''))
if entry['profile']: if entry['profile']:
res['slug'] = entry['profile'].get('path') res['slug'] = entry['profile'].get('path')
res['userpic'] = entry['profile'].get('image', {'thumborId': ''}).get('thumborId', '') # adding 'https://assets.discours.io/unsafe/1600x' in web ui try:
fn = entry['profile'].get('firstName', '') res['userpic'] = 'https://assets.discours.io/unsafe/100x/' + entry['profile']['thumborId']
ln = entry['profile'].get('lastName', '') except KeyError:
name = res['slug'] if res['slug'] else 'anonymous' try:
name = fn if fn else name res['userpic'] = entry['profile']['image']['url']
name = (name + ' ' + ln) if ln else name except KeyError:
name = entry['profile']['path'] if len(name) < 2 else name res['userpic'] = ''
res['name'] = name fn = entry['profile'].get('firstName', '')
fb = entry['profile'].get('facebook', False) ln = entry['profile'].get('lastName', '')
if fb: name = res['slug'] if res['slug'] else 'anonymous'
res['links'].append(fb) name = fn if fn else name
vk = entry['profile'].get('vkontakte', False) name = (name + ' ' + ln) if ln else name
if vk: name = entry['profile']['path'] if len(name) < 2 else name
res['links'].append(vk) res['name'] = name
tr = entry['profile'].get('twitter', False) fb = entry['profile'].get('facebook', False)
if tr: if fb:
res['links'].append(tr) res['links'].append(fb)
ws = entry['profile'].get('website', False) vk = entry['profile'].get('vkontakte', False)
if ws: if vk:
res['links'].append(ws) res['links'].append(vk)
if not res['slug']: tr = entry['profile'].get('twitter', False)
res['slug'] = res['links'][0].split('/')[-1] if tr:
if not res['slug']: res['links'].append(tr)
res['slug'] = res['email'].split('@')[0] ws = entry['profile'].get('website', False)
else: if ws:
old = res['old_id'] res['links'].append(ws)
del res['old_id'] if not res['slug']:
user = User.create(**res.copy()) res['slug'] = res['links'][0].split('/')[-1]
res['id'] = user.id if not res['slug']:
res['old_id'] = old res['slug'] = res['email'].split('@')[0]
return res else:
old = res['old_id']
del res['old_id']
user = User.create(**res.copy())
res['id'] = user.id
res['old_id'] = old
return res