discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

upgrade migration

Browse Source
This commit is contained in:
Untone
2021-10-08 07:42:59 +03:00
parent 3efba57cf3
commit c3e0c5720a
4 changed files with 260 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
migration/tables/content_items.py
View File

@@ -16,8 +16,9 @@ users_dict['0'] = {
'id': 9999999,
'slug': 'discours.io',
'name': 'Дискурс',
'userpic': 'https://discours.io/images/logo-mini.svg'
}
'userpic': 'https://discours.io/images/logo-mini.svg',
'createdAt': '2016-03-05 22:22:00.350000'
}
ts = datetime.now()
@@ -29,8 +30,9 @@ type2layout = {
'Image': 'image'
}
def migrate(entry, limit=3626, start=0):
'''
def migrate(entry):
'''
type Shout {
slug: String!
author: Int!
@@ -41,7 +43,7 @@ def migrate(entry, limit=3626, start=0):
deletedBy: Int
rating: Int
ratigns: [Rating]
published: Bool!
published: Bool!
publishedAt: DateTime # if there is no published field - it is not published
replyTo: String # another shout
tags: [String] # actual values
@@ -53,17 +55,19 @@ def migrate(entry, limit=3626, start=0):
views: Int
}
'''
content = ''
r = {
'layout': type2layout[entry['type']],
'title': entry['title'],
'community': 0,
'authors': [],
'topics': [],
'published': entry.get('published', False),
'views': entry.get('views', 0),
'rating': entry.get('rating', 0),
'ratings': []
}
'layout': type2layout[entry['type']],
'title': entry['title'],
'community': 0,
'authors': [],
'topics': [],
'published': entry.get('published', False),
'views': entry.get('views', 0),
'rating': entry.get('rating', 0),
'ratings': [],
'createdAt': '2016-03-05 22:22:00.350000'
}
r['slug'] = entry.get('slug', '')
body_orig = entry.get('body', '')
if not r['slug'] and entry.get('friendlySlugs') is not None:
@@ -88,7 +92,8 @@ def migrate(entry, limit=3626, start=0):
if body_orig == '':
print('EMPTY BODY!')
else:
body_html = str(BeautifulSoup(body_orig, features="html.parser"))
body_html = str(BeautifulSoup(
body_orig, features="html.parser"))
r['body'] = html2text(body_html).replace('****', '**')
r['old_id'] = entry.get('_id')
else:
@@ -103,20 +108,20 @@ def migrate(entry, limit=3626, start=0):
if videoUrl == '#':
print(entry.get('media', 'NO MEDIA!'))
# raise Exception
r['body'] = '<ShoutVideo src=\"' + videoUrl + '\" />' + html2text(m.get('body', '')) # FIXME
r['body'] = '<ShoutVideo src=\"' + videoUrl + \
'\" />' + html2text(m.get('body', '')) # FIXME
elif entry.get('type') == 'Music':
r['body'] = '<ShoutMusic media={\"' + json.dumps(entry['media']) +'\"} />' # FIXME
r['body'] = '<ShoutMusic media={\"' + \
json.dumps(entry['media']) + '\"} />' # FIXME
if r.get('body') is None:
body_orig = entry.get('body', '')
body_html = str(BeautifulSoup(body_orig, features="html.parser"))
r['body'] = html2text(body_html).replace('****', '**')
r['old_id'] = entry.get('_id')
body = r.get('body')
user = None
try:
userdata = users_dict[entry['createdBy']]
userdata = users_dict.get(entry['createdBy'], users_dict['0'])
slug = userdata['slug']
name = userdata['name']
userpic = userdata['userpic']
@@ -137,10 +142,11 @@ def migrate(entry, limit=3626, start=0):
user = User.create(**authordata)
except IntegrityError:
with local_session() as session:
user = session.query(User).filter(User.email == authordata['email']).first()
user = session.query(User).filter(
User.email == authordata['email']).first()
if user is None:
user = session.query(User).filter(User.slug == authordata['slug']).first()
user = session.query(User).filter(
User.slug == authordata['slug']).first()
slug = user['slug']
name = user['name']
userpic = user.userpic
@@ -167,15 +173,15 @@ def migrate(entry, limit=3626, start=0):
post = frontmatter.Post(body, **metadata)
dumped = frontmatter.dumps(post)
if entry['published']:
#if r.get('old_id', None):
if entry['published']:
# if r.get('old_id', None):
# ext = 'html'
# content = str(body).replace('<p></p>', '').replace('<p> </p>', '')
#else:
# else:
ext = 'md'
content = dumped
open('migration/content/' + metadata['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content)
open('migration/content/' +
metadata['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content)
try:
shout_dict = r.copy()
@@ -190,8 +196,8 @@ def migrate(entry, limit=3626, start=0):
else:
shout_dict['publishedAt'] = ts
del shout_dict['published']
del shout_dict['views'] # FIXME
del shout_dict['rating'] # FIXME
del shout_dict['views'] # FIXME
del shout_dict['rating'] # FIXME
del shout_dict['ratings']
try:
s = Shout.create(**shout_dict)
@@ -203,4 +209,4 @@ def migrate(entry, limit=3626, start=0):
print(r)
# print(s)
raise Exception
return r
return (r, content)

28
migration/tables/tags.py
View File

@@ -1,20 +1,36 @@
import json
from os.path import abspath
from datetime import datetime
users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read())
users_dict['0'] = {
'id': 9999999,
'slug': 'discours.io',
'name': 'Дискурс',
'userpic': 'https://discours.io/images/logo-mini.svg',
'createdAt': '2016-03-05 22:22:00.350000'
}
ts = datetime.now()
def migrate(entry):
```
'''
type Topic {
slug: String! # ID
createdBy: Int! # User
createdAt: DateTime!
value: String
title: String
parents: [String] # NOTE: topic can have parent topics
children: [String] # and children
}
```
creator = get_new_user_id(entry['createdBy'])
'''
creator = users_dict.get(entry['createdBy'], users_dict['0'])
return {
'slug': entry['slug'],
'createdBy': creator_id, # NOTE: uses an old user id
'createdBy': creator['id'], # NOTE: uses an old user id
'createdAt': entry['createdAt'],
'title': entry['value'].lower(),
'title': entry['title'].lower(),
'parents': [],
'children': []
}

150
migration/tables/users.py
View File

@@ -7,76 +7,82 @@ from migration.html2text import html2text
counter = 0
def migrate(entry, limit=668):
'''
type User {
username: String! # email
createdAt: DateTime!
email: String
password: String
oauth: String # provider:token
name: String # to display
userpic: String
links: [String]
emailConfirmed: Boolean # should contain all emails too
id: Int!
muted: Boolean
rating: Int
roles: [Role]
updatedAt: DateTime
wasOnlineAt: DateTime
ratings: [Rating]
slug: String
bio: String
notifications: [Int]
}
'''
'''
res = {}
res['old_id'] = entry['_id']
res['password'] = entry['services']['password'].get('bcrypt', '')
res['username'] = entry['emails'][0]['address']
res['email'] = res['username']
res['wasOnlineAt'] = parse(entry.get('loggedInAt', entry['createdAt']))
res['emailConfirmed'] = entry['emails'][0]['verified']
res['createdAt'] = parse(entry['createdAt'])
res['rating'] = entry['rating'] # number
res['roles'] = [] # entry['roles'] # roles by community
res['ratings'] = [] # entry['ratings']
res['notifications'] = []
res['links'] = []
res['muted'] = False
res['bio'] = html2text(entry.get('bio', ''))
if entry['profile']:
res['slug'] = entry['profile'].get('path')
res['userpic'] = entry['profile'].get('image', {'thumborId': ''}).get('thumborId', '') # adding 'https://assets.discours.io/unsafe/1600x' in web ui
fn = entry['profile'].get('firstName', '')
ln = entry['profile'].get('lastName', '')
name = res['slug'] if res['slug'] else 'anonymous'
name = fn if fn else name
name = (name + ' ' + ln) if ln else name
name = entry['profile']['path'] if len(name) < 2 else name
res['name'] = name
fb = entry['profile'].get('facebook', False)
if fb:
res['links'].append(fb)
vk = entry['profile'].get('vkontakte', False)
if vk:
res['links'].append(vk)
tr = entry['profile'].get('twitter', False)
if tr:
res['links'].append(tr)
ws = entry['profile'].get('website', False)
if ws:
res['links'].append(ws)
if not res['slug']:
res['slug'] = res['links'][0].split('/')[-1]
if not res['slug']:
res['slug'] = res['email'].split('@')[0]
else:
old = res['old_id']
del res['old_id']
user = User.create(**res.copy())
res['id'] = user.id
res['old_id'] = old
return res
type User {
username: String! # email
createdAt: DateTime!
email: String
password: String
oauth: String # provider:token
name: String # to display
userpic: String
links: [String]
emailConfirmed: Boolean # should contain all emails too
id: Int!
muted: Boolean
rating: Int
roles: [Role]
updatedAt: DateTime
wasOnlineAt: DateTime
ratings: [Rating]
slug: String
bio: String
notifications: [Int]
}
'''
res = {}
res['old_id'] = entry['_id']
res['password'] = entry['services']['password'].get('bcrypt', '')
res['username'] = entry['emails'][0]['address']
res['email'] = res['username']
res['wasOnlineAt'] = parse(entry.get('loggedInAt', entry['createdAt']))
res['emailConfirmed'] = entry['emails'][0]['verified']
res['createdAt'] = parse(entry['createdAt'])
res['rating'] = entry['rating'] # number
res['roles'] = [] # entry['roles'] # roles by community
res['ratings'] = [] # entry['ratings']
res['notifications'] = []
res['links'] = []
res['muted'] = False
res['bio'] = html2text(entry.get('bio', ''))
if entry['profile']:
res['slug'] = entry['profile'].get('path')
try:
res['userpic'] = 'https://assets.discours.io/unsafe/100x/' + entry['profile']['thumborId']
except KeyError:
try:
res['userpic'] = entry['profile']['image']['url']
except KeyError:
res['userpic'] = ''
fn = entry['profile'].get('firstName', '')
ln = entry['profile'].get('lastName', '')
name = res['slug'] if res['slug'] else 'anonymous'
name = fn if fn else name
name = (name + ' ' + ln) if ln else name
name = entry['profile']['path'] if len(name) < 2 else name
res['name'] = name
fb = entry['profile'].get('facebook', False)
if fb:
res['links'].append(fb)
vk = entry['profile'].get('vkontakte', False)
if vk:
res['links'].append(vk)
tr = entry['profile'].get('twitter', False)
if tr:
res['links'].append(tr)
ws = entry['profile'].get('website', False)
if ws:
res['links'].append(ws)
if not res['slug']:
res['slug'] = res['links'][0].split('/')[-1]
if not res['slug']:
res['slug'] = res['email'].split('@')[0]
else:
old = res['old_id']
del res['old_id']
user = User.create(**res.copy())
res['id'] = user.id
res['old_id'] = old
return res