discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

mgrate topic descriptions and shout views

Browse Source
This commit is contained in:
knst-kotov 2021-12-15 10:39:32 +03:00
parent bc52d34b29
commit 01a974e974
3 changed files with 243 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
migration/tables/content_item_categories.py
View File

@ -3,30 +3,31 @@ from orm import Topic, Community
from dateutil.parser import parse as date_parse from dateutil.parser import parse as date_parse
def migrate(entry): def migrate(entry):
''' '''
type Topic { type Topic {
slug: String! # ID slug: String! # ID
createdBy: Int! # User createdBy: Int! # User
createdAt: DateTime! createdAt: DateTime!
value: String value: String
children: [String] # children topic children: [String] # children topic
} }
''' '''
topic_dict = { topic_dict = {
'slug': entry['slug'], 'slug': entry['slug'],
# 'createdBy': entry['createdBy'], # 'createdBy': entry['createdBy'],
# 'createdAt': date_parse(entry['createdAt']), # 'createdAt': date_parse(entry['createdAt']),
'title': entry['title'].lower(), 'title': entry['title'].lower(),
'children': [], 'children': [],
'community' : Community.default_community.slug 'community' : Community.default_community.slug,
} 'body' : entry.get('description')
try: }
with local_session() as session: try:
topic = session.query(Topic).filter(Topic.slug == entry['slug']).first() with local_session() as session:
if not topic: topic = session.query(Topic).filter(Topic.slug == entry['slug']).first()
topic = Topic.create(**topic_dict) if not topic:
except Exception as e: topic = Topic.create(**topic_dict)
print(e) except Exception as e:
raise e print(e)
topic_dict['cat_id'] = entry['_id'] raise e
return topic_dict topic_dict['cat_id'] = entry['_id']
return topic_dict

405
migration/tables/content_items.py
View File

@ -3,7 +3,7 @@ import frontmatter
import json import json
import sqlite3 import sqlite3
import sqlalchemy import sqlalchemy
from orm import Shout, Comment, Topic, ShoutTopic, ShoutRating, User from orm import Shout, Comment, Topic, ShoutTopic, ShoutRating, ShoutViewByDay, User
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from migration.html2text import html2text from migration.html2text import html2text
from migration.tables.comments import migrate as migrateComment from migration.tables.comments import migrate as migrateComment
@ -14,220 +14,227 @@ from orm.base import local_session
from orm.community import Community from orm.community import Community
DISCOURS_USER = { DISCOURS_USER = {
'id': 9999999, 'id': 9999999,
'slug': 'discours', 'slug': 'discours',
'name': 'Дискурс', 'name': 'Дискурс',
'userpic': 'https://discours.io/images/logo-mini.svg', 'userpic': 'https://discours.io/images/logo-mini.svg',
'createdAt': '2016-03-05 22:22:00.350000' 'createdAt': '2016-03-05 22:22:00.350000'
} }
ts = datetime.now() ts = datetime.now()
type2layout = { type2layout = {
'Article': 'article', 'Article': 'article',
'Literature': 'prose', 'Literature': 'prose',
'Music': 'music', 'Music': 'music',
'Video': 'video', 'Video': 'video',
'Image': 'image' 'Image': 'image'
} }
def get_metadata(r): def get_metadata(r):
metadata = {} metadata = {}
metadata['title'] = r.get('title') metadata['title'] = r.get('title')
metadata['authors'] = r.get('authors') metadata['authors'] = r.get('authors')
metadata['createdAt'] = r.get('createdAt', ts) metadata['createdAt'] = r.get('createdAt', ts)
metadata['layout'] = r['layout'] metadata['layout'] = r['layout']
metadata['topics'] = [topic['slug'] for topic in r['topics']] metadata['topics'] = [topic['slug'] for topic in r['topics']]
if r.get('cover', False): if r.get('cover', False):
metadata['cover'] = r.get('cover') metadata['cover'] = r.get('cover')
return metadata return metadata
def migrate(entry, users_by_oid, topics_by_oid): def migrate(entry, users_by_oid, topics_by_oid):
''' '''
type Shout { type Shout {
slug: String! slug: String!
author: Int! author: Int!
body: String! body: String!
createdAt: DateTime! createdAt: DateTime!
updatedAt: DateTime! updatedAt: DateTime!
deletedAt: DateTime deletedAt: DateTime
deletedBy: Int deletedBy: Int
rating: Int rating: Int
ratings: [Rating] ratings: [Rating]
published: Bool! published: Bool!
publishedAt: DateTime # if there is no published field - it is not published publishedAt: DateTime # if there is no published field - it is not published
replyTo: String # another shout replyTo: String # another shout
tags: [String] # actual values tags: [String] # actual values
topics: [String] # topic-slugs, order has matter topics: [String] # topic-slugs, order has matter
title: String title: String
versionOf: String versionOf: String
visibleForRoles: [String] # role ids are strings visibleForRoles: [String] # role ids are strings
visibleForUsers: [Int] visibleForUsers: [Int]
views: Int views: Int
} }
''' '''
# print(entry) # print(entry)
content = '' content = ''
r = { r = {
'layout': type2layout[entry['type']], 'layout': type2layout[entry['type']],
'title': entry['title'], 'title': entry['title'],
'community': Community.default_community.id, 'community': Community.default_community.id,
'authors': [], 'authors': [],
'topics': [], 'topics': [],
'views': entry.get('views', 0), 'rating': entry.get('rating', 0),
'rating': entry.get('rating', 0), 'ratings': [],
'ratings': [], 'createdAt': entry.get('createdAt', '2016-03-05 22:22:00.350000')
'createdAt': entry.get('createdAt', '2016-03-05 22:22:00.350000') }
} r['slug'] = entry.get('slug', '')
r['slug'] = entry.get('slug', '') body_orig = entry.get('body', '')
body_orig = entry.get('body', '') if not r['slug'] and entry.get('friendlySlugs') is not None:
if not r['slug'] and entry.get('friendlySlugs') is not None: r['slug'] = entry['friendlySlugs']['slug'][0]['slug']
r['slug'] = entry['friendlySlugs']['slug'][0]['slug'] if(r['slug'] is None):
if(r['slug'] is None): r['slug'] = entry['friendlySlugs'][0]['slug']
r['slug'] = entry['friendlySlugs'][0]['slug'] if not r['slug']:
if not r['slug']: print('NO SLUG ERROR')
print('NO SLUG ERROR') # print(entry)
# print(entry) raise Exception
raise Exception try:
try: r['topics'].append(topics_by_oid[entry['category']])
r['topics'].append(topics_by_oid[entry['category']]) except Exception:
except Exception: print("invalid category %s" % (entry['category']))
print("invalid category %s" % (entry['category'])) if entry.get('image') is not None:
if entry.get('image') is not None: r['cover'] = entry['image']['url']
r['cover'] = entry['image']['url'] if entry.get('thumborId') is not None:
if entry.get('thumborId') is not None: r['cover'] = 'https://assets.discours.io/unsafe/1600x/' + entry['thumborId']
r['cover'] = 'https://assets.discours.io/unsafe/1600x/' + entry['thumborId'] if entry.get('updatedAt') is not None:
if entry.get('updatedAt') is not None: r['updatedAt'] = date_parse(entry['updatedAt'])
r['updatedAt'] = date_parse(entry['updatedAt']) if entry.get('type') == 'Literature':
if entry.get('type') == 'Literature': media = entry.get('media', '')
media = entry.get('media', '') # print(media[0]['literatureBody'])
# print(media[0]['literatureBody']) if type(media) == list:
if type(media) == list: body_orig = media[0].get('literatureBody', '')
body_orig = media[0].get('literatureBody', '') if body_orig == '':
if body_orig == '': print('EMPTY BODY!')
print('EMPTY BODY!') else:
else: body_html = str(BeautifulSoup(
body_html = str(BeautifulSoup( body_orig, features="html.parser"))
body_orig, features="html.parser")) r['body'] = body_html # html2text(body_html)
r['body'] = body_html # html2text(body_html) else:
else: print(r['slug'] + ': literature has no media')
print(r['slug'] + ': literature has no media') elif entry.get('type') == 'Video':
elif entry.get('type') == 'Video': m = entry['media'][0]
m = entry['media'][0] yt = m.get('youtubeId', '')
yt = m.get('youtubeId', '') vm = m.get('vimeoId', '')
vm = m.get('vimeoId', '') video_url = 'https://www.youtube.com/watch?v=' + yt if yt else '#'
video_url = 'https://www.youtube.com/watch?v=' + yt if yt else '#' if video_url == '#':
if video_url == '#': video_url = 'https://vimeo.com/' + vm if vm else '#'
video_url = 'https://vimeo.com/' + vm if vm else '#' if video_url == '#':
if video_url == '#': print(entry.get('media', 'NO MEDIA!'))
print(entry.get('media', 'NO MEDIA!')) # raise Exception
# raise Exception r['body'] = '<ShoutVideo src=\"' + video_url + \
r['body'] = '<ShoutVideo src=\"' + video_url + \ '\" />' + html2text(m.get('body', '')) # FIXME
'\" />' + html2text(m.get('body', '')) # FIXME elif entry.get('type') == 'Music':
elif entry.get('type') == 'Music': r['body'] = '<ShoutMusic media={\"' + \
r['body'] = '<ShoutMusic media={\"' + \ json.dumps(entry['media']) + '\"} />' # FIXME
json.dumps(entry['media']) + '\"} />' # FIXME if r.get('body') is None:
if r.get('body') is None: body_orig = entry.get('body', '')
body_orig = entry.get('body', '') body_html = str(BeautifulSoup(body_orig, features="html.parser"))
body_html = str(BeautifulSoup(body_orig, features="html.parser")) r['body'] = body_html # html2text(body_html)
r['body'] = body_html # html2text(body_html) body = r.get('body', '')
body = r.get('body', '')
# get author data # get author data
userdata = {} userdata = {}
try: userdata = users_by_oid[entry['createdBy']] try: userdata = users_by_oid[entry['createdBy']]
except KeyError: except KeyError:
app = entry.get('application') app = entry.get('application')
if app: if app:
userslug = translit(app['name'], 'ru', reversed=True).replace(' ', '-').replace('\'', '').replace('.', '-').lower() userslug = translit(app['name'], 'ru', reversed=True).replace(' ', '-').replace('\'', '').replace('.', '-').lower()
userdata = { userdata = {
'username': app['email'], 'username': app['email'],
'email': app['email'], 'email': app['email'],
'name': app['name'], 'name': app['name'],
'bio': app.get('bio', ''), 'bio': app.get('bio', ''),
'emailConfirmed': False, 'emailConfirmed': False,
'slug': userslug, 'slug': userslug,
'createdAt': ts, 'createdAt': ts,
'wasOnlineAt': ts 'wasOnlineAt': ts
} }
if userdata == {}: if userdata == {}:
userdata = { userdata = {
'name': 'Дискурс', 'name': 'Дискурс',
'slug': 'discours', 'slug': 'discours',
'userpic': 'https://discours.io/image/logo-mini.svg' 'userpic': 'https://discours.io/image/logo-mini.svg'
} }
# set author data # set author data
shout_dict = r.copy() shout_dict = r.copy()
author = { # a short version for public listings author = { # a short version for public listings
'slug': userdata.get('slug', 'discours'), 'slug': userdata.get('slug', 'discours'),
'name': userdata.get('name', 'Дискурс'), 'name': userdata.get('name', 'Дискурс'),
'userpic': userdata.get('userpic', '') 'userpic': userdata.get('userpic', '')
} }
shout_dict['authors'] = [ author, ] shout_dict['authors'] = [ author, ]
if entry['published']: if entry['published']:
metadata = get_metadata(r) metadata = get_metadata(r)
content = frontmatter.dumps(frontmatter.Post(body, **metadata)) content = frontmatter.dumps(frontmatter.Post(body, **metadata))
ext = 'md' ext = 'md'
open('migration/content/' + r['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content) open('migration/content/' + r['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content)
try: try:
shout_dict['createdAt'] = date_parse(r.get('createdAt')) if entry.get('createdAt') else ts shout_dict['createdAt'] = date_parse(r.get('createdAt')) if entry.get('createdAt') else ts
shout_dict['publishedAt'] = date_parse(entry.get('publishedAt')) if entry.get('published') else None shout_dict['publishedAt'] = date_parse(entry.get('publishedAt')) if entry.get('published') else None
if entry.get('deletedAt') is not None: if entry.get('deletedAt') is not None:
shout_dict['deletedAt'] = date_parse(entry.get('deletedAt')) shout_dict['deletedAt'] = date_parse(entry.get('deletedAt'))
shout_dict['deletedBy'] = entry.get('deletedBy', '0') shout_dict['deletedBy'] = entry.get('deletedBy', '0')
del shout_dict['topics'] # FIXME: AttributeError: 'str' object has no attribute '_sa_instance_state' del shout_dict['topics'] # FIXME: AttributeError: 'str' object has no attribute '_sa_instance_state'
del shout_dict['views'] # FIXME: TypeError: 'views' is an invalid keyword argument for Shout del shout_dict['rating'] # FIXME: TypeError: 'rating' is an invalid keyword argument for Shout
del shout_dict['rating'] # FIXME: TypeError: 'rating' is an invalid keyword argument for Shout del shout_dict['ratings']
del shout_dict['ratings']
# get user # get user
user = None user = None
email = userdata.get('email') email = userdata.get('email')
slug = userdata.get('slug') slug = userdata.get('slug')
with local_session() as session: with local_session() as session:
try: try:
if email: user = session.query(User).filter(User.email == email).first() if email: user = session.query(User).filter(User.email == email).first()
if not user and slug: user = session.query(User).filter(User.slug == slug).first() if not user and slug: user = session.query(User).filter(User.slug == slug).first()
if not user and userdata: user = User.create(**userdata) if not user and userdata: user = User.create(**userdata)
except: except:
print(userdata) print(userdata)
assert user, 'could not get a user' assert user, 'could not get a user'
shout_dict['authors'] = [ user, ] shout_dict['authors'] = [ user, ]
try: try:
with local_session() as session: s = Shout.create(**shout_dict)
s = Shout.create(**shout_dict)
if s: # shout ratings
# shout ratings shout_dict['ratings'] = []
shout_dict['ratings'] = [] for shout_rating_old in entry.get('ratings',[]):
for shout_rating_old in entry.get('ratings',[]): with local_session() as session:
rater = session.query(User).filter(User.old_id == shout_rating_old['createdBy']).first() rater = session.query(User).\
if rater: filter(User.old_id == shout_rating_old['createdBy']).first()
shout_rating_dict = { if rater:
'value': shout_rating_old['value'], shout_rating_dict = {
'rater': rater.id, 'value': shout_rating_old['value'],
'shout': s.slug 'rater': rater.id,
} 'shout': s.slug
cts = shout_rating_old.get('createdAt') }
if cts: shout_rating_dict['rater_id'] = date_parse(cts) cts = shout_rating_old.get('createdAt')
try: shout_rating = ShoutRating.create(**shout_rating_dict) if cts: shout_rating_dict['rater_id'] = date_parse(cts)
except sqlalchemy.exc.IntegrityError: pass try: shout_rating = ShoutRating.create(**shout_rating_dict)
shout_dict['ratings'].append(shout_rating_dict) except sqlalchemy.exc.IntegrityError: pass
# shout topics shout_dict['ratings'].append(shout_rating_dict)
shout_dict['topics'] = []
for topic in r['topics']: # shout topics
ShoutTopic.create(**{ 'shout': s.slug, 'topic': topic['slug'] }) shout_dict['topics'] = []
shout_dict['topics'].append(topic['slug']) for topic in r['topics']:
except Exception as e: ShoutTopic.create(**{ 'shout': s.slug, 'topic': topic['slug'] })
raise e shout_dict['topics'].append(topic['slug'])
except Exception as e:
if not shout_dict['body']: r['body'] = 'body moved' views = entry.get('views', 1)
raise e ShoutViewByDay.create(
shout_dict['old_id'] = entry.get('_id') shout = s.slug,
return shout_dict # for json value = views
)
except Exception as e:
raise e
except Exception as e:
if not shout_dict['body']: r['body'] = 'body moved'
raise e
shout_dict['old_id'] = entry.get('_id')
return shout_dict # for json

3
migration/tables/tags.py
View File

@ -25,7 +25,8 @@ def migrate(entry):
# 'createdAt': ts, # 'createdAt': ts,
'title': entry['title'].lower(), 'title': entry['title'].lower(),
'children': [], 'children': [],
'community' : Community.default_community.slug 'community' : Community.default_community.slug,
'body' : entry.get('description')
} }
try: try:
with local_session() as session: with local_session() as session: