format and lint orm
This commit is contained in:
@@ -1,4 +1,3 @@
|
||||
|
||||
from datetime import datetime
|
||||
import json
|
||||
import os
|
||||
@@ -6,100 +5,150 @@ import frontmatter
|
||||
from .extract import extract_html, prepare_html_body
|
||||
from .utils import DateTimeEncoder
|
||||
|
||||
OLD_DATE = '2016-03-05 22:22:00.350000'
|
||||
EXPORT_DEST = '../discoursio-web/data/'
|
||||
parentDir = '/'.join(os.getcwd().split('/')[:-1])
|
||||
contentDir = parentDir + '/discoursio-web/content/'
|
||||
OLD_DATE = "2016-03-05 22:22:00.350000"
|
||||
EXPORT_DEST = "../discoursio-web/data/"
|
||||
parentDir = "/".join(os.getcwd().split("/")[:-1])
|
||||
contentDir = parentDir + "/discoursio-web/content/"
|
||||
ts = datetime.now()
|
||||
|
||||
|
||||
def get_metadata(r):
|
||||
authors = []
|
||||
for a in r['authors']:
|
||||
authors.append({ # a short version for public listings
|
||||
'slug': a.slug or 'discours',
|
||||
'name': a.name or 'Дискурс',
|
||||
'userpic': a.userpic or 'https://discours.io/static/img/discours.png'
|
||||
})
|
||||
metadata = {}
|
||||
metadata['title'] = r.get('title', '').replace('{', '(').replace('}', ')')
|
||||
metadata['authors'] = authors
|
||||
metadata['createdAt'] = r.get('createdAt', ts)
|
||||
metadata['layout'] = r['layout']
|
||||
metadata['topics'] = [topic for topic in r['topics']]
|
||||
metadata['topics'].sort()
|
||||
if r.get('cover', False): metadata['cover'] = r.get('cover')
|
||||
return metadata
|
||||
|
||||
authors = []
|
||||
for a in r["authors"]:
|
||||
authors.append(
|
||||
{ # a short version for public listings
|
||||
"slug": a.slug or "discours",
|
||||
"name": a.name or "Дискурс",
|
||||
"userpic": a.userpic or "https://discours.io/static/img/discours.png",
|
||||
}
|
||||
)
|
||||
metadata = {}
|
||||
metadata["title"] = r.get("title", "").replace("{", "(").replace("}", ")")
|
||||
metadata["authors"] = authors
|
||||
metadata["createdAt"] = r.get("createdAt", ts)
|
||||
metadata["layout"] = r["layout"]
|
||||
metadata["topics"] = [topic for topic in r["topics"]]
|
||||
metadata["topics"].sort()
|
||||
if r.get("cover", False):
|
||||
metadata["cover"] = r.get("cover")
|
||||
return metadata
|
||||
|
||||
|
||||
def export_mdx(r):
|
||||
# print('[export] mdx %s' % r['slug'])
|
||||
content = ''
|
||||
metadata = get_metadata(r)
|
||||
content = frontmatter.dumps(frontmatter.Post(r['body'], **metadata))
|
||||
ext = 'mdx'
|
||||
filepath = contentDir + r['slug']
|
||||
bc = bytes(content,'utf-8').decode('utf-8','ignore')
|
||||
open(filepath + '.' + ext, 'w').write(bc)
|
||||
# print('[export] mdx %s' % r['slug'])
|
||||
content = ""
|
||||
metadata = get_metadata(r)
|
||||
content = frontmatter.dumps(frontmatter.Post(r["body"], **metadata))
|
||||
ext = "mdx"
|
||||
filepath = contentDir + r["slug"]
|
||||
bc = bytes(content, "utf-8").decode("utf-8", "ignore")
|
||||
open(filepath + "." + ext, "w").write(bc)
|
||||
|
||||
|
||||
def export_body(shout, storage):
|
||||
entry = storage['content_items']['by_oid'][shout['oid']]
|
||||
if entry:
|
||||
shout['body'] = prepare_html_body(entry) # prepare_md_body(entry)
|
||||
export_mdx(shout)
|
||||
print('[export] html for %s' % shout['slug'])
|
||||
body = extract_html(entry)
|
||||
open(contentDir + shout['slug'] + '.html', 'w').write(body)
|
||||
else:
|
||||
raise Exception('no content_items entry found')
|
||||
entry = storage["content_items"]["by_oid"][shout["oid"]]
|
||||
if entry:
|
||||
shout["body"] = prepare_html_body(entry) # prepare_md_body(entry)
|
||||
export_mdx(shout)
|
||||
print("[export] html for %s" % shout["slug"])
|
||||
body = extract_html(entry)
|
||||
open(contentDir + shout["slug"] + ".html", "w").write(body)
|
||||
else:
|
||||
raise Exception("no content_items entry found")
|
||||
|
||||
|
||||
def export_slug(slug, storage):
|
||||
shout = storage['shouts']['by_slug'][slug]
|
||||
shout = storage['shouts']['by_slug'].get(slug)
|
||||
assert shout, '[export] no shout found by slug: %s ' % slug
|
||||
author = shout['authors'][0]
|
||||
assert author, '[export] no author error'
|
||||
export_body(shout, storage)
|
||||
shout = storage["shouts"]["by_slug"][slug]
|
||||
shout = storage["shouts"]["by_slug"].get(slug)
|
||||
assert shout, "[export] no shout found by slug: %s " % slug
|
||||
author = shout["authors"][0]
|
||||
assert author, "[export] no author error"
|
||||
export_body(shout, storage)
|
||||
|
||||
|
||||
def export_email_subscriptions():
|
||||
email_subscriptions_data = json.loads(open('migration/data/email_subscriptions.json').read())
|
||||
for data in email_subscriptions_data:
|
||||
# migrate_email_subscription(data)
|
||||
pass
|
||||
print('[migration] ' + str(len(email_subscriptions_data)) + ' email subscriptions exported')
|
||||
email_subscriptions_data = json.loads(
|
||||
open("migration/data/email_subscriptions.json").read()
|
||||
)
|
||||
for data in email_subscriptions_data:
|
||||
# migrate_email_subscription(data)
|
||||
pass
|
||||
print(
|
||||
"[migration] "
|
||||
+ str(len(email_subscriptions_data))
|
||||
+ " email subscriptions exported"
|
||||
)
|
||||
|
||||
|
||||
def export_shouts(storage):
|
||||
# update what was just migrated or load json again
|
||||
if len(storage['users']['by_slugs'].keys()) == 0:
|
||||
storage['users']['by_slugs'] = json.loads(open(EXPORT_DEST + 'authors.json').read())
|
||||
print('[migration] ' + str(len(storage['users']['by_slugs'].keys())) + ' exported authors ')
|
||||
if len(storage['shouts']['by_slugs'].keys()) == 0:
|
||||
storage['shouts']['by_slugs'] = json.loads(open(EXPORT_DEST + 'articles.json').read())
|
||||
print('[migration] ' + str(len(storage['shouts']['by_slugs'].keys())) + ' exported articles ')
|
||||
for slug in storage['shouts']['by_slugs'].keys(): export_slug(slug, storage)
|
||||
# update what was just migrated or load json again
|
||||
if len(storage["users"]["by_slugs"].keys()) == 0:
|
||||
storage["users"]["by_slugs"] = json.loads(
|
||||
open(EXPORT_DEST + "authors.json").read()
|
||||
)
|
||||
print(
|
||||
"[migration] "
|
||||
+ str(len(storage["users"]["by_slugs"].keys()))
|
||||
+ " exported authors "
|
||||
)
|
||||
if len(storage["shouts"]["by_slugs"].keys()) == 0:
|
||||
storage["shouts"]["by_slugs"] = json.loads(
|
||||
open(EXPORT_DEST + "articles.json").read()
|
||||
)
|
||||
print(
|
||||
"[migration] "
|
||||
+ str(len(storage["shouts"]["by_slugs"].keys()))
|
||||
+ " exported articles "
|
||||
)
|
||||
for slug in storage["shouts"]["by_slugs"].keys():
|
||||
export_slug(slug, storage)
|
||||
|
||||
def export_json(export_articles = {}, export_authors = {}, export_topics = {}, export_comments = {}):
|
||||
open(EXPORT_DEST + 'authors.json', 'w').write(json.dumps(export_authors,
|
||||
cls=DateTimeEncoder,
|
||||
indent=4,
|
||||
sort_keys=True,
|
||||
ensure_ascii=False))
|
||||
print('[migration] ' + str(len(export_authors.items())) + ' authors exported')
|
||||
open(EXPORT_DEST + 'topics.json', 'w').write(json.dumps(export_topics,
|
||||
cls=DateTimeEncoder,
|
||||
indent=4,
|
||||
sort_keys=True,
|
||||
ensure_ascii=False))
|
||||
print('[migration] ' + str(len(export_topics.keys())) + ' topics exported')
|
||||
|
||||
open(EXPORT_DEST + 'articles.json', 'w').write(json.dumps(export_articles,
|
||||
cls=DateTimeEncoder,
|
||||
indent=4,
|
||||
sort_keys=True,
|
||||
ensure_ascii=False))
|
||||
print('[migration] ' + str(len(export_articles.items())) + ' articles exported')
|
||||
open(EXPORT_DEST + 'comments.json', 'w').write(json.dumps(export_comments,
|
||||
cls=DateTimeEncoder,
|
||||
indent=4,
|
||||
sort_keys=True,
|
||||
ensure_ascii=False))
|
||||
print('[migration] ' + str(len(export_comments.items())) + ' exported articles with comments')
|
||||
|
||||
def export_json(
|
||||
export_articles={}, export_authors={}, export_topics={}, export_comments={}
|
||||
):
|
||||
open(EXPORT_DEST + "authors.json", "w").write(
|
||||
json.dumps(
|
||||
export_authors,
|
||||
cls=DateTimeEncoder,
|
||||
indent=4,
|
||||
sort_keys=True,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
print("[migration] " + str(len(export_authors.items())) + " authors exported")
|
||||
open(EXPORT_DEST + "topics.json", "w").write(
|
||||
json.dumps(
|
||||
export_topics,
|
||||
cls=DateTimeEncoder,
|
||||
indent=4,
|
||||
sort_keys=True,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
print("[migration] " + str(len(export_topics.keys())) + " topics exported")
|
||||
|
||||
open(EXPORT_DEST + "articles.json", "w").write(
|
||||
json.dumps(
|
||||
export_articles,
|
||||
cls=DateTimeEncoder,
|
||||
indent=4,
|
||||
sort_keys=True,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
print("[migration] " + str(len(export_articles.items())) + " articles exported")
|
||||
open(EXPORT_DEST + "comments.json", "w").write(
|
||||
json.dumps(
|
||||
export_comments,
|
||||
cls=DateTimeEncoder,
|
||||
indent=4,
|
||||
sort_keys=True,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
print(
|
||||
"[migration] "
|
||||
+ str(len(export_comments.items()))
|
||||
+ " exported articles with comments"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user