diff --git a/Pipfile b/Pipfile
index cb2e780c..5810d5c1 100644
--- a/Pipfile
+++ b/Pipfile
@@ -19,6 +19,7 @@ Authlib = "*"
bson = "*"
python-frontmatter = "*"
bs4 = "*"
+transliterate = "*"
[dev-packages]
diff --git a/migrate.py b/migrate.py
index c9135010..92354091 100644
--- a/migrate.py
+++ b/migrate.py
@@ -48,22 +48,31 @@ def shouts(limit):
data = json.loads(open('migration/data/content_items.json').read())
newdata = {}
print(str(len(data)) + ' entries was loaded. now migrating...')
+ errored = []
+
for entry in data:
- oid = entry['_id']
- newdata[oid] = migrateShout(entry)
- counter += 1
- author = newdata[oid]['authors'][0]['slug']
- if author == 'discours':
- discoursAuthor += 1
- line = str(counter) + ': ' + newdata[oid]['slug'] + " @" + author
- print(line)
- open('./shouts.id.log','a').write(line + '\n')
- if counter > limit:
- break
+ try:
+ oid = entry['_id']
+ newdata[oid] = migrateShout(entry)
+ counter += 1
+
+ author = newdata[oid]['authors'][0]['slug']
+ if author == 'discours':
+ discoursAuthor += 1
+ line = str(counter) + ': ' + newdata[oid]['slug'] + " @" + str(author)
+ print(line)
+ open('./shouts.id.log','a').write(line + '\n')
+ if counter > limit:
+ break
+ except Exception:
+ print(entry['_id'])
+ errored.append(entry)
+ raise Exception
open('migration/data/shouts.dict.json','w').write( json.dumps(newdata, cls=DateTimeEncoder) )
print(str(counter) + ' shouts were migrated')
- print(str(discoursAuthor) + ' from them by uknown users')
+ print(str(discoursAuthor) + ' from them by @discours')
+ print(str(len(errored)) + ' shouts without authors')
if __name__ == '__main__':
import sys
diff --git a/migration/html2text.py b/migration/html2text.py
index 4bdb8c77..88253d93 100644
--- a/migration/html2text.py
+++ b/migration/html2text.py
@@ -18,18 +18,10 @@ def has_key(x, y):
if hasattr(x, 'has_key'): return x.has_key(y)
else: return y in x
-try:
- import htmlentitydefs
- import urlparse
- import HTMLParser
-except ImportError: #Python3
- import html.entities as htmlentitydefs
- import urllib.parse as urlparse
- import html.parser as HTMLParser
-try: #Python3
- import urllib.request as urllib
-except:
- import urllib
+import html.entities as htmlentitydefs
+import urllib.parse as urlparse
+import html.parser as HTMLParser
+import urllib.request as urllib
import optparse, re, sys, codecs, types
try: from textwrap import wrap
@@ -45,11 +37,11 @@ ESCAPE_SNOB = 0
LINKS_EACH_PARAGRAPH = 0
# Wrap long lines at position. 0 for no wrapping. (Requires Python 2.3.)
-BODY_WIDTH = 78
+BODY_WIDTH = 0
# Don't show internal links (href="#local-anchor") -- corresponding link targets
# won't be visible in the plain text file anyway.
-SKIP_INTERNAL_LINKS = True
+SKIP_INTERNAL_LINKS = False
# Use inline, rather than reference, formatting for images and links
INLINE_LINKS = True
diff --git a/migration/tables/comments.py b/migration/tables/comments.py
index 4d1b6044..a5f63ac6 100644
--- a/migration/tables/comments.py
+++ b/migration/tables/comments.py
@@ -1,4 +1,4 @@
-# from html2md import Converter
+from html2text import html2text
import datetime
# markdown = Converter()
@@ -8,7 +8,7 @@ def migrate(entry):
# is comment
type Shout {
org: String!
- slug: String!
+ slug: String
author: Int!
body: String!
createdAt: DateTime!
@@ -28,7 +28,7 @@ def migrate(entry):
'''
# TODO: implement comments migration
return {
- 'org': 'discours.io',
+ 'org_id': 0,
'slug': entry['slug'],
'createdAt': entry['createdAt'],
'body': html2text(entry['body']),
diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py
index fee88d56..25c73165 100644
--- a/migration/tables/content_items.py
+++ b/migration/tables/content_items.py
@@ -1,16 +1,25 @@
-# from migration.html2md import Converter
from dateutil.parser import parse
+from orm import User
from os.path import abspath
import frontmatter
import json
from orm import Shout
from bs4 import BeautifulSoup
from migration.html2text import html2text
+from transliterate import translit
+from datetime import datetime
+from sqlalchemy.exc import IntegrityError
+from orm.base import local_session
users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read())
-users_dict['0'] = {'id': 9999999, 'slug': 'discours', 'viewname': 'Дискурс' }
+users_dict['0'] = {
+ 'id': 9999999,
+ 'slug': 'discours',
+ 'viewname': 'Дискурс',
+ 'userpic': 'https://discours.io/images/logo-mini.svg'
+ }
-# markdown = Converter()
+ts = datetime.now()
type2layout = {
'Article': 'article',
@@ -20,7 +29,7 @@ type2layout = {
'Image': 'image'
}
-def migrate(entry, data=users_dict):
+def migrate(entry, limit=3626, start=0):
'''
type Shout {
org_id: Int!
@@ -45,21 +54,15 @@ def migrate(entry, data=users_dict):
views: Int
}
'''
- try:
- author = data[entry['createdBy']]
- except KeyError:
- author = data['0']
-
- # print(author)
r = {
'org_id': 0,
'layout': type2layout[entry['type']],
'title': entry['title'],
- 'authors': [ { 'slug': author['slug'], 'name': author['viewname'], 'pic': author.get('userpic', '') }, ],
+ 'authors': [],
'topics': [],
- 'published': entry['published'],
- 'views': entry['views'],
- 'rating': entry['rating'],
+ 'published': entry.get('published', False),
+ 'views': entry.get('views', 0),
+ 'rating': entry.get('rating', 0),
'ratings': []
}
r['slug'] = entry.get('slug', '')
@@ -76,12 +79,8 @@ def migrate(entry, data=users_dict):
r['cover'] = entry['image']['url']
if entry.get('thumborId') is not None:
r['cover'] = 'https://assets.discours.io/unsafe/1600x/' + entry['thumborId']
- if entry.get('publishedAt') is not None:
- r['publishedAt'] = entry['publishedAt']
- if entry.get('createdAt') is not None:
- r['createdAt'] = entry['createdAt']
if entry.get('updatedAt') is not None:
- r['updatedAt'] = entry['updatedAt']
+ r['updatedAt'] = parse(entry['updatedAt'])
if entry.get('type') == 'Literature':
media = entry.get('media', '')
# print(media[0]['literatureBody'])
@@ -91,10 +90,9 @@ def migrate(entry, data=users_dict):
print('EMPTY BODY!')
else:
# body_html = str(BeautifulSoup(body_orig, features="html.parser"))
- #markdown.feed(body_html)
- body = html2text(body_orig).replace('****', '**')
- r['body'] = body
- # r['body2'] = markdown.md_file
+ # body = html2text(body_orig).replace('****', '**')
+ r['old_id'] = entry.get('_id')
+ r['body'] = body_orig
else:
print(r['slug'] + ': literature has no media')
elif entry.get('type') == 'Video':
@@ -105,7 +103,7 @@ def migrate(entry, data=users_dict):
if videoUrl == '#':
videoUrl = 'https://vimeo.com/' + vm if vm else '#'
if videoUrl == '#':
- print(m)
+ print(entry.get('media', 'NO MEDIA!'))
# raise Exception
r['body'] = '
', '') + else: + ext = 'md' + content = dumped + open('migration/content/' + entry['type'].lower() + '/' + r['slug'] + '.' + ext, 'w').write(content) + + try: + shout_dict = r.copy() + shout_dict['authors'] = [user, ] + if entry.get('createdAt') is not None: + shout_dict['createdAt'] = parse(entry.get('createdAt')) + else: + shout_dict['createdAt'] = ts + if entry.get('published'): + if entry.get('publishedAt') is not None: + shout_dict['publishedAt'] = parse(entry.get('publishedAt')) + else: + shout_dict['publishedAt'] = ts + s = Shout.create(**shout_dict) + r['id'] = s.id + except: + r['body'] = 'body moved' + print(r) + # print(s) + raise Exception return r diff --git a/migration/tables/users.py b/migration/tables/users.py index 4dfa94b9..56b608d3 100644 --- a/migration/tables/users.py +++ b/migration/tables/users.py @@ -6,13 +6,7 @@ from migration.html2text import html2text # markdown = Converter() counter = 0 - -def add(data): - data.emailConfirmed = False - user = User.create(**data) - return user - -def migrate(entry): +def migrate(entry, limit=668): ''' type User { diff --git a/orm/like.py b/orm/like.py index cd077985..91996087 100644 --- a/orm/like.py +++ b/orm/like.py @@ -11,7 +11,7 @@ class Like(Base): id: int = None user_id: str = Column(ForeignKey("user.id"), comment="Author", primary_key = True) - shout: str = Column(String, ForeignKey("shout.slug"), comment="Liked shout slug", primary_key = True) + shout_id: int = Column(Integer, ForeignKey("shout.id"), comment="Liked shout id", primary_key = True) value: int = Column(Integer, nullable=False, comment="Value") # TODO: add resolvers, debug, etc. diff --git a/orm/proposal.py b/orm/proposal.py index 73232791..d84fee0a 100644 --- a/orm/proposal.py +++ b/orm/proposal.py @@ -10,9 +10,9 @@ class Proposal(Base): __tablename__ = 'proposal' author_id: int = Column(Integer, ForeignKey("user.id"), nullable=False, comment="Author") + shout_id: int = Column(Integer, ForeignKey("shout.id"), nullable=False, comment="Shout") body: str = Column(String, nullable=False, comment="Body") createdAt: str = Column(datetime, nullable=False, comment="Created at") - shout: str = Column(String, ForeignKey("shout.slug"), nullable=False, comment="Updated at") range: str = Column(String, nullable=True, comment="Range in format