diff --git a/migration/__init__.py b/migration/__init__.py index eaded31b..aadbd8de 100644 --- a/migration/__init__.py +++ b/migration/__init__.py @@ -96,16 +96,16 @@ async def shouts_handle(storage, args): continue # migrate - shout = await migrateShout(entry, storage) - if shout: - storage["shouts"]["by_oid"][entry["_id"]] = shout - storage["shouts"]["by_slug"][shout["slug"]] = shout + shout_dict = await migrateShout(entry, storage) + if shout_dict: + storage["shouts"]["by_oid"][entry["_id"]] = shout_dict + storage["shouts"]["by_slug"][shout_dict["slug"]] = shout_dict # shouts.topics - if not shout["topics"]: + if not shout_dict["topics"]: print("[migration] no topics!") # with author - author: str = shout["authors"][0].dict() + author = shout_dict["authors"][0] if author["slug"] == "discours": discours_author += 1 if author["slug"] == "anonymous": @@ -114,18 +114,20 @@ async def shouts_handle(storage, args): if entry.get("published"): if "mdx" in args: - export_mdx(shout) + export_mdx(shout_dict) pub_counter += 1 # print main counter counter += 1 - print('[migration] shouts_handle %d: %s @%s' % ((counter + 1), shout["slug"], author["slug"])) + print('[migration] shouts_handle %d: %s @%s' % ( + (counter + 1), shout_dict["slug"], author["slug"] + )) - b = bs4.BeautifulSoup(shout["body"], "html.parser") - texts = [shout["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")] + b = bs4.BeautifulSoup(shout_dict["body"], "html.parser") + texts = [shout_dict["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")] texts = texts + b.findAll(text=True) topics_dataset_bodies.append(" ".join([x.strip().lower() for x in texts])) - topics_dataset_tlist.append(shout["topics"]) + topics_dataset_tlist.append(shout_dict["topics"]) else: ignored += 1 @@ -133,9 +135,7 @@ async def shouts_handle(storage, args): # ', fmt='%s') print("[migration] " + str(counter) + " content items were migrated") - print("[migration] " + str(ignored) + " content items were ignored") print("[migration] " + str(pub_counter) + " have been published") - print("[migration] " + str(discours_author) + " authored by @discours") print("[migration] " + str(anonymous_author) + " authored by @anonymous") diff --git a/migration/tables/comments.py b/migration/tables/comments.py index 138b3327..4fde9569 100644 --- a/migration/tables/comments.py +++ b/migration/tables/comments.py @@ -5,13 +5,93 @@ from dateutil.parser import parse as date_parse from base.orm import local_session from migration.html2text import html2text from orm.reaction import Reaction, ReactionKind -from orm.shout import ShoutReactionsFollower, Shout -from orm.topic import TopicFollower, Topic +from orm.shout import ShoutReactionsFollower +from orm.topic import TopicFollower from orm.user import User +from orm.shout import Shout +# from services.stat.reacted import ReactedStorage ts = datetime.now(tz=timezone.utc) +def auto_followers(session, topics, reaction_dict): + # creating shout's reactions following for reaction author + following1 = session.query( + ShoutReactionsFollower + ).where( + ShoutReactionsFollower.follower == reaction_dict["createdBy"] + ).filter( + ShoutReactionsFollower.shout == reaction_dict["shout"] + ).first() + if not following1: + following1 = ShoutReactionsFollower.create( + follower=reaction_dict["createdBy"], + shout=reaction_dict["shout"], + auto=True + ) + session.add(following1) + # creating topics followings for reaction author + for t in topics: + tf = session.query( + TopicFollower + ).where( + TopicFollower.follower == reaction_dict["createdBy"] + ).filter( + TopicFollower.topic == t['id'] + ).first() + if not tf: + topic_following = TopicFollower.create( + follower=reaction_dict["createdBy"], + topic=t['id'], + auto=True + ) + session.add(topic_following) + + +def migrate_ratings(session, entry, reaction_dict): + for comment_rating_old in entry.get("ratings", []): + rater = ( + session.query(User) + .filter(User.oid == comment_rating_old["createdBy"]) + .first() + ) + re_reaction_dict = { + "shout": reaction_dict["shout"], + "replyTo": reaction_dict["id"], + "kind": ReactionKind.LIKE + if comment_rating_old["value"] > 0 + else ReactionKind.DISLIKE, + "createdBy": rater.id if rater else 1, + } + cts = comment_rating_old.get("createdAt") + if cts: + re_reaction_dict["createdAt"] = date_parse(cts) + try: + # creating reaction from old rating + rr = Reaction.create(**re_reaction_dict) + following2 = session.query( + ShoutReactionsFollower + ).where( + ShoutReactionsFollower.follower == re_reaction_dict['createdBy'] + ).filter( + ShoutReactionsFollower.shout == rr.shout + ).first() + if not following2: + following2 = ShoutReactionsFollower.create( + follower=re_reaction_dict['createdBy'], + shout=rr.shout, + auto=True + ) + session.add(following2) + session.add(rr) + # await ReactedStorage.react(rr) + + except Exception as e: + print("[migration] comment rating error: %r" % re_reaction_dict) + raise e + session.commit() + + async def migrate(entry, storage): """ { @@ -29,9 +109,7 @@ async def migrate(entry, storage): "updatedAt": "2020-05-27 19:22:57.091000+00:00", "updatedBy": "0" } - -> - type Reaction { id: Int! shout: Shout! @@ -49,10 +127,9 @@ async def migrate(entry, storage): old_thread: String } """ + old_ts = entry.get("createdAt") reaction_dict = { - "createdAt": ( - ts if not entry.get("createdAt") else date_parse(entry.get("createdAt")) - ), + "createdAt": (ts if not old_ts else date_parse(old_ts)), "body": html2text(entry.get("body", "")), "oid": entry["_id"], } @@ -65,140 +142,71 @@ async def migrate(entry, storage): raise Exception return else: + stage = "started" + reaction = None with local_session() as session: author = session.query(User).filter(User.oid == entry["createdBy"]).first() - shout_dict = storage["shouts"]["by_oid"][shout_oid] - if shout_dict: - shout = session.query( - Shout - ).where(Shout.slug == shout_dict["slug"]).one() - - reaction_dict["shout"] = shout.id - reaction_dict["createdBy"] = author.id if author else 1 - reaction_dict["kind"] = ReactionKind.COMMENT - - # creating reaction from old comment - reaction = Reaction.create(**reaction_dict) - session.add(reaction) - - # creating shout's reactions following for reaction author - following1 = session.query( - ShoutReactionsFollower - ).join( - User - ).join( - Shout - ).where( - User.id == reaction_dict["createdBy"] - ).filter( - ShoutReactionsFollower.shout == reaction.shout - ).first() - - if not following1: - following1 = ShoutReactionsFollower.create( - follower=reaction_dict["createdBy"], - shout=reaction.shout, - auto=True - ) - session.add(following1) - - # creating topics followings for reaction author - for t in shout_dict["topics"]: - tf = session.query( - TopicFollower - ).join( - Topic - ).where( - TopicFollower.follower == reaction_dict["createdBy"] - ).filter( - Topic.slug == t - ).first() - - if not tf: - topic = session.query( - Topic - ).where(Topic.slug == t).one() - - topic_following = TopicFollower.create( - follower=reaction_dict["createdBy"], - topic=topic.id, - auto=True - ) - session.add(topic_following) - - reaction_dict["id"] = reaction.id - for comment_rating_old in entry.get("ratings", []): - rater = ( - session.query(User) - .filter(User.oid == comment_rating_old["createdBy"]) - .first() - ) - re_reaction_dict = { - "shout": reaction_dict["shout"], - "replyTo": reaction.id, - "kind": ReactionKind.LIKE - if comment_rating_old["value"] > 0 - else ReactionKind.DISLIKE, - "createdBy": rater.id if rater else 1, - } - cts = comment_rating_old.get("createdAt") - if cts: - re_reaction_dict["createdAt"] = date_parse(cts) - try: - # creating reaction from old rating - rr = Reaction.create(**re_reaction_dict) - following2 = session.query( - ShoutReactionsFollower - ).where( - ShoutReactionsFollower.follower == re_reaction_dict['createdBy'] - ).filter( - ShoutReactionsFollower.shout == rr.shout - ).first() - if not following2: - following2 = ShoutReactionsFollower.create( - follower=re_reaction_dict['createdBy'], - shout=rr.shout, - auto=True - ) - session.add(following2) - session.add(rr) - - except Exception as e: - print("[migration] comment rating error: %r" % re_reaction_dict) - raise e - session.commit() + old_shout = storage["shouts"]["by_oid"].get(shout_oid) + if not old_shout: + raise Exception("no old shout in storage") else: - print( - "[migration] error: cannot find shout for comment %r" - % reaction_dict - ) - return reaction + stage = "author and old id found" + try: + shout = session.query( + Shout + ).where(Shout.slug == old_shout["slug"]).one() + if shout: + reaction_dict["shout"] = shout.id + reaction_dict["createdBy"] = author.id if author else 1 + reaction_dict["kind"] = ReactionKind.COMMENT + + # creating reaction from old comment + reaction = Reaction.create(**reaction_dict) + session.add(reaction) + # session.commit() + stage = "new reaction commited" + reaction_dict = reaction.dict() + topics = [t.dict() for t in shout.topics] + auto_followers(session, topics, reaction_dict) + + migrate_ratings(session, entry, reaction_dict) + + return reaction + except Exception as e: + print(e) + print(reaction) + raise Exception(stage) + return -def migrate_2stage(rr, old_new_id): - reply_oid = rr.get("replyTo") - if not reply_oid: - return - - new_id = old_new_id.get(rr.get("oid")) - if not new_id: - return - - with local_session() as session: - comment = session.query(Reaction).filter(Reaction.id == new_id).first() - comment.replyTo = old_new_id.get(reply_oid) - session.add(comment) - - srf = session.query(ShoutReactionsFollower).where( - ShoutReactionsFollower.shout == comment.shout - ).filter( - ShoutReactionsFollower.follower == comment.createdBy - ).first() - - if not srf: - srf = ShoutReactionsFollower.create(shout=comment.shout, follower=comment.createdBy, auto=True) - session.add(srf) - - session.commit() - if not rr["body"]: - raise Exception(rr) +def migrate_2stage(old_comment, idmap): + if old_comment.get('body'): + new_id = idmap.get(old_comment.get('oid')) + if new_id: + new_replyto_id = None + old_replyto_id = old_comment.get("replyTo") + if old_replyto_id: + new_replyto_id = int(idmap.get(old_replyto_id, "0")) + with local_session() as session: + comment = session.query(Reaction).where(Reaction.id == new_id).first() + try: + if new_replyto_id: + new_reply = session.query(Reaction).where(Reaction.id == new_replyto_id).first() + if not new_reply: + print(new_replyto_id) + raise Exception("cannot find reply by id!") + comment.replyTo = new_reply.id + session.add(comment) + srf = session.query(ShoutReactionsFollower).where( + ShoutReactionsFollower.shout == comment.shout + ).filter( + ShoutReactionsFollower.follower == comment.createdBy + ).first() + if not srf: + srf = ShoutReactionsFollower.create( + shout=comment.shout, follower=comment.createdBy, auto=True + ) + session.add(srf) + session.commit() + except Exception: + raise Exception("cannot find a comment by oldid") diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index 628a4a1b..158a2253 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -9,7 +9,9 @@ from orm.reaction import Reaction, ReactionKind from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower from orm.user import User from orm.topic import TopicFollower, Topic +# from services.stat.reacted import ReactedStorage from services.stat.viewed import ViewedStorage +import re OLD_DATE = "2016-03-05 22:22:00.350000" ts = datetime.now(tz=timezone.utc) @@ -21,6 +23,8 @@ type2layout = { "Image": "image", } +anondict = {"slug": "anonymous", "id": 1, "name": "Аноним"} + def get_shout_slug(entry): slug = entry.get("slug", "") @@ -29,6 +33,7 @@ def get_shout_slug(entry): slug = friend.get("slug", "") if slug: break + slug = re.sub('[^0-9a-zA-Z]+', '-', slug) return slug @@ -39,13 +44,8 @@ def create_author_from_app(app): user = session.query(User).where(User.email == app['email']).first() if not user: name = app.get('name') - slug = ( - translit(name, "ru", reversed=True) - .replace(" ", "-") - .replace("'", "") - .replace(".", "-") - .lower() - ) + slug = translit(name, "ru", reversed=True).lower() + slug = re.sub('[^0-9a-zA-Z]+', '-', slug) # check if nameslug is used user = session.query(User).where(User.slug == slug).first() # get slug from email @@ -74,122 +74,99 @@ def create_author_from_app(app): session.commit() userdata = user.dict() if not userdata: - userdata = User.default_user.dict() + userdata = User.default_user.dict() # anonymous except Exception as e: print(app) raise e return userdata -async def create_shout(shout_dict, userslug): +async def create_shout(shout_dict, user): s = Shout.create(**shout_dict) with local_session() as session: - follower = session.query(User).where(User.slug == userslug).one() - - srf = session.query( - ShoutReactionsFollower - ).join( - User - ).where( + srf = session.query(ShoutReactionsFollower).where( ShoutReactionsFollower.shout == s.id ).filter( - User.slug == userslug + ShoutReactionsFollower.follower == user.id ).first() if not srf: - srf = ShoutReactionsFollower.create(shout=s.id, follower=follower.id, auto=True) + srf = ShoutReactionsFollower.create(shout=s.id, follower=user.id, auto=True) session.add(srf) session.commit() + return s + + +def get_userdata(entry, storage): + user_oid = entry.get("createdBy", "") + userdata = None + app = entry.get("application") + if app: + userdata = create_author_from_app(app) or anondict + else: + userdata = storage["users"]["by_oid"].get(user_oid) or anondict + slug = userdata.get("slug") + slug = re.sub('[^0-9a-zA-Z]+', '-', slug) + userdata["slug"] = slug + return userdata, user_oid async def migrate(entry, storage): - # init, set title and layout + userdata, user_oid = get_userdata(entry, storage) + user = await get_user(userdata, storage, user_oid) r = { "layout": type2layout[entry["type"]], "title": entry["title"], - "authors": [], - "topics": set([]) + "authors": [userdata["slug"], ], + "slug": get_shout_slug(entry), + "cover": ( + "https://assets.discours.io/unsafe/1600x/" + + entry["thumborId"] if entry.get("thumborId") else entry.get("image", {}).get("url") + ), + "visibility": "public" if entry.get("published") else "authors", + "publishedAt": date_parse(entry.get("publishedAt")) if entry.get("published") else None, + "deletedAt": date_parse(entry.get("deletedAt")) if entry.get("deletedAt") else None, + "createdAt": date_parse(entry.get("createdAt", OLD_DATE)), + "updatedAt": date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts, + "topics": await add_topics_follower(entry, storage, user), + "body": extract_html(entry) } - # author - users_by_oid = storage["users"]["by_oid"] - user_oid = entry.get("createdBy", "") - userdata = users_by_oid.get(user_oid) - user = None - if not userdata: - app = entry.get("application") - if app: - userdata = create_author_from_app(app) - if userdata: - userslug = userdata.get('slug') - else: - userslug = "anonymous" # bad old id slug was found - r["authors"] = [userslug, ] + # main topic patch + r['mainTopic'] = r['topics'][0] - # slug - slug = get_shout_slug(entry) - if slug: - r["slug"] = slug - else: - raise Exception - - # cover - c = "" - if entry.get("thumborId"): - c = "https://assets.discours.io/unsafe/1600x/" + entry["thumborId"] - else: - c = entry.get("image", {}).get("url") - if not c or "cloudinary" in c: - c = "" - r["cover"] = c - - # timestamps - r["createdAt"] = date_parse(entry.get("createdAt", OLD_DATE)) - r["updatedAt"] = date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts - - # visibility + # published author auto-confirm if entry.get("published"): - r["publishedAt"] = date_parse(entry.get("publishedAt", OLD_DATE)) - r["visibility"] = "public" with local_session() as session: # update user.emailConfirmed if published - author = session.query(User).where(User.slug == userslug).first() + author = session.query(User).where(User.slug == userdata["slug"]).first() author.emailConfirmed = True session.add(author) session.commit() - else: - r["visibility"] = "authors" - if "deletedAt" in entry: - r["deletedAt"] = date_parse(entry["deletedAt"]) - - # topics - r['topics'] = await add_topics_follower(entry, storage, userslug) - r['mainTopic'] = r['topics'][0] - - entry["topics"] = r["topics"] - entry["cover"] = r["cover"] - - # body - r["body"] = extract_html(entry) + # media media = extract_media(entry) - if media: - r["media"] = json.dumps(media, ensure_ascii=True) + r["media"] = json.dumps(media, ensure_ascii=True) if media else None + # ----------------------------------- copy shout_dict = r.copy() # user - user = await get_user(userslug, userdata, storage, user_oid) shout_dict["authors"] = [user, ] del shout_dict["topics"] try: # save shout to db - await create_shout(shout_dict, userslug) + shout_dict["oid"] = entry.get("_id", "") + shout = await create_shout(shout_dict, user) except IntegrityError as e: - print(e) - await resolve_create_shout(shout_dict, userslug) + print('[migration] create_shout integrity error', e) + shout = await resolve_create_shout(shout_dict, userdata["slug"]) except Exception as e: raise Exception(e) + # udpate data + shout_dict = shout.dict() + shout_dict["authors"] = [user.dict(), ] + # shout topics aftermath shout_dict["topics"] = await topics_aftermath(r, storage) @@ -200,13 +177,12 @@ async def migrate(entry, storage): await ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1)) # del shout_dict['ratings'] - shout_dict["oid"] = entry.get("_id", "") storage["shouts"]["by_oid"][entry["_id"]] = shout_dict - storage["shouts"]["by_slug"][slug] = shout_dict + storage["shouts"]["by_slug"][shout_dict["slug"]] = shout_dict return shout_dict -async def add_topics_follower(entry, storage, userslug): +async def add_topics_follower(entry, storage, user): topics = set([]) category = entry.get("category") topics_by_oid = storage["topics"]["by_oid"] @@ -218,29 +194,26 @@ async def add_topics_follower(entry, storage, userslug): ttt = list(topics) # add author as TopicFollower with local_session() as session: - for tpc in topics: + for tpcslug in topics: try: - topic = session.query(Topic).where(Topic.slug == tpc).one() - follower = session.query(User).where(User.slug == userslug).one() - + tpc = session.query(Topic).where(Topic.slug == tpcslug).first() tf = session.query( TopicFollower ).where( - TopicFollower.follower == follower.id + TopicFollower.follower == user.id ).filter( - TopicFollower.topic == topic.id + TopicFollower.topic == tpc.id ).first() - if not tf: tf = TopicFollower.create( - topic=topic.id, - follower=follower.id, + topic=tpc.id, + follower=user.id, auto=True ) session.add(tf) session.commit() except IntegrityError: - print('[migration.shout] hidden by topic ' + tpc) + print('[migration.shout] hidden by topic ' + tpc.slug) # main topic maintopic = storage["replacements"].get(topics_by_oid.get(category, {}).get("slug")) if maintopic in ttt: @@ -249,23 +222,28 @@ async def add_topics_follower(entry, storage, userslug): return ttt -async def get_user(userslug, userdata, storage, oid): +async def get_user(userdata, storage, oid): user = None with local_session() as session: - if not user and userslug: - user = session.query(User).filter(User.slug == userslug).first() - if not user and userdata: + uid = userdata.get("id") + if uid: + user = session.query(User).filter(User.id == uid).first() + elif userdata: try: - userdata["slug"] = userdata["slug"].lower().strip().replace(" ", "-") + slug = userdata["slug"].lower().strip() + slug = re.sub('[^0-9a-zA-Z]+', '-', slug) + userdata["slug"] = slug user = User.create(**userdata) session.add(user) session.commit() except IntegrityError: - print("[migration] user error: " + userdata) - userdata["id"] = user.id - userdata["createdAt"] = user.createdAt - storage["users"]["by_slug"][userdata["slug"]] = userdata - storage["users"]["by_oid"][oid] = userdata + print("[migration] user creating with slug %s" % userdata["slug"]) + print("[migration] from userdata: %r" % userdata) + raise Exception("[migration] cannot create user in content_items.get_user()") + userdata["id"] = user.id + userdata["createdAt"] = user.createdAt + storage["users"]["by_slug"][userdata["slug"]] = userdata + storage["users"]["by_oid"][oid] = userdata if not user: raise Exception("could not get a user") return user @@ -303,6 +281,7 @@ async def resolve_create_shout(shout_dict, userslug): print("[migration] something went wrong with shout: \n%r" % shout_dict) raise Exception("") session.commit() + return s async def topics_aftermath(entry, storage): @@ -318,22 +297,22 @@ async def topics_aftermath(entry, storage): shout_topic_old = ( session.query(ShoutTopic) - .join(Shout) - .join(Topic) - .filter(Shout.slug == entry["slug"]) - .filter(Topic.slug == oldslug) - .first() + .join(Shout) + .join(Topic) + .filter(Shout.slug == entry["slug"]) + .filter(Topic.slug == oldslug) + .first() ) if shout_topic_old: shout_topic_old.update({"topic": new_topic.id}) else: shout_topic_new = ( session.query(ShoutTopic) - .join(Shout) - .join(Topic) - .filter(Shout.slug == entry["slug"]) - .filter(Topic.slug == newslug) - .first() + .join(Shout) + .join(Topic) + .filter(Shout.slug == entry["slug"]) + .filter(Topic.slug == newslug) + .first() ) if not shout_topic_new: try: @@ -357,45 +336,37 @@ async def content_ratings_to_reactions(entry, slug): for content_rating in entry.get("ratings", []): rater = ( session.query(User) - .filter(User.oid == content_rating["createdBy"]) - .first() + .filter(User.oid == content_rating["createdBy"]) + .first() + ) or User.default_user + shout = session.query(Shout).where(Shout.slug == slug).first() + cts = content_rating.get("createdAt") + reaction_dict = { + "createdAt": date_parse(cts) if cts else None, + "kind": ReactionKind.LIKE + if content_rating["value"] > 0 + else ReactionKind.DISLIKE, + "createdBy": rater.id, + "shout": shout.id + } + reaction = ( + session.query(Reaction) + .filter(Reaction.shout == reaction_dict["shout"]) + .filter(Reaction.createdBy == reaction_dict["createdBy"]) + .filter(Reaction.kind == reaction_dict["kind"]) + .first() ) - reactedBy = ( - rater - if rater - else session.query(User).filter(User.slug == "anonymous").first() - ) - if rater: - shout = session.query(Shout).where(Shout.slug == slug).one() - - reaction_dict = { - "kind": ReactionKind.LIKE - if content_rating["value"] > 0 - else ReactionKind.DISLIKE, - "createdBy": reactedBy.id, - "shout": shout.id, - } - cts = content_rating.get("createdAt") - if cts: - reaction_dict["createdAt"] = date_parse(cts) - reaction = ( - session.query(Reaction).filter( - Reaction.shout == reaction_dict["shout"] - ).filter( - Reaction.createdBy == reaction_dict["createdBy"] - ).filter( - Reaction.kind == reaction_dict["kind"] - ).first() - ) - if reaction: - k = ReactionKind.AGREE if content_rating["value"] > 0 else ReactionKind.DISAGREE - reaction_dict["kind"] = k - reaction.update(reaction_dict) - else: - rea = Reaction.create(**reaction_dict) - session.add(rea) - # shout_dict['ratings'].append(reaction_dict) + if reaction: + k = ReactionKind.AGREE if content_rating["value"] > 0 else ReactionKind.DISAGREE + reaction_dict["kind"] = k + reaction.update(reaction_dict) + session.add(reaction) + else: + rea = Reaction.create(**reaction_dict) + session.add(rea) + # await ReactedStorage.react(rea) + # shout_dict['ratings'].append(reaction_dict) session.commit() except Exception: - raise Exception("[migration] content_item.ratings error: \n%r" % content_rating) + print("[migration] content_item.ratings error: \n%r" % content_rating) diff --git a/migration/tables/users.py b/migration/tables/users.py index 1917ed47..cb3e87bc 100644 --- a/migration/tables/users.py +++ b/migration/tables/users.py @@ -1,7 +1,7 @@ from dateutil.parser import parse from sqlalchemy.exc import IntegrityError from bs4 import BeautifulSoup - +import re from base.orm import local_session from orm.user import AuthorFollower, User, UserRating @@ -32,9 +32,9 @@ def migrate(entry): user_dict["lastSeen"] = parse(entry["wasOnlineAt"]) if entry.get("profile"): # slug - user_dict["slug"] = ( - entry["profile"].get("path").lower().replace(" ", "-").strip() - ) + slug = entry["profile"].get("path").lower() + slug = re.sub('[^0-9a-zA-Z]+', '-', slug).strip() + user_dict["slug"] = slug bio = BeautifulSoup(entry.get("profile").get("bio") or "", features="lxml").text if bio.startswith('<'): print('[migration] bio! ' + bio) diff --git a/schema.graphql b/schema.graphql index 01918c01..5b84ae54 100644 --- a/schema.graphql +++ b/schema.graphql @@ -371,6 +371,12 @@ type User { oid: String } +type Draft { + title: String + body: String + createdBy: Int +} + type Collab { authors: [String]! invites: [String]