from dateutil.parser import parse as date_parse import sqlalchemy from orm.shout import Shout, ShoutTopic, User from services.stat.reacted import ReactedStorage from services.stat.viewed import ViewedByDay from transliterate import translit from datetime import datetime from base.orm import local_session from migration.extract import prepare_html_body from orm.community import Community from orm.reaction import Reaction, ReactionKind OLD_DATE = "2016-03-05 22:22:00.350000" ts = datetime.now() type2layout = { "Article": "article", "Literature": "prose", "Music": "music", "Video": "video", "Image": "image", } def get_shout_slug(entry): slug = entry.get("slug", "") if not slug: for friend in entry.get("friendlySlugs", []): slug = friend.get("slug", "") if slug: break return slug async def migrate(entry, storage): # init, set title and layout r = { "layout": type2layout[entry["type"]], "title": entry["title"], "community": Community.default_community.id, "authors": [], "topics": set([]), # 'rating': 0, # 'ratings': [], "createdAt": [], } topics_by_oid = storage["topics"]["by_oid"] users_by_oid = storage["users"]["by_oid"] # author oid = entry.get("createdBy", entry.get("_id", entry.get("oid"))) userdata = users_by_oid.get(oid) if not userdata: app = entry.get("application") if app: userslug = ( translit(app["name"], "ru", reversed=True) .replace(" ", "-") .replace("'", "") .replace(".", "-") .lower() ) userdata = { "username": app["email"], "email": app["email"], "name": app["name"], "bio": app.get("bio", ""), "emailConfirmed": False, "slug": userslug, "createdAt": ts, "wasOnlineAt": ts, } else: userdata = User.default_user.dict() assert userdata, "no user found for %s from " % [oid, len(users_by_oid.keys())] r["authors"] = [ userdata, ] # slug slug = get_shout_slug(entry) if slug: r["slug"] = slug else: raise Exception # cover c = "" if entry.get("thumborId"): c = "https://assets.discours.io/unsafe/1600x/" + entry["thumborId"] else: c = entry.get("image", {}).get("url") if not c or "cloudinary" in c: c = "" r["cover"] = c # timestamps r["createdAt"] = date_parse(entry.get("createdAt", OLD_DATE)) r["updatedAt"] = date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts if entry.get("published"): r["publishedAt"] = date_parse(entry.get("publishedAt", OLD_DATE)) if "deletedAt" in entry: r["deletedAt"] = date_parse(entry["deletedAt"]) # topics category = entry["category"] mainTopic = topics_by_oid.get(category) if mainTopic: r["mainTopic"] = storage["replacements"].get( mainTopic["slug"], mainTopic["slug"] ) topic_oids = [ category, ] topic_oids.extend(entry.get("tags", [])) for oid in topic_oids: if oid in storage["topics"]["by_oid"]: r["topics"].add(storage["topics"]["by_oid"][oid]["slug"]) else: print("[migration] unknown old topic id: " + oid) r["topics"] = list(r["topics"]) entry["topics"] = r["topics"] entry["cover"] = r["cover"] entry["authors"] = r["authors"] # body r["body"] = prepare_html_body(entry) # save shout to db s = object() shout_dict = r.copy() user = None del shout_dict[ "topics" ] # NOTE: AttributeError: 'str' object has no attribute '_sa_instance_state' # del shout_dict['rating'] # NOTE: TypeError: 'rating' is an invalid keyword argument for Shout # del shout_dict['ratings'] email = userdata.get("email") slug = userdata.get("slug") if not slug: raise Exception with local_session() as session: # c = session.query(Community).all().pop() if email: user = session.query(User).filter(User.email == email).first() if not user and slug: user = session.query(User).filter(User.slug == slug).first() if not user and userdata: try: userdata["slug"] = userdata["slug"].lower().strip().replace(" ", "-") user = User.create(**userdata) except sqlalchemy.exc.IntegrityError: print("[migration] user error: " + userdata) userdata["id"] = user.id userdata["createdAt"] = user.createdAt storage["users"]["by_slug"][userdata["slug"]] = userdata storage["users"]["by_oid"][entry["_id"]] = userdata assert user, "could not get a user" shout_dict["authors"] = [ user, ] try: s = Shout.create(**shout_dict) except sqlalchemy.exc.IntegrityError as e: with local_session() as session: s = session.query(Shout).filter(Shout.slug == shout_dict["slug"]).first() bump = False if s: for key in shout_dict: if key in s.__dict__: if s.__dict__[key] != shout_dict[key]: print( "[migration] shout already exists, but differs in %s" % key ) bump = True else: print("[migration] shout already exists, but lacks %s" % key) bump = True if bump: s.update(shout_dict) else: print("[migration] something went wrong with shout: \n%r" % shout_dict) raise e session.commit() except Exception as e: print(e) print(s) raise Exception # shout topics aftermath shout_dict["topics"] = [] for tpc in r["topics"]: oldslug = tpc newslug = storage["replacements"].get(oldslug, oldslug) if newslug: with local_session() as session: shout_topic_old = ( session.query(ShoutTopic) .filter(ShoutTopic.shout == shout_dict["slug"]) .filter(ShoutTopic.topic == oldslug) .first() ) if shout_topic_old: shout_topic_old.update({"slug": newslug}) else: shout_topic_new = ( session.query(ShoutTopic) .filter(ShoutTopic.shout == shout_dict["slug"]) .filter(ShoutTopic.topic == newslug) .first() ) if not shout_topic_new: try: ShoutTopic.create( **{"shout": shout_dict["slug"], "topic": newslug} ) except: print("[migration] shout topic error: " + newslug) session.commit() if newslug not in shout_dict["topics"]: shout_dict["topics"].append(newslug) else: print("[migration] ignored topic slug: \n%r" % tpc["slug"]) # raise Exception # content_item ratings to reactions try: for content_rating in entry.get("ratings", []): with local_session() as session: rater = ( session.query(User) .filter(User.oid == content_rating["createdBy"]) .first() ) reactedBy = ( rater if rater else session.query(User).filter(User.slug == "noname").first() ) if rater: reaction_dict = { "kind": ReactionKind.LIKE if content_rating["value"] > 0 else ReactionKind.DISLIKE, "createdBy": reactedBy.slug, "shout": shout_dict["slug"], } cts = content_rating.get("createdAt") if cts: reaction_dict["createdAt"] = date_parse(cts) reaction = ( session.query(Reaction) .filter(Reaction.shout == reaction_dict["shout"]) .filter(Reaction.createdBy == reaction_dict["createdBy"]) .filter(Reaction.kind == reaction_dict["kind"]) .first() ) if reaction: reaction_dict["kind"] = ( ReactionKind.AGREE if content_rating["value"] > 0 else ReactionKind.DISAGREE, ) reaction.update(reaction_dict) else: day = (reaction_dict.get("createdAt") or ts).replace( hour=0, minute=0, second=0, microsecond=0 ) rea = Reaction.create(**reaction_dict) await ReactedStorage.increment(rea) # shout_dict['ratings'].append(reaction_dict) except: print("[migration] content_item.ratings error: \n%r" % content_rating) raise Exception # shout views ViewedByDay.create(shout=shout_dict["slug"], value=entry.get("views", 1)) # del shout_dict['ratings'] shout_dict["oid"] = entry.get("_id") storage["shouts"]["by_oid"][entry["_id"]] = shout_dict storage["shouts"]["by_slug"][slug] = shout_dict return shout_dict