discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

migrate tags for shout

Browse Source
This commit is contained in:
knst-kotov 2021-12-15 18:28:54 +03:00
parent e2c8ceedbf
commit 46b0be08cc
2 changed files with 19 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
migrate.py
View File

@ -74,7 +74,7 @@ def users(users_by_oid, users_by_slug, users_data):
print(users_by_oid)
def topics(export_topics, topics_by_slug, topics_by_cat, topics_by_tag, cats_data, tags_data):
def topics(export_topics, topics_by_slug, topics_by_oid, cats_data, tags_data):
''' topics from categories and tags '''
# limiting
limit = len(cats_data) + len(tags_data)
@ -86,28 +86,17 @@ def topics(export_topics, topics_by_slug, topics_by_cat, topics_by_tag, cats_dat
# cat["createdBy"] = user_id_map[old_id]
try: topic = migrateCategory(cat)
except Exception as e: raise e
topics_by_cat[topic['cat_id']] = topic
topics_by_oid[topic['cat_id']] = topic
topics_by_slug[topic['slug']] = topic
counter += 1
for tag in tags_data:
old_id = tag["createdBy"]
tag["createdBy"] = user_id_map.get(old_id, 0)
topic = migrateTag(tag)
topics_by_tag[topic['tag_id']] = topic
topics_by_oid[topic['tag_id']] = topic
if not topics_by_slug.get(topic['slug']): topics_by_slug[topic['slug']] = topic
counter += 1
export_topics = dict(topics_by_slug.items()) # sorted(topics_by_slug.items(), key=lambda item: str(item[1]['createdAt']))) # NOTE: sorting does not work :)
open('migration/data/topics.slug.json','w').write(json.dumps(topics_by_slug,
cls=DateTimeEncoder,
indent=4,
sort_keys=True,
ensure_ascii=False))
open('migration/data/topics.cat_id.json','w').write(json.dumps(topics_by_cat,
cls=DateTimeEncoder,
indent=4,
sort_keys=True,
ensure_ascii=False))
def shouts(content_data, shouts_by_slug, shouts_by_oid):
''' migrating content items one by one '''
@ -125,7 +114,7 @@ def shouts(content_data, shouts_by_slug, shouts_by_oid):
for entry in content_data[:limit]:
try:
shout = migrateShout(entry, users_by_oid, topics_by_cat)
shout = migrateShout(entry, users_by_oid, topics_by_oid)
author = shout['authors'][0]
shout['authors'] = [ author.id, ]
shouts_by_slug[shout['slug']] = shout
@ -265,8 +254,7 @@ if __name__ == '__main__':
cats_data = json.loads(open('migration/data/content_item_categories.json').read())
print(str(len(cats_data)) + ' cats loaded')
topics_by_cat = {}
topics_by_tag = {}
topics_by_oid = {}
topics_by_slug = {}
content_data = json.loads(open('migration/data/content_items.json').read())
@ -296,7 +284,7 @@ if __name__ == '__main__':
if cmd == "users":
users(users_by_oid, users_by_slug, users_data)
elif cmd == "topics":
topics(export_topics, topics_by_slug, topics_by_cat, topics_by_tag, cats_data, tags_data)
topics(export_topics, topics_by_slug, topics_by_oid, cats_data, tags_data)
elif cmd == "shouts":
shouts(content_data, shouts_by_slug, shouts_by_oid) # NOTE: listens limit
elif cmd == "comments":
@ -305,7 +293,7 @@ if __name__ == '__main__':
export_shouts(shouts_by_slug, export_articles, export_authors, content_dict)
elif cmd == "all":
users(users_by_oid, users_by_slug, users_data)
topics(export_topics, topics_by_slug, topics_by_cat, topics_by_tag, cats_data, tags_data)
topics(export_topics, topics_by_slug, topics_by_oid, cats_data, tags_data)
shouts(content_data, shouts_by_slug, shouts_by_oid)
comments(comments_data)
elif cmd == 'slug':

18
migration/tables/content_items.py
View File

@ -89,10 +89,13 @@ def migrate(entry, users_by_oid, topics_by_oid):
print('NO SLUG ERROR')
# print(entry)
raise Exception
try:
r['topics'].append(topics_by_oid[entry['category']])
except Exception:
print("invalid category %s" % (entry['category']))
topic_oids = set(entry['category'])
topic_oids.update(entry.get("tags", []))
for oid in topic_oids:
if oid in topics_by_oid:
r['topics'].append(topics_by_oid[oid])
if entry.get('image') is not None:
r['cover'] = entry['image']['url']
if entry.get('thumborId') is not None:
@ -222,8 +225,11 @@ def migrate(entry, users_by_oid, topics_by_oid):
# shout topics
shout_dict['topics'] = []
for topic in r['topics']:
ShoutTopic.create(**{ 'shout': s.slug, 'topic': topic['slug'] })
shout_dict['topics'].append(topic['slug'])
try:
ShoutTopic.create(**{ 'shout': s.slug, 'topic': topic['slug'] })
shout_dict['topics'].append(topic['slug'])
except sqlalchemy.exc.IntegrityError:
pass
views = entry.get('views', 1)
ShoutViewByDay.create(