discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

bookmarked not count, migation fixed

Browse Source
This commit is contained in:
tonyrewin 2022-08-18 09:12:46 +03:00
parent 68fd4d9936
commit 04f0352a84
9 changed files with 114 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
migration/__init__.py
View File

@ -1,5 +1,6 @@
''' cmd managed migration '''
import csv
import asyncio
from datetime import datetime
import json
import subprocess
@ -74,7 +75,7 @@ def topics_handle(storage):
return storage
def shouts_handle(storage, args):
async def shouts_handle(storage, args):
''' migrating content items one by one '''
counter = 0
discours_author = 0
@ -89,7 +90,7 @@ def shouts_handle(storage, args):
if '-' in args and slug not in args: continue
# migrate
shout = migrateShout(entry, storage)
shout = await migrateShout(entry, storage)
storage['shouts']['by_oid'][entry['_id']] = shout
storage['shouts']['by_slug'][shout['slug']] = shout
# shouts.topics
@ -123,13 +124,13 @@ def shouts_handle(storage, args):
return storage
def comments_handle(storage):
async def comments_handle(storage):
id_map = {}
ignored_counter = 0
missed_shouts = {}
for oldcomment in storage['reactions']['data']:
if not oldcomment.get('deleted'):
reaction = migrateComment(oldcomment, storage)
reaction = await migrateComment(oldcomment, storage)
if type(reaction) == str:
missed_shouts[reaction] = oldcomment
elif type(reaction) == Reaction:
@ -166,11 +167,11 @@ def export_one(slug, storage):
export_slug(slug, storage)
def all_handle(storage, args):
async def all_handle(storage, args):
print('[migration] handle everything')
users_handle(storage)
topics_handle(storage)
shouts_handle(storage, args)
await shouts_handle(storage, args)
comments_handle(storage)
# export_email_subscriptions()
print('[migration] done!')
@ -213,26 +214,26 @@ def data_load():
content_data = []
try:
users_data = json.loads(open('migration/data/users.json').read())
print('[migration] ' + str(len(users_data)) + ' users ')
print('[migration.load] ' + str(len(users_data)) + ' users ')
tags_data = json.loads(open('migration/data/tags.json').read())
storage['topics']['tags'] = tags_data
print('[migration] ' + str(len(tags_data)) + ' tags ')
print('[migration.load] ' + str(len(tags_data)) + ' tags ')
cats_data = json.loads(
open('migration/data/content_item_categories.json').read())
storage['topics']['cats'] = cats_data
print('[migration] ' + str(len(cats_data)) + ' cats ')
print('[migration.load] ' + str(len(cats_data)) + ' cats ')
comments_data = json.loads(open('migration/data/comments.json').read())
storage['reactions']['data'] = comments_data
print('[migration] ' + str(len(comments_data)) + ' comments ')
print('[migration.load] ' + str(len(comments_data)) + ' comments ')
content_data = json.loads(open('migration/data/content_items.json').read())
storage['shouts']['data'] = content_data
print('[migration] ' + str(len(content_data)) + ' content items ')
print('[migration.load] ' + str(len(content_data)) + ' content items ')
# fill out storage
for x in users_data:
storage['users']['by_oid'][x['_id']] = x
# storage['users']['by_slug'][x['slug']] = x
# no user.slug yet
print('[migration] ' + str(len(storage['users']
print('[migration.load] ' + str(len(storage['users']
['by_oid'].keys())) + ' users by oid')
for x in tags_data:
storage['topics']['by_oid'][x['_id']] = x
@ -240,20 +241,20 @@ def data_load():
for x in cats_data:
storage['topics']['by_oid'][x['_id']] = x
storage['topics']['by_slug'][x['slug']] = x
print('[migration] ' + str(len(storage['topics']
print('[migration.load] ' + str(len(storage['topics']
['by_slug'].keys())) + ' topics by slug')
for item in content_data:
slug = get_shout_slug(item)
storage['content_items']['by_slug'][slug] = item
storage['content_items']['by_oid'][item['_id']] = item
print('[migration] ' + str(len(content_data)) + ' content items')
print('[migration.load] ' + str(len(content_data)) + ' content items')
for x in comments_data:
storage['reactions']['by_oid'][x['_id']] = x
cid = x['contentItem']
storage['reactions']['by_content'][cid] = x
ci = storage['content_items']['by_oid'].get(cid, {})
if 'slug' in ci: storage['reactions']['by_slug'][ci['slug']] = x
print('[migration] ' + str(len(storage['reactions']
print('[migration.load] ' + str(len(storage['reactions']
['by_content'].keys())) + ' with comments')
except Exception as e: raise e
storage['users']['data'] = users_data
@ -288,29 +289,25 @@ def create_pgdump():
])
def handle_auto():
async def handle_auto():
print('[migration] no command given, auto mode')
mongo_download(os.getenv('MONGODB_URL'))
url = os.getenv('MONGODB_URL')
if url: mongo_download(url)
bson_handle()
all_handle(data_load(), sys.argv)
await all_handle(data_load(), sys.argv)
create_pgdump()
def migrate():
async def main():
if len(sys.argv) > 1:
cmd=sys.argv[1]
if type(cmd) == str: print('[migration] command: ' + cmd)
if cmd == 'mongodb':
mongo_download(sys.argv[2])
elif cmd == 'bson':
bson_handle()
await handle_auto()
else:
storage=data_load()
if cmd == '-': export_one(sys.argv[2], storage)
else: all_handle(storage, sys.argv)
elif len(sys.argv) == 1:
handle_auto()
else:
print('[migration] usage: python migrate.py <command>')
print('[migration] commands: mongodb, bson, all, all mdx, - <slug>')
print('[migration] usage: python server.py migrate')
if __name__ == '__main__': migrate()
def migrate():
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
if __name__ == '__main__':
migrate()

6
migration/tables/comments.py
View File

@ -8,7 +8,7 @@ from services.stat.reacted import ReactedStorage
ts = datetime.now()
def migrate(entry, storage):
async def migrate(entry, storage):
'''
{
"_id": "hdtwS8fSyFLxXCgSC",
@ -72,7 +72,7 @@ def migrate(entry, storage):
# creating reaction from old comment
day = (reaction_dict.get('createdAt') or ts).replace(hour=0, minute=0, second=0, microsecond=0)
reaction = Reaction.create(**reaction_dict)
ReactedStorage.increment(reaction)
await ReactedStorage.increment(reaction)
reaction_dict['id'] = reaction.id
for comment_rating_old in entry.get('ratings',[]):
@ -89,7 +89,7 @@ def migrate(entry, storage):
try:
# creating reaction from old rating
rr = Reaction.create(**re_reaction_dict)
ReactedStorage.increment(rr)
await ReactedStorage.increment(rr)
except Exception as e:
print('[migration] comment rating error: %r' % re_reaction_dict)

4
migration/tables/content_items.py
View File

@ -28,7 +28,7 @@ def get_shout_slug(entry):
if slug: break
return slug
def migrate(entry, storage):
async def migrate(entry, storage):
# init, set title and layout
r = {
'layout': type2layout[entry['type']],
@ -218,7 +218,7 @@ def migrate(entry, storage):
else:
day = (reaction_dict.get('createdAt') or ts).replace(hour=0, minute=0, second=0, microsecond=0)
rea = Reaction.create(**reaction_dict)
ReactedStorage.increment(rea)
await ReactedStorage.increment(rea)
# shout_dict['ratings'].append(reaction_dict)
except:
print('[migration] content_item.ratings error: \n%r' % content_rating)

3
orm/shout.py
View File

@ -67,6 +67,5 @@ class Shout(Base):
"viewed": await ViewedStorage.get_shout(self.slug),
"reacted": len(await ReactedStorage.get_shout(self.slug)),
"commented": len(await ReactedStorage.get_comments(self.slug)),
"rating": await ReactedStorage.get_rating(self.slug),
"bookmarked": len(await ReactedStorage.get_bookmarked(self.slug))
"rating": await ReactedStorage.get_rating(self.slug)
}

1
requirements.txt
View File

@ -16,3 +16,4 @@ transliterate
requests
bcrypt
websockets
bson

21
resolvers/reactions.py
View File

@ -1,4 +1,5 @@
from sqlalchemy import desc
from sqlalchemy.orm import joinedload, selectinload
from orm.reaction import Reaction
from base.orm import local_session
from orm.shout import ShoutReactionsFollower
@ -112,31 +113,31 @@ async def get_shout_reactions(_, info, slug, page, size):
filter(Reaction.shout == slug).\
limit(size).offset(offset).all()
for r in reactions:
r.createdBy = await UserStorage.get_user(r.createdBy)
r.createdBy = await UserStorage.get_user(r.createdBy or 'discours')
return reactions
@query.field("reactionsAll")
def get_all_reactions(_, info, page=1, size=10):
async def get_all_reactions(_, info, page=1, size=10):
offset = page * size
reactions = []
with local_session() as session:
# raw sql: statement = text(open('queries/reactions-all.sql', 'r').read()))
statement = session.query(Reaction).\
reactions = session.query(Reaction).\
filter(Reaction.deletedAt == None).\
order_by(desc("createdAt")).\
offset(offset).limit(size)
reactions = []
for row in session.execute(statement):
reaction = row.Reaction
reactions.append(reaction)
for r in reactions:
r.createdBy = await UserStorage.get_user(r.createdBy or 'discours')
reactions = list(reactions)
reactions.sort(key=lambda x: x.createdAt, reverse=True)
return reactions
@query.field("reactionsByAuthor")
def get_reactions_by_author(_, info, slug, page=1, size=50):
async def get_reactions_by_author(_, info, slug, page=1, size=50):
offset = page * size
reactions = []
with local_session() as session:
reactions = session.query(Reaction).filter(Reaction.createdBy == slug).limit(size).offset(offset).all()
reactions = session.query(Reaction).filter(Reaction.createdBy == slug).limit(size).offset(offset)
for r in reactions:
r.createdBy = await UserStorage.get_user(r.createdBy or 'discours')
return reactions

1
schema.graphql
View File

@ -414,7 +414,6 @@ type Stat {
reacted: Int
rating: Int
commented: Int
bookmarked: Int
}
type Community {

106
services/stat/reacted.py
View File

@ -1,7 +1,7 @@
import asyncio
from datetime import datetime
from sqlalchemy.types import Enum
from sqlalchemy import Column, DateTime, ForeignKey
from sqlalchemy import Column, DateTime, ForeignKey, Boolean
# from sqlalchemy.orm.attributes import flag_modified
from sqlalchemy import Enum
import enum
@ -37,6 +37,7 @@ def kind_to_rate(kind) -> int:
ReactionKind.REJECT
]: return -1
else: return 0
class ReactedByDay(Base):
__tablename__ = "reacted_by_day"
@ -46,6 +47,8 @@ class ReactedByDay(Base):
replyTo = Column(ForeignKey('reaction.id'), nullable=True)
kind: int = Column(Enum(ReactionKind), nullable=False, comment="Reaction kind")
day = Column(DateTime, primary_key=True, default=datetime.now)
comment = Column(Boolean, nullable=True)
class ReactedStorage:
reacted = {
@ -63,34 +66,6 @@ class ReactedStorage:
period = 30*60 # sec
lock = asyncio.Lock()
@staticmethod
def init(session):
self = ReactedStorage
all_reactions = session.query(ReactedByDay).all()
print('[stat.reacted] %d reactions total' % len(all_reactions))
for reaction in all_reactions:
shout = reaction.shout
topics = session.query(ShoutTopic.topic).where(ShoutTopic.shout == shout).all()
kind = reaction.kind
self.reacted['shouts'][shout] = self.reacted['shouts'].get(shout, [])
self.reacted['shouts'][shout].append(reaction)
self.rating['shouts'][shout] = self.rating['shouts'].get(shout, 0) + kind_to_rate(kind)
for t in topics:
self.reacted['topics'][t] = self.reacted['topics'].get(t, [])
self.reacted['topics'][t].append(reaction)
self.rating['topics'][t] = self.rating['topics'].get(t, 0) + kind_to_rate(kind) # rating
if reaction.replyTo:
self.reacted['reactions'][reaction.replyTo] = self.reacted['reactions'].get(reaction.replyTo, [])
self.reacted['reactions'][reaction.replyTo].append(reaction)
self.rating['reactions'][reaction.replyTo] = self.rating['reactions'].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind)
ttt = self.reacted['topics'].values()
print('[stat.reacted] %d topics reacted' % len(ttt))
print('[stat.reacted] %d shouts reacted' % len(self.reacted['shouts']))
print('[stat.reacted] %d reactions reacted' % len(self.reacted['reactions']))
@staticmethod
async def get_shout(shout_slug):
self = ReactedStorage
@ -103,6 +78,24 @@ class ReactedStorage:
async with self.lock:
return self.reacted['topics'].get(topic_slug, [])
@staticmethod
async def get_comments(shout_slug):
self = ReactedStorage
async with self.lock:
return list(filter(lambda r: r.comment, self.reacted['shouts'].get(shout_slug, [])))
@staticmethod
async def get_topic_comments(topic_slug):
self = ReactedStorage
async with self.lock:
return list(filter(lambda r: r.comment, self.reacted['topics'].get(topic_slug, [])))
@staticmethod
async def get_reaction_comments(reaction_id):
self = ReactedStorage
async with self.lock:
return list(filter(lambda r: r.comment, self.reacted['reactions'].get(reaction_id)))
@staticmethod
async def get_reaction(reaction_id):
self = ReactedStorage
@ -137,17 +130,52 @@ class ReactedStorage:
return rating
@staticmethod
async def increment(shout_slug, kind, reply_id = None):
async def increment(reaction):
self = ReactedStorage
reaction: ReactedByDay = None
async with self.lock:
with local_session() as session:
reaction = ReactedByDay.create(shout=shout_slug, kind=kind, reply=reply_id)
self.reacted['shouts'][shout_slug] = self.reacted['shouts'].get(shout_slug, [])
self.reacted['shouts'][shout_slug].append(reaction)
if reply_id:
self.reacted['reaction'][reply_id] = self.reacted['reactions'].get(shout_slug, [])
self.reacted['reaction'][reply_id].append(reaction)
self.rating['reactions'][reply_id] = self.rating['reactions'].get(reply_id, 0) + kind_to_rate(kind)
r = {
"day": datetime.now().replace(hour=0, minute=0, second=0, microsecond=0),
"reaction": reaction.id,
"kind": reaction.kind,
"shout": reaction.shout
}
if reaction.replyTo: r['replyTo'] = reaction.replyTo
if reaction.body: r['comment'] = True
reaction = ReactedByDay.create(**r)
self.reacted['shouts'][reaction.shout] = self.reacted['shouts'].get(reaction.shout, [])
self.reacted['shouts'][reaction.shout].append(reaction)
if reaction.replyTo:
self.reacted['reaction'][reaction.replyTo] = self.reacted['reactions'].get(reaction.shout, [])
self.reacted['reaction'][reaction.replyTo].append(reaction)
self.rating['reactions'][reaction.replyTo] = self.rating['reactions'].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind)
else:
self.rating['shouts'][shout_slug] = self.rating['shouts'].get(shout_slug, 0) + kind_to_rate(kind)
self.rating['shouts'][reaction.replyTo] = self.rating['shouts'].get(reaction.shout, 0) + kind_to_rate(reaction.kind)
@staticmethod
def init(session):
self = ReactedStorage
all_reactions = session.query(ReactedByDay).all()
print('[stat.reacted] %d reactions total' % len(all_reactions))
for reaction in all_reactions:
shout = reaction.shout
topics = session.query(ShoutTopic.topic).where(ShoutTopic.shout == shout).all()
kind = reaction.kind
self.reacted['shouts'][shout] = self.reacted['shouts'].get(shout, [])
self.reacted['shouts'][shout].append(reaction)
self.rating['shouts'][shout] = self.rating['shouts'].get(shout, 0) + kind_to_rate(kind)
for t in topics:
self.reacted['topics'][t] = self.reacted['topics'].get(t, [])
self.reacted['topics'][t].append(reaction)
self.rating['topics'][t] = self.rating['topics'].get(t, 0) + kind_to_rate(kind) # rating
if reaction.replyTo:
self.reacted['reactions'][reaction.replyTo] = self.reacted['reactions'].get(reaction.replyTo, [])
self.reacted['reactions'][reaction.replyTo].append(reaction)
self.rating['reactions'][reaction.replyTo] = self.rating['reactions'].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind)
ttt = self.reacted['topics'].values()
print('[stat.reacted] %d topics reacted' % len(ttt))
print('[stat.reacted] %d shouts reacted' % len(self.reacted['shouts']))
print('[stat.reacted] %d reactions reacted' % len(self.reacted['reactions']))

1
services/stat/topicstat.py
View File

@ -67,7 +67,6 @@ class TopicStat:
"viewed": await ViewedStorage.get_topic(topic),
"reacted" : len(await ReactedStorage.get_topic(topic)),
"commented": len(await ReactedStorage.get_topic_comments(topic)),
"bookmarked": len(await ReactedStorage.get_topic_bookmarked(topic)),
"rating" : await ReactedStorage.get_topic_rating(topic),
}