resolvers, orm, migration, schema fixes

.gitignore

@@ -145,5 +145,5 @@ migration/content/**/*.md
 .DS_Store
 dump
 .vscode
-*.sql
+*dump.sql
 *.csv

base/resolvers.py

@@ -1,15 +1,13 @@
 from ariadne import MutationType, QueryType, SubscriptionType, ScalarType
 
 
-query = QueryType()
-mutation = MutationType()
-subscription = SubscriptionType()
-
-
 datetime_scalar = ScalarType("DateTime")
 
 @datetime_scalar.serializer
 def serialize_datetime(value):
 	return value.isoformat()
 
+query = QueryType()
+mutation = MutationType()
+subscription = SubscriptionType()
 resolvers = [query, mutation, subscription, datetime_scalar]

migration/tables/content_items.py

@@ -123,12 +123,15 @@ def migrate(entry, storage):
 	#del shout_dict['ratings']
 	email = userdata.get('email')
 	slug = userdata.get('slug')
+	if not slug: raise Exception
 	with local_session() as session:
 		# c = session.query(Community).all().pop()
 		if email: user = session.query(User).filter(User.email == email).first()
 		if not user and slug: user = session.query(User).filter(User.slug == slug).first()
 		if not user and userdata: 
-			try: user = User.create(**userdata)
+			try: 
+				userdata['slug'] = userdata['slug'].lower().strip().replace(' ', '-')
+				user = User.create(**userdata)
 			except sqlalchemy.exc.IntegrityError:
 				print('[migration] user error: ' + userdata)
 			userdata['id'] = user.id

migration/tables/users.py

@@ -29,7 +29,7 @@ def migrate(entry):
 	if 'wasOnineAt' in entry: user_dict['wasOnlineAt'] = parse(entry['wasOnlineAt'])
 	if entry.get('profile'):
 		# slug
-		user_dict['slug'] = entry['profile'].get('path')
+		user_dict['slug'] = entry['profile'].get('path').lower().replace(' ', '-').strip()
 		user_dict['bio'] = html2text(entry.get('profile').get('bio') or '')
 
 		# userpic
@@ -41,10 +41,10 @@ def migrate(entry):
 		# name
 		fn = entry['profile'].get('firstName', '')
 		ln = entry['profile'].get('lastName', '')
-		name = user_dict['slug'] if user_dict['slug'] else 'noname'
+		name = user_dict['slug'] if user_dict['slug'] else 'anonymous'
 		name = fn if fn else name
 		name = (name + ' ' + ln) if ln else name
-		name = entry['profile']['path'].lower().replace(' ', '-') if len(name) < 2 else name
+		name = entry['profile']['path'].lower().strip().replace(' ', '-') if len(name) < 2 else name
 		user_dict['name'] = name
 
 		# links
@@ -63,6 +63,7 @@ def migrate(entry):
 
 	user_dict['slug'] = user_dict.get('slug', user_dict['email'].split('@')[0])
 	oid = user_dict['oid']
+	user_dict['slug'] = user_dict['slug'].lower().strip().replace(' ', '-')
 	try: user = User.create(**user_dict.copy())
 	except sqlalchemy.exc.IntegrityError:
 		print('[migration] cannot create user ' + user_dict['slug'])

orm/reaction.py

@@ -1,52 +1,10 @@
 from datetime import datetime
 from sqlalchemy import Column, String, ForeignKey, DateTime
-from base.orm import Base, local_session
+from base.orm import Base
-import enum
 from sqlalchemy import Enum
+from services.stat.reacted import ReactedStorage, ReactionKind
 from services.stat.viewed import ViewedStorage
 
-class ReactionKind(enum.Enum):
-	AGREE 	= 1 # +1
-	DISAGREE = 2 # -1
-	PROOF 	= 3	# +1
-	DISPROOF = 4 # -1
-	ASK		= 5 # +0 bookmark
-	PROPOSE	= 6 # +0
-	QUOTE	= 7 # +0 bookmark
-	COMMENT	= 8 # +0
-	ACCEPT	= 9 # +1
-	REJECT	= 0 # -1
-	LIKE	= 11 # +1
-	DISLIKE	= 12 # -1
-	# TYPE = <reaction index> # rating diff
-
-def kind_to_rate(kind) -> int:
-	if kind in [
-		ReactionKind.AGREE,
-		ReactionKind.LIKE,
-		ReactionKind.PROOF,
-		ReactionKind.ACCEPT
-	]: return 1
-	elif kind in [
-		ReactionKind.DISAGREE,
-		ReactionKind.DISLIKE,
-		ReactionKind.DISPROOF,
-		ReactionKind.REJECT
-	]: return -1
-	else: return 0
-
-def get_bookmarked(reactions):
-    c = 0
-    for r in reactions:
-        c += 1 if r.kind in [ ReactionKind.QUOTE, ReactionKind.ASK] else 0
-    return c
-
-def get_rating(reactions):
-    rating = 0
-    for r in reactions:
-        rating += kind_to_rate(r.kind)
-    return rating
-
 class Reaction(Base):
 	__tablename__ = 'reaction'
 	body: str = Column(String, nullable=True, comment="Reaction Body")
@@ -64,15 +22,10 @@ class Reaction(Base):
 
 	@property
 	async def stat(self):
-		reacted = []
+		rrr = await ReactedStorage.get_reaction(self.id)
-		try:
+		print(rrr[0])
-			with local_session() as session:
-				reacted = session.query(Reaction).filter(Reaction.replyTo == self.id).all()
-		except Exception as e:
-			print(e)
 		return {
 			"viewed": await ViewedStorage.get_reaction(self.id),
-			"reacted": reacted.count(),
+			"reacted": len(rrr),
-			"rating": get_rating(reacted),
+			"rating": await ReactedStorage.get_reaction_rating(self.id)
-			"bookmarked": get_bookmarked(reacted)
+		}
-		}

orm/shout.py

@@ -3,10 +3,10 @@ from sqlalchemy import Column, Integer, String, ForeignKey, DateTime, Boolean
 from sqlalchemy.orm import relationship
 from orm.user import User
 from orm.topic import Topic, ShoutTopic
-from orm.reaction import Reaction, get_bookmarked
+from orm.reaction import Reaction
-from services.stat.reacted import ReactedStorage
+from services.stat.reacted import ReactedStorage, ReactionKind
 from services.stat.viewed import ViewedStorage
-from base.orm import Base, local_session
+from base.orm import Base
 
 
 class ShoutReactionsFollower(Base):
@@ -63,15 +63,9 @@ class Shout(Base):
 
     @property
     async def stat(self):
-        reacted = []
+        rrr = await ReactedStorage.get_shout(self.slug)
-        try:
-            with local_session() as session:
-                reacted = session.query(Reaction).where(Reaction.shout == self.slug).all()
-        except Exception as e:
-            print(e)
         return {
                 "viewed": await ViewedStorage.get_shout(self.slug),
-                "reacted": await ReactedStorage.get_shout(self.slug),
+			    "reacted": len(rrr),
-                "rating": await ReactedStorage.get_rating(self.slug),
+			    "rating": await ReactedStorage.get_rating(self.slug)
-                "bookmarked": get_bookmarked(reacted)
+		}
-            }

queries/shout-by-slug.sql

@@ -0,0 +1,4 @@
+SELECT s.*, a.*, sa.* FROM shout s 
+JOIN shout_author sa ON s.slug = sa.shout 
+JOIN user a ON a.slug = sa.user 
+WHERE sa.slug = a.slug AND a.slug = %s;

resolvers/reactions.py

@@ -6,6 +6,7 @@ from orm.user import User
 from base.resolvers import mutation, query
 from auth.authenticate import login_required
 from datetime import datetime
+from services.auth.users import UserStorage
 from services.stat.reacted import ReactedStorage
 
 def reactions_follow(user, slug, auto=False):
@@ -103,11 +104,15 @@ async def delete_reaction(_, info, id):
     return {}
 
 @query.field("reactionsByShout")
-def get_shout_reactions(_, info, slug, page, size):
+async def get_shout_reactions(_, info, slug, page, size):
     offset = page * size
     reactions = []
     with local_session() as session:
-        reactions = session.query(Reaction).filter(Reaction.shout == slug).limit(size).offset(offset).all()
+        reactions = session.query(Reaction).\
+            filter(Reaction.shout == slug).\
+            limit(size).offset(offset).all()
+    for r in reactions:
+        r.createdBy = await UserStorage.get_user(r.createdBy)
     return reactions
 
 
@@ -116,12 +121,13 @@ def get_all_reactions(_, info, page=1, size=10):
     offset = page * size
     reactions = []
     with local_session() as session:
-        stmt = session.query(Reaction).\
+        # raw sql: statement = text(open('queries/reactions-all.sql', 'r').read()))
+        statement = session.query(Reaction).\
             filter(Reaction.deletedAt == None).\
             order_by(desc("createdAt")).\
             offset(offset).limit(size)
         reactions = []
-        for row in session.execute(stmt):
+        for row in session.execute(statement):
             reaction = row.Reaction
             reactions.append(reaction)
         reactions.sort(key=lambda x: x.createdAt, reverse=True)

resolvers/zine.py

@@ -3,6 +3,7 @@ from orm.shout import Shout, ShoutAuthor, ShoutTopic
 from orm.topic import Topic
 from base.orm import local_session
 from base.resolvers import mutation, query
+from services.zine.shoutauthor import ShoutAuthorStorage
 from services.zine.shoutscache import ShoutsCache
 from services.stat.viewed import ViewedStorage
 from resolvers.profile import author_follow, author_unfollow
@@ -10,152 +11,171 @@ from resolvers.topics import topic_follow, topic_unfollow
 from resolvers.community import community_follow, community_unfollow
 from resolvers.reactions import reactions_follow, reactions_unfollow
 from auth.authenticate import login_required
-from sqlalchemy import select, desc, and_
+from sqlalchemy import select, desc, and_, text
-from sqlalchemy.orm import selectinload, joinedload
+from sqlalchemy.orm import selectinload
-
+from sqlalchemy.dialects import postgresql
 
 @query.field("topViewed")
 async def top_viewed(_, info, page, size):
-	async with ShoutsCache.lock:
+    async with ShoutsCache.lock:
-		return ShoutsCache.top_viewed[(page - 1) * size : page * size]
+        return ShoutsCache.top_viewed[(page - 1) * size : page * size]
 
 @query.field("topMonth")
 async def top_month(_, info, page, size):
-	async with ShoutsCache.lock:
+    async with ShoutsCache.lock:
-		return ShoutsCache.top_month[(page - 1) * size : page * size]
+        return ShoutsCache.top_month[(page - 1) * size : page * size]
 
 @query.field("topOverall")
 async def top_overall(_, info, page, size):
-	async with ShoutsCache.lock:
+    async with ShoutsCache.lock:
-		return ShoutsCache.top_overall[(page - 1) * size : page * size]
+        return ShoutsCache.top_overall[(page - 1) * size : page * size]
 
 @query.field("recentPublished")
 async def recent_published(_, info, page, size):
-	async with ShoutsCache.lock:
+    async with ShoutsCache.lock:
-		return ShoutsCache.recent_published[(page - 1) * size : page * size]
+        return ShoutsCache.recent_published[(page - 1) * size : page * size]
 
 @query.field("recentAll")
 async def recent_all(_, info, page, size):
-	async with ShoutsCache.lock:
+    async with ShoutsCache.lock:
-		return ShoutsCache.recent_all[(page - 1) * size : page * size]
+        return ShoutsCache.recent_all[(page - 1) * size : page * size]
 
 @query.field("recentReacted")
 async def recent_reacted(_, info, page, size):
-	async with ShoutsCache.lock:
+    async with ShoutsCache.lock:
-		return ShoutsCache.recent_reacted[(page - 1) * size : page * size]
+        return ShoutsCache.recent_reacted[(page - 1) * size : page * size]
 
 @mutation.field("viewShout")
 async def view_shout(_, info, slug):
-	await ViewedStorage.inc_shout(slug)
+    await ViewedStorage.inc_shout(slug)
-	return {"error" : ""}
+    return {"error" : ""}
 
 @query.field("getShoutBySlug")
 async def get_shout_by_slug(_, info, slug):
-	shout = None
+    all_fields = [node.name.value for node in info.field_nodes[0].selection_set.selections]
-	# FIXME: append captions anyhow
+    selected_fields = set(["authors", "topics"]).intersection(all_fields)
-	with local_session() as session:
+    select_options = [selectinload(getattr(Shout, field)) for field in selected_fields]
-		shout = session.query(Shout, ShoutAuthor.caption.label("author_caption")).\
-			options([
-				selectinload(Shout.topics),
-				selectinload(Shout.reactions),
-				joinedload(Shout.authors),
-				selectinload(ShoutAuthor.caption)
-			]).\
-			join(ShoutAuthor.shout == slug ).\
-			filter(Shout.slug == slug).first()
 
-	if not shout:
+    with local_session() as session:
-		print(f"[resolvers.zine] error: shout with slug {slug} not exist")
+        try: s = text(open('src/queries/shout-by-slug.sql', 'r').read() % slug)
-		return {"error" : "shout not found"}
+        except: pass
-	
+        shout_q = session.query(Shout).\
-	return shout
+            options(select_options).\
+            filter(Shout.slug == slug)
+        
+        print(shout_q.statement)
+        
+        shout = shout_q.first()
+        for a in shout.authors:
+            a.caption = await ShoutAuthorStorage.get_author_caption(slug, a.slug)
+
+    if not shout:
+        print(f"shout with slug {slug} not exist")
+        return {"error" : "shout not found"}
+    return shout
 
 @query.field("shoutsByTopics")
 async def shouts_by_topics(_, info, slugs, page, size):
-	page = page - 1
+    page = page - 1
-	with local_session() as session:
+    with local_session() as session:
-		shouts = session.query(Shout).\
+        shouts = session.query(Shout).\
-			join(ShoutTopic).\
+            join(ShoutTopic).\
-			where(and_(ShoutTopic.topic.in_(slugs), Shout.publishedAt != None)).\
+            where(and_(ShoutTopic.topic.in_(slugs), Shout.publishedAt != None)).\
-			order_by(desc(Shout.publishedAt)).\
+            order_by(desc(Shout.publishedAt)).\
-			limit(size).\
+            limit(size).\
-			offset(page * size)
+            offset(page * size)
-	return shouts
+    
+    for s in shouts:
+        for a in s.authors:
+            a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug)
+    return shouts
 
 @query.field("shoutsByCollection")
 async def shouts_by_topics(_, info, collection, page, size):
-	page = page - 1
+    page = page - 1
-	with local_session() as session:
+    shouts = []
-		shouts = session.query(Shout).\
+    with local_session() as session:
-			join(ShoutCollection, ShoutCollection.collection == collection).\
+        shouts = session.query(Shout).\
-			where(and_(ShoutCollection.shout == Shout.slug, Shout.publishedAt != None)).\
+            join(ShoutCollection, ShoutCollection.collection == collection).\
-			order_by(desc(Shout.publishedAt)).\
+            where(and_(ShoutCollection.shout == Shout.slug, Shout.publishedAt != None)).\
-			limit(size).\
+            order_by(desc(Shout.publishedAt)).\
-			offset(page * size)
+            limit(size).\
-	return shouts
+            offset(page * size)
+    for s in shouts:
+        for a in s.authors:
+            a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug)
+    return shouts
 
 @query.field("shoutsByAuthors")
 async def shouts_by_authors(_, info, slugs, page, size):
-	page = page - 1
+    page = page - 1
-	with local_session() as session:
+    with local_session() as session:
 
-		shouts = session.query(Shout).\
+        shouts = session.query(Shout).\
-			join(ShoutAuthor).\
+            join(ShoutAuthor).\
-			where(and_(ShoutAuthor.user.in_(slugs), Shout.publishedAt != None)).\
+            where(and_(ShoutAuthor.user.in_(slugs), Shout.publishedAt != None)).\
-			order_by(desc(Shout.publishedAt)).\
+            order_by(desc(Shout.publishedAt)).\
-			limit(size).\
+            limit(size).\
-			offset(page * size)
+            offset(page * size)
-	return shouts
+    
+    for s in shouts:
+        for a in s.authors:
+            a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug)
+    return shouts
 
 @query.field("shoutsByCommunities")
 async def shouts_by_communities(_, info, slugs, page, size):
-	page = page - 1
+    page = page - 1
-	with local_session() as session:
+    with local_session() as session:
-		#TODO fix postgres high load
+        #TODO fix postgres high load
-		shouts = session.query(Shout).distinct().\
+        shouts = session.query(Shout).distinct().\
-			join(ShoutTopic).\
+            join(ShoutTopic).\
-			where(and_(Shout.publishedAt != None,\
+            where(and_(Shout.publishedAt != None,\
-				ShoutTopic.topic.in_(\
+                ShoutTopic.topic.in_(\
-				select(Topic.slug).where(Topic.community.in_(slugs))\
+                select(Topic.slug).where(Topic.community.in_(slugs))\
-			))).\
+            ))).\
-			order_by(desc(Shout.publishedAt)).\
+            order_by(desc(Shout.publishedAt)).\
-			limit(size).\
+            limit(size).\
-			offset(page * size)
+            offset(page * size)
-	return shouts
+    
+    for s in shouts:
+        for a in s.authors:
+            a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug)
+    return shouts
 
 @mutation.field("follow")
 @login_required
 async def follow(_, info, what, slug):
-	user = info.context["request"].user
+    user = info.context["request"].user
-	try:
+    try:
-		if what == "AUTHOR":
+        if what == "AUTHOR":
-			author_follow(user, slug)
+            author_follow(user, slug)
-		elif what == "TOPIC":
+        elif what == "TOPIC":
-			topic_follow(user, slug)
+            topic_follow(user, slug)
-		elif what == "COMMUNITY":
+        elif what == "COMMUNITY":
-			community_follow(user, slug)
+            community_follow(user, slug)
-		elif what == "REACTIONS":
+        elif what == "REACTIONS":
-			reactions_follow(user, slug)
+            reactions_follow(user, slug)
-	except Exception as e:
+    except Exception as e:
-		return {"error" : str(e)}
+        return {"error" : str(e)}
 
-	return {}
+    return {}
 
 @mutation.field("unfollow")
 @login_required
 async def unfollow(_, info, what, slug):
-	user = info.context["request"].user
+    user = info.context["request"].user
 
-	try:
+    try:
-		if what == "AUTHOR":
+        if what == "AUTHOR":
-			author_unfollow(user, slug)
+            author_unfollow(user, slug)
-		elif what == "TOPIC":
+        elif what == "TOPIC":
-			topic_unfollow(user, slug)
+            topic_unfollow(user, slug)
-		elif what == "COMMUNITY":
+        elif what == "COMMUNITY":
-			community_unfollow(user, slug)
+            community_unfollow(user, slug)
-		elif what == "REACTIONS":
+        elif what == "REACTIONS":
-			reactions_unfollow(user, slug)
+            reactions_unfollow(user, slug)
-	except Exception as e:
+    except Exception as e:
-		return {"error" : str(e)}
+        return {"error" : str(e)}
 
-	return {}
+    return {}

schema.graphql

@@ -241,7 +241,7 @@ type Query {
   # reactons
   reactionsAll(page: Int!, size: Int!): [Reaction]!
   reactionsByAuthor(slug: String!, page: Int!, size: Int!): [Reaction]!
-  reactionsByShout(slug: String!): [Reaction]!
+  reactionsByShout(slug: String!, page: Int!, size: Int!): [Reaction]!
 
   # collab
   inviteAuthor(slug: String!, author: String!): Result!

services/stat/reacted.py

@@ -1,12 +1,42 @@
 import asyncio
 from datetime import datetime
 from sqlalchemy.types import Enum
-from sqlalchemy import Column, DateTime, ForeignKey, Integer
+from sqlalchemy import Column, DateTime, ForeignKey
 # from sqlalchemy.orm.attributes import flag_modified
+from sqlalchemy import Enum
+import enum
 from base.orm import Base, local_session
-from orm.reaction import Reaction, ReactionKind, kind_to_rate
 from orm.topic import ShoutTopic
 
+class ReactionKind(enum.Enum):
+	AGREE 	= 1 # +1
+	DISAGREE = 2 # -1
+	PROOF 	= 3	# +1
+	DISPROOF = 4 # -1
+	ASK		= 5 # +0 bookmark
+	PROPOSE	= 6 # +0
+	QUOTE	= 7 # +0 bookmark
+	COMMENT	= 8 # +0
+	ACCEPT	= 9 # +1
+	REJECT	= 0 # -1
+	LIKE	= 11 # +1
+	DISLIKE	= 12 # -1
+	# TYPE = <reaction index> # rating diff
+
+def kind_to_rate(kind) -> int:
+	if kind in [
+		ReactionKind.AGREE,
+		ReactionKind.LIKE,
+		ReactionKind.PROOF,
+		ReactionKind.ACCEPT
+	]: return 1
+	elif kind in [
+		ReactionKind.DISAGREE,
+		ReactionKind.DISLIKE,
+		ReactionKind.DISPROOF,
+		ReactionKind.REJECT
+	]: return -1
+	else: return 0
 class ReactedByDay(Base):
 	__tablename__ = "reacted_by_day"
 
@@ -37,12 +67,8 @@ class ReactedStorage:
 	def init(session):
 		self = ReactedStorage
 		all_reactions = session.query(ReactedByDay).all()
-		all_reactions2 = session.query(Reaction).filter(Reaction.deletedAt == None).all()
+		print('[stat.reacted] %d reactions total' % len(all_reactions))
-		print('[stat.reacted] %d reactions total' % len(all_reactions or all_reactions2))
+		for reaction in all_reactions:
-		rrr = (all_reactions or all_reactions2)
-		create = False
-		if not all_reactions: create = True
-		for reaction in rrr:
 			shout = reaction.shout
 			topics = session.query(ShoutTopic.topic).where(ShoutTopic.shout == shout).all()
 			kind = reaction.kind
@@ -64,12 +90,6 @@ class ReactedStorage:
 		print('[stat.reacted] %d topics reacted' % len(ttt))
 		print('[stat.reacted] %d shouts reacted' % len(self.reacted['shouts']))
 		print('[stat.reacted] %d reactions reacted' % len(self.reacted['reactions']))
-  
-		if len(all_reactions) == 0 and len(all_reactions2) != 0:
-			with local_session() as session:
-				for r in all_reactions2:
-					session.add(ReactedByDay(reaction=r.id, shout=r.shout, reply=r.replyTo, kind=r.kind, day=r.createdAt.replace(hour=0, minute=0, second=0)))
-				session.commit()
 
 	@staticmethod
 	async def get_shout(shout_slug):

services/stat/topicstat.py

@@ -65,9 +65,9 @@ class TopicStat:
 			"shouts" : len(shouts),
 			"authors" : len(authors),
 			"followers" : len(followers),
-			"viewed": ViewedStorage.get_topic(topic),
+			"viewed": await ViewedStorage.get_topic(topic),
-			"reacted" : ReactedStorage.get_topic(topic),
+			"reacted" : await ReactedStorage.get_topic(topic),
-			"rating" : ReactedStorage.get_topic_rating(topic),
+			"rating" : await ReactedStorage.get_topic_rating(topic),
 		}
 
 	@staticmethod

services/zine/shoutauthor.py

@@ -12,14 +12,10 @@ class ShoutAuthorStorage:
 	@staticmethod
 	async def load(session):
 		self = ShoutAuthorStorage
-		authors = session.query(ShoutAuthor).all()
+		sas = session.query(ShoutAuthor).all()
-		for author in authors:
+		for sa in sas:
-			user = author.user
+			self.authors_by_shout[sa.shout] = self.authors_by_shout.get(sa.shout, [])
-			shout = author.shout
+			self.authors_by_shout[sa.shout].append([sa.user, sa.caption])
-			if shout in self.authors_by_shout:
-				self.authors_by_shout[shout].append(user)
-			else:
-				self.authors_by_shout[shout] = [user]
 		print('[zine.authors] %d shouts preprocessed' % len(self.authors_by_shout))
 
 	@staticmethod
@@ -28,6 +24,15 @@ class ShoutAuthorStorage:
 		async with self.lock:
 			return self.authors_by_shout.get(shout, [])
 
+	@staticmethod
+	async def get_author_caption(shout, author):
+		self = ShoutAuthorStorage
+		async with self.lock:
+			for a in self.authors_by_shout.get(shout, []):
+				if author in a:
+					return a[1]
+		return { "error": "author caption not found" }
+
 	@staticmethod
 	async def worker():
 		self = ShoutAuthorStorage