batch-render-follows

main.py

@@ -7,7 +7,7 @@ from ariadne.asgi import GraphQL
 from starlette.applications import Starlette
 from starlette.routing import Route
 
-from services.follows import FollowsCached
+from services.follows import start_cached_follows
 from services.rediscache import redis
 from services.schema import resolvers
 from services.search import search_service
@@ -37,7 +37,7 @@ app = Starlette(
     on_startup=[
         redis.connect,
         ViewedStorage.init,
-        FollowsCached.worker,
+        start_cached_follows,
         search_service.info,
         # start_sentry,
         start,

services/follows.py

@@ -119,17 +119,24 @@ class FollowsCached:
 
     @staticmethod
     async def update_cache():
+        BATCH_SIZE = 30  # Adjust batch size as needed
         with local_session() as session:
-            for author in session.query(Author).all():
+            authors = session.query(Author).all()
-                if isinstance(author, Author):
+            total_authors = len(authors)
-                    redis_key = f"user:{author.user}:author"
+            for i in range(0, total_authors, BATCH_SIZE):
-                    author_dict = author.dict()
+                batch_authors = authors[i:i+BATCH_SIZE]
-                    if isinstance(author_dict, dict):
+                await asyncio.gather(*[FollowsCached.update_author_cache(author) for author in batch_authors])
-                        await redis.execute("set", redis_key, json.dumps(author_dict))
+
-                    follows = await get_author_follows(None, None, user=author.user)
+    @staticmethod
-                    if isinstance(follows, dict):
+    async def update_author_cache(author):
-                        redis_key = f"user:{author.user}:follows"
+        redis_key = f"user:{author.user}:author"
-                        await redis.execute("set", redis_key, json.dumps(follows))
+        author_dict = author.dict()
+        if isinstance(author_dict, dict):
+            await redis.execute("set", redis_key, json.dumps(author_dict))
+        follows = await get_author_follows(None, None, user=author.user)
+        if isinstance(follows, dict):
+            redis_key = f"user:{author.user}:follows"
+            await redis.execute("set", redis_key, json.dumps(follows))
 
     @staticmethod
     async def worker():
@@ -146,3 +153,6 @@ class FollowsCached:
                 break
             except Exception as exc:
                 logger.error(exc)
+
+async def start_cached_follows():
+    await FollowsCached.worker()