batch-render-follows

main.py

@@ -7,7 +7,7 @@ from ariadne.asgi import GraphQL
 from starlette.applications import Starlette
 from starlette.routing import Route
 
-from services.follows import FollowsCached
+from services.follows import start_cached_follows
 from services.rediscache import redis
 from services.schema import resolvers
 from services.search import search_service
@@ -37,7 +37,7 @@ app = Starlette(
     on_startup=[
         redis.connect,
         ViewedStorage.init,
-        FollowsCached.worker,
+        start_cached_follows,
         search_service.info,
         # start_sentry,
         start,

services/follows.py

@@ -119,9 +119,16 @@ class FollowsCached:
 
     @staticmethod
     async def update_cache():
+        BATCH_SIZE = 30  # Adjust batch size as needed
         with local_session() as session:
-            for author in session.query(Author).all():
-                if isinstance(author, Author):
+            authors = session.query(Author).all()
+            total_authors = len(authors)
+            for i in range(0, total_authors, BATCH_SIZE):
+                batch_authors = authors[i:i+BATCH_SIZE]
+                await asyncio.gather(*[FollowsCached.update_author_cache(author) for author in batch_authors])
+
+    @staticmethod
+    async def update_author_cache(author):
         redis_key = f"user:{author.user}:author"
         author_dict = author.dict()
         if isinstance(author_dict, dict):
@@ -146,3 +153,6 @@ class FollowsCached:
                 break
             except Exception as exc:
                 logger.error(exc)
+
+async def start_cached_follows():
+    await FollowsCached.worker()