feat(queue): integrate redis queue and 2-minute paced ai consumer worker

services/collector.py

@@ -7,6 +7,7 @@ from telethon.tl.types import MessageMediaPhoto, MessageMediaDocument
 from db.repository import Repository
 from core.dedup import compute_content_hash, compute_file_hash
 from services.ai_processor import AIProcessor
+from core.queue import RedisQueue
 from core.metrics import COLLECTED_POSTS_TOTAL
 from core.proxy import get_telegram_proxy
 
@@ -20,6 +21,7 @@ class CollectorService:
         self,
         repo: Repository,
         ai_processor: AIProcessor,
+        queue: Optional[RedisQueue] = None,
         api_id: Optional[int] = None,
         api_hash: Optional[str] = None,
         phone: Optional[str] = None,
@@ -27,6 +29,7 @@ class CollectorService:
     ):
         self.repo = repo
         self.ai_processor = ai_processor
+        self.queue = queue
         self.api_id = api_id or int(os.getenv("API_ID", "0"))
         self.api_hash = api_hash or os.getenv("API_HASH", "")
         self.phone = phone or os.getenv("PHONE")
@@ -139,7 +142,10 @@ class CollectorService:
             if post_id:
                 COLLECTED_POSTS_TOTAL.labels(source_channel_id=str(chat_id)).inc()
                 logger.info(f"Collected new post ID {post_id} from channel {chat_id}")
-                await self.ai_processor.process_post(post_id)
+                if self.queue:
+                    await self.queue.push(post_id)
+                else:
+                    await self.ai_processor.process_post(post_id)
         except Exception as e:
             logger.error(f"Error handling message from {event.chat_id}: {e}", exc_info=True)
 
@@ -205,13 +211,16 @@ class CollectorService:
                     collected_count += 1
                     COLLECTED_POSTS_TOTAL.labels(source_channel_id=str(channel_id)).inc()
                     logger.info(f"Backfilled historical post ID {post_id} from {channel_id}")
-                    await self.ai_processor.process_post(post_id)
+                    if self.queue:
+                        await self.queue.push(post_id)
+                    else:
+                        await self.ai_processor.process_post(post_id)
                 else:
                     skipped_count += 1
 
             if progress_callback:
                 await progress_callback(
-                    f"✅ Scraped <b>{collected_count}</b> new historical posts from <code>{channel_id}</code> (Skipped {skipped_count} existing/empty)."
+                    f"✅ Scraped <b>{collected_count}</b> new posts from <code>{channel_id}</code> and queued in Redis! (Skipped {skipped_count} existing/empty)."
                 )
             return collected_count
         except Exception as e: