feat(publishing): implement immediate raw ingestion, per-target Redis queues, and configurable post intervals & sleep windows

core/queue.py

@@ -1,6 +1,7 @@
 import os
+import json
 import logging
-from typing import Optional
+from typing import Optional, Dict, Any
 import redis.asyncio as redis
 
 logger = logging.getLogger(__name__)
@@ -8,9 +9,8 @@ logger = logging.getLogger(__name__)
 REDIS_URL = os.getenv("REDIS_URL", "redis://copykar_redis:6379/0" if os.path.exists("/app") else "redis://localhost:6379/0")
 
 class RedisQueue:
-    def __init__(self, redis_url: Optional[str] = None, queue_key: str = "copykar:queue:incoming"):
+    def __init__(self, redis_url: Optional[str] = None):
         self.redis_url = redis_url or REDIS_URL
-        self.queue_key = queue_key
         self.client: Optional[redis.Redis] = None
 
     async def connect(self):
@@ -19,22 +19,43 @@ class RedisQueue:
             await self.client.ping()
             logger.info(f"Connected to Redis at {self.redis_url}")
 
-    async def push(self, post_id: int):
-        if not self.client:
-            await self.connect()
-        await self.client.rpush(self.queue_key, str(post_id))
-        logger.info(f"Enqueued post ID {post_id} to Redis queue [{self.queue_key}]")
+    def _get_target_key(self, target_id: int) -> str:
+        return f"copykar:queue:target:{target_id}"
 
-    async def pop(self) -> Optional[int]:
+    async def push_target_post(self, target_id: int, payload: Dict[str, Any]):
         if not self.client:
             await self.connect()
-        val = await self.client.lpop(self.queue_key)
-        return int(val) if val else None
+        key = self._get_target_key(target_id)
+        raw_json = json.dumps(payload)
+        await self.client.rpush(key, raw_json)
+        logger.info(f"Enqueued post {payload.get('post_id')} to Target #{target_id} queue [{key}]")
 
-    async def qsize(self) -> int:
+    async def pop_target_post(self, target_id: int) -> Optional[Dict[str, Any]]:
         if not self.client:
             await self.connect()
-        return await self.client.llen(self.queue_key)
+        key = self._get_target_key(target_id)
+        raw = await self.client.lpop(key)
+        if raw:
+            try:
+                return json.loads(raw)
+            except Exception as e:
+                logger.error(f"Error parsing queue JSON from {key}: {e}")
+        return None
+
+    async def get_target_queue_size(self, target_id: int) -> int:
+        if not self.client:
+            await self.connect()
+        key = self._get_target_key(target_id)
+        return await self.client.llen(key)
+
+    async def get_total_queued_posts(self) -> int:
+        if not self.client:
+            await self.connect()
+        keys = await self.client.keys("copykar:queue:target:*")
+        total = 0
+        for k in keys:
+            total += await self.client.llen(k)
+        return total
 
     async def close(self):
         if self.client: