diff --git a/.env.example b/.env.example
index 21672e8..3cfb627 100644
--- a/.env.example
+++ b/.env.example
@@ -37,3 +37,5 @@ METRICS_PORT=8000
MEDIA_DIR=/app/data/media
GRAFANA_USER=admin
GRAFANA_PASSWORD=admin
+REDIS_URL=redis://copykar_redis:6379/0
+AI_PROCESSING_INTERVAL_SECONDS=120
diff --git a/core/queue.py b/core/queue.py
new file mode 100644
index 0000000..07efd82
--- /dev/null
+++ b/core/queue.py
@@ -0,0 +1,42 @@
+import os
+import logging
+from typing import Optional
+import redis.asyncio as redis
+
+logger = logging.getLogger(__name__)
+
+REDIS_URL = os.getenv("REDIS_URL", "redis://copykar_redis:6379/0" if os.path.exists("/app") else "redis://localhost:6379/0")
+
+class RedisQueue:
+ def __init__(self, redis_url: Optional[str] = None, queue_key: str = "copykar:queue:incoming"):
+ self.redis_url = redis_url or REDIS_URL
+ self.queue_key = queue_key
+ self.client: Optional[redis.Redis] = None
+
+ async def connect(self):
+ if not self.client:
+ self.client = redis.from_url(self.redis_url, decode_responses=True)
+ await self.client.ping()
+ logger.info(f"Connected to Redis at {self.redis_url}")
+
+ async def push(self, post_id: int):
+ if not self.client:
+ await self.connect()
+ await self.client.rpush(self.queue_key, str(post_id))
+ logger.info(f"Enqueued post ID {post_id} to Redis queue [{self.queue_key}]")
+
+ async def pop(self) -> Optional[int]:
+ if not self.client:
+ await self.connect()
+ val = await self.client.lpop(self.queue_key)
+ return int(val) if val else None
+
+ async def qsize(self) -> int:
+ if not self.client:
+ await self.connect()
+ return await self.client.llen(self.queue_key)
+
+ async def close(self):
+ if self.client:
+ await self.client.aclose()
+ logger.info("Redis connection closed.")
diff --git a/docker-compose.yml b/docker-compose.yml
index 197541e..607b43a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -17,6 +17,20 @@ services:
timeout: 5s
retries: 5
+ redis:
+ image: redis:7-alpine
+ container_name: copykar_redis
+ restart: unless-stopped
+ ports:
+ - "6379:6379"
+ volumes:
+ - redis_data:/data
+ healthcheck:
+ test: ["CMD", "redis-cli", "ping"]
+ interval: 5s
+ timeout: 3s
+ retries: 5
+
copykar:
image: copykar:latest
build:
@@ -27,6 +41,8 @@ services:
depends_on:
postgres:
condition: service_healthy
+ redis:
+ condition: service_healthy
ports:
- "8000:8000"
env_file:
@@ -65,6 +81,7 @@ services:
volumes:
postgres_data:
+ redis_data:
prometheus_data:
grafana_data:
copykar_data:
diff --git a/main.py b/main.py
index 18c99e3..09c2a3d 100644
--- a/main.py
+++ b/main.py
@@ -8,6 +8,8 @@ from db.database import init_db, close_db_pool
from db.repository import Repository
from core.metrics import start_metrics_server
from core.llm import LLMClient
+from core.queue import RedisQueue
+from services.queue_consumer import QueueConsumerService
from services.ai_processor import AIProcessor
from services.collector import CollectorService
from services.admin_bot import AdminBotService
@@ -30,10 +32,13 @@ async def main():
metrics_port = int(os.getenv("METRICS_PORT", "8000"))
start_metrics_server(metrics_port)
- # 2. Initialize Database schema
+ # 2. Initialize Database schema & Redis Queue
await init_db()
logger.info("Database schema initialized.")
+ redis_queue = RedisQueue()
+ await redis_queue.connect()
+
repo = Repository()
llm = LLMClient()
@@ -50,13 +55,15 @@ async def main():
return post
ai_processor.process_post = process_and_notify
- collector = CollectorService(repo=repo, ai_processor=ai_processor)
+ collector = CollectorService(repo=repo, ai_processor=ai_processor, queue=redis_queue)
admin_bot.set_collector(collector)
publisher = PublisherService(repo=repo)
+ queue_consumer = QueueConsumerService(queue=redis_queue, ai_processor=ai_processor)
# 4. Start all services
await admin_bot.start()
await collector.start(notify_fn=admin_bot.notify_admins)
+ await queue_consumer.start()
await publisher.start()
logger.info("All Copykar services are active and running.")
@@ -78,8 +85,10 @@ async def main():
finally:
logger.info("Shutting down Copykar services...")
await collector.stop()
+ await queue_consumer.stop()
await publisher.stop()
await admin_bot.stop()
+ await redis_queue.close()
await close_db_pool()
logger.info("Copykar cleanly shut down.")
diff --git a/requirements.txt b/requirements.txt
index 46d2cf8..60fe689 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,3 +4,4 @@ asyncpg==0.31.0
prometheus-client==0.26.0
httpx==0.28.1
python-socks==3.0.0
+redis==8.1.0
diff --git a/services/admin_bot.py b/services/admin_bot.py
index 52ec4f9..fbcdac2 100644
--- a/services/admin_bot.py
+++ b/services/admin_bot.py
@@ -165,9 +165,11 @@ class AdminBotService:
approved = len(await self.repo.get_posts_by_status("approved", limit=1000))
published = len(await self.repo.get_posts_by_status("published", limit=1000))
rejected = len(await self.repo.get_posts_by_status("rejected", limit=1000))
+ redis_q = await self.collector.queue.qsize() if (self.collector and self.collector.queue) else 0
text = (
"📊 Copykar Fleet Metrics\n\n"
+ f"• 📥 Redis Incoming Queue: {redis_q} (Pacing: 1 post / 2m)\n"
f"• ⏳ Pending AI: {pending_ai}\n"
f"• 📋 Pending Review: {pending_review}\n"
f"• 🚀 Approved (In Queue): {approved}\n"
diff --git a/services/collector.py b/services/collector.py
index 530964a..e1a788c 100644
--- a/services/collector.py
+++ b/services/collector.py
@@ -7,6 +7,7 @@ from telethon.tl.types import MessageMediaPhoto, MessageMediaDocument
from db.repository import Repository
from core.dedup import compute_content_hash, compute_file_hash
from services.ai_processor import AIProcessor
+from core.queue import RedisQueue
from core.metrics import COLLECTED_POSTS_TOTAL
from core.proxy import get_telegram_proxy
@@ -20,6 +21,7 @@ class CollectorService:
self,
repo: Repository,
ai_processor: AIProcessor,
+ queue: Optional[RedisQueue] = None,
api_id: Optional[int] = None,
api_hash: Optional[str] = None,
phone: Optional[str] = None,
@@ -27,6 +29,7 @@ class CollectorService:
):
self.repo = repo
self.ai_processor = ai_processor
+ self.queue = queue
self.api_id = api_id or int(os.getenv("API_ID", "0"))
self.api_hash = api_hash or os.getenv("API_HASH", "")
self.phone = phone or os.getenv("PHONE")
@@ -139,7 +142,10 @@ class CollectorService:
if post_id:
COLLECTED_POSTS_TOTAL.labels(source_channel_id=str(chat_id)).inc()
logger.info(f"Collected new post ID {post_id} from channel {chat_id}")
- await self.ai_processor.process_post(post_id)
+ if self.queue:
+ await self.queue.push(post_id)
+ else:
+ await self.ai_processor.process_post(post_id)
except Exception as e:
logger.error(f"Error handling message from {event.chat_id}: {e}", exc_info=True)
@@ -205,13 +211,16 @@ class CollectorService:
collected_count += 1
COLLECTED_POSTS_TOTAL.labels(source_channel_id=str(channel_id)).inc()
logger.info(f"Backfilled historical post ID {post_id} from {channel_id}")
- await self.ai_processor.process_post(post_id)
+ if self.queue:
+ await self.queue.push(post_id)
+ else:
+ await self.ai_processor.process_post(post_id)
else:
skipped_count += 1
if progress_callback:
await progress_callback(
- f"✅ Scraped {collected_count} new historical posts from {channel_id} (Skipped {skipped_count} existing/empty)."
+ f"✅ Scraped {collected_count} new posts from {channel_id} and queued in Redis! (Skipped {skipped_count} existing/empty)."
)
return collected_count
except Exception as e:
diff --git a/services/queue_consumer.py b/services/queue_consumer.py
new file mode 100644
index 0000000..3df2138
--- /dev/null
+++ b/services/queue_consumer.py
@@ -0,0 +1,49 @@
+import os
+import asyncio
+import logging
+from typing import Optional
+from core.queue import RedisQueue
+from services.ai_processor import AIProcessor
+from core.metrics import QUEUE_POSTS_GAUGE
+
+logger = logging.getLogger(__name__)
+
+FETCH_INTERVAL_SECONDS = int(os.getenv("AI_PROCESSING_INTERVAL_SECONDS", "120"))
+
+class QueueConsumerService:
+ def __init__(self, queue: RedisQueue, ai_processor: AIProcessor, fetch_interval: int = FETCH_INTERVAL_SECONDS):
+ self.queue = queue
+ self.ai_processor = ai_processor
+ self.fetch_interval = fetch_interval
+ self._running = False
+ self._task: Optional[asyncio.Task] = None
+
+ async def start(self):
+ logger.info(f"Starting Redis Queue Consumer Service (interval: {self.fetch_interval}s / {self.fetch_interval // 60}m)...")
+ await self.queue.connect()
+ self._running = True
+ self._task = asyncio.create_task(self._consumer_loop())
+
+ async def _consumer_loop(self):
+ while self._running:
+ try:
+ qsize = await self.queue.qsize()
+ QUEUE_POSTS_GAUGE.labels(status="redis_incoming").set(qsize)
+
+ if qsize > 0:
+ post_id = await self.queue.pop()
+ if post_id:
+ logger.info(f"Paced Consumer: processing post ID {post_id} from Redis queue (remaining: {qsize - 1})")
+ await self.ai_processor.process_post(post_id)
+ new_qsize = await self.queue.qsize()
+ QUEUE_POSTS_GAUGE.labels(status="redis_incoming").set(new_qsize)
+ except Exception as e:
+ logger.error(f"Error in queue consumer loop: {e}", exc_info=True)
+
+ await asyncio.sleep(self.fetch_interval)
+
+ async def stop(self):
+ self._running = False
+ if self._task:
+ self._task.cancel()
+ logger.info("Redis Queue Consumer Service stopped.")