From ad4c111318641aceb3e1917e59ee38327955eb14 Mon Sep 17 00:00:00 2001 From: mamad Date: Fri, 28 Aug 2026 19:35:02 +0330 Subject: [PATCH] app: update service orchestration, environment configs, and documentation --- .env.example | 22 +++- .gitignore | 1 + docker-compose.yml | 30 +++++- .../plans/2026-08-28-copykar-improvements.md | 102 ++++++++++++++++++ main.py | 30 ++++-- requirements.txt | 2 + 6 files changed, 178 insertions(+), 9 deletions(-) create mode 100644 docs/superpowers/plans/2026-08-28-copykar-improvements.md diff --git a/.env.example b/.env.example index 3cfb627..eaacc08 100644 --- a/.env.example +++ b/.env.example @@ -13,12 +13,18 @@ ADMIN_USER_IDS=78649634 REVIEW_CHANNEL_ID= # ======================================================= -# AI Provider (OrcaRouter / OpenAI / Gemini) +# AI Provider (OrcaRouter / OpenAI / Gemini / AGY) # ======================================================= AI_PROVIDER=openai AI_BASE_URL=https://api.orcarouter.ai/v1 AI_API_KEY= AI_MODEL=google/gemini-3.5-flash +# Allow AI / automatic routing to directly enqueue posts to target channels (true/false) +AI_AUTO_POSTING_ENABLED=true +# Enable second-pass self-verification to fix language mixing and audit format (true/false) +AI_DOUBLE_CHECK=false +# Thinking / Reasoning effort for supported models (low / medium / high) +AI_REASONING_EFFORT= # ======================================================= # PostgreSQL (Configured for Docker Compose) @@ -33,9 +39,21 @@ DATABASE_URL=postgresql://postgres:postgres@postgres:5432/copykar # ======================================================= # Storage & Metrics # ======================================================= -METRICS_PORT=8000 +METRICS_PORT=8008 MEDIA_DIR=/app/data/media GRAFANA_USER=admin GRAFANA_PASSWORD=admin +PGADMIN_DEFAULT_EMAIL=admin@copykar.com +PGADMIN_DEFAULT_PASSWORD=admin +PGADMIN_PORT=5050 + REDIS_URL=redis://copykar_redis:6379/0 AI_PROCESSING_INTERVAL_SECONDS=120 + +# Tried in order when the primary AI_MODEL is rate-limited or unavailable +AI_FALLBACK_MODELS=nvidia/nemotron-3-super-120b-a12b:free,minimax/minimax-m2.7:free,minimax/minimax-m3:free +AI_MAX_RETRIES=2 +AI_RETRY_BACKOFF_SECONDS=2 + +# Local wall-clock offset from UTC, used for per-target sleep windows (Iran = 3.5) +TIMEZONE_OFFSET_HOURS=3.5 diff --git a/.gitignore b/.gitignore index 5ffee73..6c647e6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .venv/ .env +.env.* *.session *.session-journal __pycache__/ diff --git a/docker-compose.yml b/docker-compose.yml index 607b43a..e36d009 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,7 +44,7 @@ services: redis: condition: service_healthy ports: - - "8000:8000" + - "8008:8008" env_file: - .env extra_hosts: @@ -52,6 +52,12 @@ services: volumes: - copykar_data:/app/data - copykar_sessions:/app/sessions + - /home:/host_os/home:ro + - /projects:/host_os/projects:ro + - /run/media:/host_os/media:ro + + + prometheus: build: @@ -79,10 +85,32 @@ services: depends_on: - prometheus + + + pgadmin: + image: dpage/pgadmin4:latest + container_name: copykar_pgadmin + restart: unless-stopped + environment: + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-admin@copykar.com} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin} + + PGADMIN_CONFIG_SERVER_MODE: 'False' + ports: + - "5050:80" + + volumes: + - pgadmin_data:/var/lib/pgadmin + depends_on: + postgres: + condition: service_healthy + volumes: postgres_data: redis_data: prometheus_data: grafana_data: + pgadmin_data: copykar_data: copykar_sessions: + diff --git a/docs/superpowers/plans/2026-08-28-copykar-improvements.md b/docs/superpowers/plans/2026-08-28-copykar-improvements.md new file mode 100644 index 0000000..e378511 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-copykar-improvements.md @@ -0,0 +1,102 @@ +# Copykar Improvements Implementation Plan + +> **For agentic workers:** Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Publish through the bot account, make history fetching controllable, allow +automatic source→target routing, rebuild the dashboard from the real metric inventory, +and track error resolution — with a Telegram note to admins after every task. + +**Architecture:** Same service layout (`collector` → `admin_bot` review → Redis per-target +queues → `publisher`). Changes are additive: a new `targets.auto_source_ids` column drives +auto-routing, `error_logs` gains resolution columns, and the admin bot's channel lists +become button menus instead of one card per channel. + +**Tech Stack:** Python 3.12, Telethon 1.44, asyncpg/PostgreSQL 16, Redis 7, +prometheus-client, Grafana 11.2, Docker Compose. + +**Spec:** User request of 2026-08-28 (8 items), recorded in the task list below. + +## Global Constraints + +- All admin-facing bot text is Persian; keep the existing emoji/HTML card style. +- Telegram limits: media caption ≤ 1024 chars, text message ≤ 4096 chars, callback data ≤ 64 bytes. +- No AI/model attribution in code comments or commit messages (user's GEMINI.md). +- Minimal diffs — no drive-by refactors of untouched code. +- Never commit `.env`. Do not commit at all without asking. +- Every task ends with: tests pass, app rebuilt+restarted, Telegram note sent to admins. + +--- + +### Task 1: Publish through the bot account, not the userbot + +**Files:** Modify `main.py`, `services/publisher.py`; Test `tests/test_publisher_failures.py` + +**Interfaces:** Produces — `PublisherService(..., client=)`. + +- [ ] Pass `admin_bot.client` as the publisher client in `main.py`. +- [ ] Publisher must not `client.start()` a client the admin bot already started. +- [ ] Keep the permanent/transient failure split; a bot lacking rights raises + `ChatAdminRequiredError`, already classified permanent. +- [ ] Update the broken-target message to say the **bot** needs admin rights. +- [ ] Run suite, rebuild, restart, notify admins. + +### Task 2: Controllable history fetch + +**Files:** Modify `services/collector.py`, `services/admin_bot.py`; Test `tests/test_scrape_history.py` + +**Interfaces:** Produces — `scrape_channel_history(...) -> ScrapeResult` with fields +`collected`, `already_stored`, `duplicates`, `scanned`. + +- [ ] Return a structured result instead of a bare int so the admin sees *why* nothing was added. +- [ ] Report all four counters in the Persian progress message. +- [ ] Add a `🔢 تعداد دلخواه` button per source → state `wait_fetch_count` → user sends a number (1–500). +- [ ] Keep the 20/50 quick buttons. +- [ ] Run suite, rebuild, restart, notify admins. + +### Task 3: Automatic source→target routing + +**Files:** Modify `db/database.py`, `db/repository.py`, `db/models.py`, `services/admin_bot.py`; Test `tests/test_auto_routing.py` + +**Interfaces:** Produces — `targets.auto_source_ids BIGINT[]`, +`Repository.set_target_auto_sources(target_id, source_channel_ids)`, +`Repository.get_targets_auto_routed_from(source_channel_id)`. + +- [ ] Add `auto_source_ids BIGINT[] DEFAULT '{}'` column + migration. +- [ ] Target config card gains `🤖 ارسال خودکار از مبدا` button showing every source as a toggle. +- [ ] After the review card is sent, auto-route: rewrite for each subscribed target and enqueue. +- [ ] The admin review card is still posted — auto-routing never replaces it. +- [ ] Run suite, rebuild, restart, notify admins. + +### Task 4: Channel lists as button menus + +**Files:** Modify `services/admin_bot.py` + +- [ ] `/sources` and `/targets` send ONE message with one button per channel. +- [ ] `src_view:{id}` / `trg_view:{id}` open that channel's card with its options. +- [ ] Every card gets a back button to its list. +- [ ] Run suite, rebuild, restart, notify admins. + +### Task 5: Error resolution tracking + caption fix + +**Files:** Modify `db/database.py`, `db/repository.py`, `core/metrics.py`, `core/error_logger.py`, `services/admin_bot.py`; Test `tests/test_error_tracking.py` + +**Interfaces:** Produces — `error_logs.resolved/resolved_at/resolved_note`, +`Repository.get_open_error_summary()`, `Repository.resolve_errors(error_type, service_name, note)`, +gauges `copykar_errors_open`, counter `copykar_errors_resolved_total`. + +- [ ] Add resolution columns + migration. +- [ ] Fix `MediaCaptionTooLongError`: clamp preview/review captions to Telegram limits. +- [ ] `/errors` command lists open error groups with a `✅ رفع شد` button per group. +- [ ] Publish `copykar_errors_open` and `copykar_errors_resolved_total`; refresh the gauge periodically. +- [ ] Mark the already-fixed historical errors resolved. +- [ ] Run suite, rebuild, restart, notify admins. + +### Task 6: Rebuild the Grafana dashboard + +**Files:** Rewrite `monitoring/grafana/dashboards/copykar.json` + +- [ ] Rebuild from the live metric inventory; every metric appears exactly once. +- [ ] Remove the duplicated queue reading (`copykar_redis_queue_size` total vs per-target gauge). +- [ ] Max 3 panels per row, Persian titles + descriptions. +- [ ] Include the new error-health panels. +- [ ] Restart Grafana, verify panels return data, notify admins. diff --git a/main.py b/main.py index ed47dc4..5ac5606 100644 --- a/main.py +++ b/main.py @@ -28,9 +28,10 @@ async def main(): logger.info("Starting Copykar System with Immediate Admin Review & Paced Target Queues...") # 1. Start Prometheus metrics server - metrics_port = int(os.getenv("METRICS_PORT", "8000")) + metrics_port = int(os.getenv("METRICS_PORT", "8008")) start_metrics_server(metrics_port) + # 2. Initialize Database schema & Redis Queue await init_db() logger.info("Database schema initialized.") @@ -39,16 +40,29 @@ async def main(): await redis_queue.connect() repo = Repository() - llm = LLMClient() + await repo.ensure_default_providers() # 3. Create Services + admin_bot = AdminBotService(repo=repo, queue=redis_queue) + llm = LLMClient( + repo=repo, + on_fallback_alert=admin_bot.on_ai_fallback_alert, + on_chain_failure_alert=admin_bot.on_ai_chain_failure_alert + ) ai_processor = AIProcessor(repo=repo, llm=llm) - admin_bot = AdminBotService(repo=repo, ai_processor=ai_processor, queue=redis_queue) - collector = CollectorService(repo=repo, on_post_received=admin_bot.send_raw_review_post) + admin_bot.set_ai_processor(ai_processor) + collector = CollectorService(repo=repo, on_post_received=admin_bot.handle_collected_post) admin_bot.set_collector(collector) - # Publisher handles per-target delivery queues, intervals, and sleep windows - publisher = PublisherService(repo=repo, queue=redis_queue, client=collector.client) + # Publisher handles per-target delivery queues, intervals, and sleep windows. + # Delivery goes out over the bot account so target channels only need to grant + # posting rights to the bot, never to the personal userbot account. + publisher = PublisherService( + repo=repo, + queue=redis_queue, + client=admin_bot.client, + notify_fn=admin_bot.notify_admins, + ) # 4. Start all services await admin_bot.start() @@ -56,6 +70,10 @@ async def main(): await publisher.start() logger.info("All Copykar services are active and running.") + try: + await admin_bot.broadcast_change_notes() + except Exception as e: + logger.debug(f"Could not broadcast change notes on startup: {e}") # Graceful shutdown event stop_event = asyncio.Event() diff --git a/requirements.txt b/requirements.txt index 60fe689..6e27c55 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,5 @@ prometheus-client==0.26.0 httpx==0.28.1 python-socks==3.0.0 redis==8.1.0 +matplotlib==3.9.2 +