app: update service orchestration, environment configs, and documentation

.env.example

@@ -13,12 +13,18 @@ ADMIN_USER_IDS=78649634
 REVIEW_CHANNEL_ID=
 
 # =======================================================
-# AI Provider (OrcaRouter / OpenAI / Gemini)
+# AI Provider (OrcaRouter / OpenAI / Gemini / AGY)
 # =======================================================
 AI_PROVIDER=openai
 AI_BASE_URL=https://api.orcarouter.ai/v1
 AI_API_KEY=
 AI_MODEL=google/gemini-3.5-flash
+# Allow AI / automatic routing to directly enqueue posts to target channels (true/false)
+AI_AUTO_POSTING_ENABLED=true
+# Enable second-pass self-verification to fix language mixing and audit format (true/false)
+AI_DOUBLE_CHECK=false
+# Thinking / Reasoning effort for supported models (low / medium / high)
+AI_REASONING_EFFORT=
 
 # =======================================================
 # PostgreSQL (Configured for Docker Compose)
@@ -33,9 +39,21 @@ DATABASE_URL=postgresql://postgres:postgres@postgres:5432/copykar
 # =======================================================
 # Storage & Metrics
 # =======================================================
-METRICS_PORT=8000
+METRICS_PORT=8008
 MEDIA_DIR=/app/data/media
 GRAFANA_USER=admin
 GRAFANA_PASSWORD=admin
+PGADMIN_DEFAULT_EMAIL=admin@copykar.com
+PGADMIN_DEFAULT_PASSWORD=admin
+PGADMIN_PORT=5050
+
 REDIS_URL=redis://copykar_redis:6379/0
 AI_PROCESSING_INTERVAL_SECONDS=120
+
+# Tried in order when the primary AI_MODEL is rate-limited or unavailable
+AI_FALLBACK_MODELS=nvidia/nemotron-3-super-120b-a12b:free,minimax/minimax-m2.7:free,minimax/minimax-m3:free
+AI_MAX_RETRIES=2
+AI_RETRY_BACKOFF_SECONDS=2
+
+# Local wall-clock offset from UTC, used for per-target sleep windows (Iran = 3.5)
+TIMEZONE_OFFSET_HOURS=3.5

.gitignore

@@ -1,5 +1,6 @@
 .venv/
 .env
+.env.*
 *.session
 *.session-journal
 __pycache__/

docker-compose.yml

@@ -44,7 +44,7 @@ services:
       redis:
         condition: service_healthy
     ports:
-      - "8000:8000"
+      - "8008:8008"
     env_file:
       - .env
     extra_hosts:
@@ -52,6 +52,12 @@ services:
     volumes:
       - copykar_data:/app/data
       - copykar_sessions:/app/sessions
+      - /home:/host_os/home:ro
+      - /projects:/host_os/projects:ro
+      - /run/media:/host_os/media:ro
+
+
+
 
   prometheus:
     build:
@@ -79,10 +85,32 @@ services:
     depends_on:
       - prometheus
 
+
+
+  pgadmin:
+    image: dpage/pgadmin4:latest
+    container_name: copykar_pgadmin
+    restart: unless-stopped
+    environment:
+      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-admin@copykar.com}
+      PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin}
+
+      PGADMIN_CONFIG_SERVER_MODE: 'False'
+    ports:
+      - "5050:80"
+
+    volumes:
+      - pgadmin_data:/var/lib/pgadmin
+    depends_on:
+      postgres:
+        condition: service_healthy
+
 volumes:
   postgres_data:
   redis_data:
   prometheus_data:
   grafana_data:
+  pgadmin_data:
   copykar_data:
   copykar_sessions:
+

docs/superpowers/plans/2026-08-28-copykar-improvements.md

@@ -0,0 +1,102 @@
+# Copykar Improvements Implementation Plan
+
+> **For agentic workers:** Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Publish through the bot account, make history fetching controllable, allow
+automatic source→target routing, rebuild the dashboard from the real metric inventory,
+and track error resolution — with a Telegram note to admins after every task.
+
+**Architecture:** Same service layout (`collector` → `admin_bot` review → Redis per-target
+queues → `publisher`). Changes are additive: a new `targets.auto_source_ids` column drives
+auto-routing, `error_logs` gains resolution columns, and the admin bot's channel lists
+become button menus instead of one card per channel.
+
+**Tech Stack:** Python 3.12, Telethon 1.44, asyncpg/PostgreSQL 16, Redis 7,
+prometheus-client, Grafana 11.2, Docker Compose.
+
+**Spec:** User request of 2026-08-28 (8 items), recorded in the task list below.
+
+## Global Constraints
+
+- All admin-facing bot text is Persian; keep the existing emoji/HTML card style.
+- Telegram limits: media caption ≤ 1024 chars, text message ≤ 4096 chars, callback data ≤ 64 bytes.
+- No AI/model attribution in code comments or commit messages (user's GEMINI.md).
+- Minimal diffs — no drive-by refactors of untouched code.
+- Never commit `.env`. Do not commit at all without asking.
+- Every task ends with: tests pass, app rebuilt+restarted, Telegram note sent to admins.
+
+---
+
+### Task 1: Publish through the bot account, not the userbot
+
+**Files:** Modify `main.py`, `services/publisher.py`; Test `tests/test_publisher_failures.py`
+
+**Interfaces:** Produces — `PublisherService(..., client=<bot client>)`.
+
+- [ ] Pass `admin_bot.client` as the publisher client in `main.py`.
+- [ ] Publisher must not `client.start()` a client the admin bot already started.
+- [ ] Keep the permanent/transient failure split; a bot lacking rights raises
+      `ChatAdminRequiredError`, already classified permanent.
+- [ ] Update the broken-target message to say the **bot** needs admin rights.
+- [ ] Run suite, rebuild, restart, notify admins.
+
+### Task 2: Controllable history fetch
+
+**Files:** Modify `services/collector.py`, `services/admin_bot.py`; Test `tests/test_scrape_history.py`
+
+**Interfaces:** Produces — `scrape_channel_history(...) -> ScrapeResult` with fields
+`collected`, `already_stored`, `duplicates`, `scanned`.
+
+- [ ] Return a structured result instead of a bare int so the admin sees *why* nothing was added.
+- [ ] Report all four counters in the Persian progress message.
+- [ ] Add a `🔢 تعداد دلخواه` button per source → state `wait_fetch_count` → user sends a number (1–500).
+- [ ] Keep the 20/50 quick buttons.
+- [ ] Run suite, rebuild, restart, notify admins.
+
+### Task 3: Automatic source→target routing
+
+**Files:** Modify `db/database.py`, `db/repository.py`, `db/models.py`, `services/admin_bot.py`; Test `tests/test_auto_routing.py`
+
+**Interfaces:** Produces — `targets.auto_source_ids BIGINT[]`,
+`Repository.set_target_auto_sources(target_id, source_channel_ids)`,
+`Repository.get_targets_auto_routed_from(source_channel_id)`.
+
+- [ ] Add `auto_source_ids BIGINT[] DEFAULT '{}'` column + migration.
+- [ ] Target config card gains `🤖 ارسال خودکار از مبدا` button showing every source as a toggle.
+- [ ] After the review card is sent, auto-route: rewrite for each subscribed target and enqueue.
+- [ ] The admin review card is still posted — auto-routing never replaces it.
+- [ ] Run suite, rebuild, restart, notify admins.
+
+### Task 4: Channel lists as button menus
+
+**Files:** Modify `services/admin_bot.py`
+
+- [ ] `/sources` and `/targets` send ONE message with one button per channel.
+- [ ] `src_view:{id}` / `trg_view:{id}` open that channel's card with its options.
+- [ ] Every card gets a back button to its list.
+- [ ] Run suite, rebuild, restart, notify admins.
+
+### Task 5: Error resolution tracking + caption fix
+
+**Files:** Modify `db/database.py`, `db/repository.py`, `core/metrics.py`, `core/error_logger.py`, `services/admin_bot.py`; Test `tests/test_error_tracking.py`
+
+**Interfaces:** Produces — `error_logs.resolved/resolved_at/resolved_note`,
+`Repository.get_open_error_summary()`, `Repository.resolve_errors(error_type, service_name, note)`,
+gauges `copykar_errors_open`, counter `copykar_errors_resolved_total`.
+
+- [ ] Add resolution columns + migration.
+- [ ] Fix `MediaCaptionTooLongError`: clamp preview/review captions to Telegram limits.
+- [ ] `/errors` command lists open error groups with a `✅ رفع شد` button per group.
+- [ ] Publish `copykar_errors_open` and `copykar_errors_resolved_total`; refresh the gauge periodically.
+- [ ] Mark the already-fixed historical errors resolved.
+- [ ] Run suite, rebuild, restart, notify admins.
+
+### Task 6: Rebuild the Grafana dashboard
+
+**Files:** Rewrite `monitoring/grafana/dashboards/copykar.json`
+
+- [ ] Rebuild from the live metric inventory; every metric appears exactly once.
+- [ ] Remove the duplicated queue reading (`copykar_redis_queue_size` total vs per-target gauge).
+- [ ] Max 3 panels per row, Persian titles + descriptions.
+- [ ] Include the new error-health panels.
+- [ ] Restart Grafana, verify panels return data, notify admins.

main.py

@@ -28,9 +28,10 @@ async def main():
     logger.info("Starting Copykar System with Immediate Admin Review & Paced Target Queues...")
 
     # 1. Start Prometheus metrics server
-    metrics_port = int(os.getenv("METRICS_PORT", "8000"))
+    metrics_port = int(os.getenv("METRICS_PORT", "8008"))
     start_metrics_server(metrics_port)
 
+
     # 2. Initialize Database schema & Redis Queue
     await init_db()
     logger.info("Database schema initialized.")
@@ -39,16 +40,29 @@ async def main():
     await redis_queue.connect()
 
     repo = Repository()
-    llm = LLMClient()
+    await repo.ensure_default_providers()
 
     # 3. Create Services
+    admin_bot = AdminBotService(repo=repo, queue=redis_queue)
+    llm = LLMClient(
+        repo=repo,
+        on_fallback_alert=admin_bot.on_ai_fallback_alert,
+        on_chain_failure_alert=admin_bot.on_ai_chain_failure_alert
+    )
     ai_processor = AIProcessor(repo=repo, llm=llm)
-    admin_bot = AdminBotService(repo=repo, ai_processor=ai_processor, queue=redis_queue)
+    admin_bot.set_ai_processor(ai_processor)
-    collector = CollectorService(repo=repo, on_post_received=admin_bot.send_raw_review_post)
+    collector = CollectorService(repo=repo, on_post_received=admin_bot.handle_collected_post)
     admin_bot.set_collector(collector)
     
-    # Publisher handles per-target delivery queues, intervals, and sleep windows
+    # Publisher handles per-target delivery queues, intervals, and sleep windows.
-    publisher = PublisherService(repo=repo, queue=redis_queue, client=collector.client)
+    # Delivery goes out over the bot account so target channels only need to grant
+    # posting rights to the bot, never to the personal userbot account.
+    publisher = PublisherService(
+        repo=repo,
+        queue=redis_queue,
+        client=admin_bot.client,
+        notify_fn=admin_bot.notify_admins,
+    )
 
     # 4. Start all services
     await admin_bot.start()
@@ -56,6 +70,10 @@ async def main():
     await publisher.start()
 
     logger.info("All Copykar services are active and running.")
+    try:
+        await admin_bot.broadcast_change_notes()
+    except Exception as e:
+        logger.debug(f"Could not broadcast change notes on startup: {e}")
 
     # Graceful shutdown event
     stop_event = asyncio.Event()

requirements.txt

@@ -5,3 +5,5 @@ prometheus-client==0.26.0
 httpx==0.28.1
 python-socks==3.0.0
 redis==8.1.0
+matplotlib==3.9.2
+