feat: add source websites with automated endpoint analysis, target queue dispatch ordering, and markdown styling

db/database.py

@@ -33,6 +33,7 @@ CREATE TABLE IF NOT EXISTS targets (
     is_sleep_enabled BOOLEAN DEFAULT FALSE,
     auto_source_ids BIGINT[] DEFAULT '{}',
     language VARCHAR(32) DEFAULT 'fa',
+    dispatch_order VARCHAR(32) DEFAULT 'order',
     last_post_time TIMESTAMPTZ,
     is_active BOOLEAN DEFAULT TRUE,
     created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
@@ -131,6 +132,24 @@ CREATE TABLE IF NOT EXISTS channel_categories (
     created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
 );
 
+CREATE TABLE IF NOT EXISTS source_websites (
+    id SERIAL PRIMARY KEY,
+    name VARCHAR(255) NOT NULL,
+    url TEXT NOT NULL UNIQUE,
+    category_id INT REFERENCES channel_categories(id) ON DELETE SET NULL,
+    check_interval_min INT DEFAULT 30,
+    auto_reanalyze_hours INT DEFAULT 24,
+    last_reanalyzed_at TIMESTAMPTZ,
+    last_fetched_at TIMESTAMPTZ,
+    api_config JSONB DEFAULT '{}'::jsonb,
+    last_error TEXT,
+    last_error_at TIMESTAMPTZ,
+    is_active BOOLEAN DEFAULT TRUE,
+    created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX IF NOT EXISTS idx_source_websites_active ON source_websites(is_active);
+
 CREATE INDEX IF NOT EXISTS idx_ai_providers_active ON ai_providers(is_active);
 
 -- Migration safety for existing tables
@@ -152,6 +171,7 @@ ALTER TABLE targets ADD COLUMN IF NOT EXISTS is_sleep_enabled BOOLEAN DEFAULT FA
 ALTER TABLE targets ADD COLUMN IF NOT EXISTS auto_source_ids BIGINT[] DEFAULT '{}';
 ALTER TABLE targets ADD COLUMN IF NOT EXISTS language VARCHAR(32) DEFAULT 'fa';
 ALTER TABLE targets ADD COLUMN IF NOT EXISTS custom_prompt TEXT DEFAULT '';
+ALTER TABLE targets ADD COLUMN IF NOT EXISTS dispatch_order VARCHAR(32) DEFAULT 'order';
 ALTER TABLE sources ADD COLUMN IF NOT EXISTS context_message_count INT DEFAULT 0;
 ALTER TABLE posts ADD COLUMN IF NOT EXISTS source_created_at TIMESTAMPTZ;
 CREATE INDEX IF NOT EXISTS idx_posts_source_created ON posts(source_channel_id, source_created_at DESC);

db/models.py

@@ -20,6 +20,22 @@ class SourceChannel:
     is_active: bool = True
     created_at: Optional[str] = None
 
+@dataclass
+class SourceWebsite:
+    id: Optional[int]
+    name: str
+    url: str
+    category_id: Optional[int] = None
+    check_interval_min: int = 30
+    auto_reanalyze_hours: int = 24
+    last_reanalyzed_at: Optional[str] = None
+    last_fetched_at: Optional[str] = None
+    api_config: Dict[str, Any] = field(default_factory=dict)
+    last_error: Optional[str] = None
+    last_error_at: Optional[str] = None
+    is_active: bool = True
+    created_at: Optional[str] = None
+
 @dataclass
 class TargetChannel:
     id: Optional[int]
@@ -37,6 +53,7 @@ class TargetChannel:
     # Telegram channel_ids of sources whose posts are queued to this target automatically.
     auto_source_ids: List[int] = field(default_factory=list)
     language: str = "fa"
+    dispatch_order: str = "order"  # "order" (FIFO) or "random"
     last_post_time: Optional[str] = None
     is_active: bool = True
     created_at: Optional[str] = None

db/repository.py

@@ -2,7 +2,7 @@ import json
 import asyncpg
 from datetime import datetime, timezone
 from typing import List, Optional, Dict, Any, Tuple
-from db.models import SourceChannel, TargetChannel, Post, Setting, AILog, AIProviderProfile, ChannelCategory
+from db.models import SourceChannel, SourceWebsite, TargetChannel, Post, Setting, AILog, AIProviderProfile, ChannelCategory
 from db.database import get_db_pool
 
 
@@ -17,6 +17,18 @@ def _parse_post_row(row: asyncpg.Record) -> Post:
         data["published_to"] = []
     return Post(**data)
 
+
+def _parse_source_website_row(row: asyncpg.Record) -> SourceWebsite:
+    data = dict(row)
+    if isinstance(data.get("api_config"), str):
+        try:
+            data["api_config"] = json.loads(data["api_config"])
+        except Exception:
+            data["api_config"] = {}
+    elif data.get("api_config") is None:
+        data["api_config"] = {}
+    return SourceWebsite(**data)
+
 class Repository:
     def __init__(self, dsn: Optional[str] = None):
         self.dsn = dsn
@@ -72,6 +84,117 @@ class Repository:
         async with pool.acquire() as conn:
             await conn.execute("UPDATE sources SET is_active = FALSE WHERE id = $1;", source_id)
 
+    # --- Source Websites ---
+    async def add_source_website(
+        self,
+        name: str,
+        url: str,
+        category_id: Optional[int] = None,
+        check_interval_min: int = 30,
+        auto_reanalyze_hours: int = 24,
+        api_config: Optional[Dict[str, Any]] = None
+    ) -> int:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            cfg_json = json.dumps(api_config or {})
+            row = await conn.fetchrow(
+                """
+                INSERT INTO source_websites (name, url, category_id, check_interval_min, auto_reanalyze_hours, api_config)
+                VALUES ($1, $2, $3, $4, $5, $6::jsonb)
+                ON CONFLICT(url) DO UPDATE SET
+                    name = EXCLUDED.name,
+                    category_id = COALESCE(EXCLUDED.category_id, source_websites.category_id),
+                    is_active = TRUE
+                RETURNING id;
+                """,
+                name, url, category_id, check_interval_min, auto_reanalyze_hours, cfg_json
+            )
+            return row["id"]
+
+    async def get_active_source_websites(self) -> List[SourceWebsite]:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            rows = await conn.fetch("SELECT * FROM source_websites WHERE is_active = TRUE ORDER BY id ASC;")
+            return [_parse_source_website_row(r) for r in rows]
+
+    async def get_source_websites(self) -> List[SourceWebsite]:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            rows = await conn.fetch("SELECT * FROM source_websites WHERE is_active = TRUE ORDER BY id ASC;")
+            return [_parse_source_website_row(r) for r in rows]
+
+    async def get_source_website_by_id(self, site_id: int) -> Optional[SourceWebsite]:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            row = await conn.fetchrow("SELECT * FROM source_websites WHERE id = $1;", site_id)
+            return _parse_source_website_row(row) if row else None
+
+    async def get_source_websites_by_category(self, category_id: Optional[int]) -> List[SourceWebsite]:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            if category_id is None:
+                rows = await conn.fetch("SELECT * FROM source_websites WHERE is_active = TRUE AND category_id IS NULL ORDER BY id ASC;")
+            else:
+                rows = await conn.fetch("SELECT * FROM source_websites WHERE is_active = TRUE AND category_id = $1 ORDER BY id ASC;", category_id)
+            return [_parse_source_website_row(r) for r in rows]
+
+    async def update_source_website_category(self, site_id: int, category_id: Optional[int]) -> None:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            await conn.execute("UPDATE source_websites SET category_id = $1 WHERE id = $2;", category_id, site_id)
+
+    async def update_source_website_api_config(self, site_id: int, api_config: Dict[str, Any]) -> None:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            await conn.execute(
+                """
+                UPDATE source_websites
+                SET api_config = $1::jsonb,
+                    last_reanalyzed_at = CURRENT_TIMESTAMP,
+                    last_error = NULL,
+                    last_error_at = NULL
+                WHERE id = $2;
+                """,
+                json.dumps(api_config), site_id
+            )
+
+    async def update_source_website_fetch_status(self, site_id: int, error: Optional[str] = None) -> None:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            if error:
+                await conn.execute(
+                    """
+                    UPDATE source_websites
+                    SET last_error = $1, last_error_at = CURRENT_TIMESTAMP
+                    WHERE id = $2;
+                    """,
+                    error, site_id
+                )
+            else:
+                await conn.execute(
+                    """
+                    UPDATE source_websites
+                    SET last_fetched_at = CURRENT_TIMESTAMP, last_error = NULL, last_error_at = NULL
+                    WHERE id = $1;
+                    """,
+                    site_id
+                )
+
+    async def update_source_website_interval(self, site_id: int, interval_min: int) -> None:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            await conn.execute("UPDATE source_websites SET check_interval_min = $1 WHERE id = $2;", max(1, interval_min), site_id)
+
+    async def update_source_website_reanalyze_hours(self, site_id: int, hours: int) -> None:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            await conn.execute("UPDATE source_websites SET auto_reanalyze_hours = $1 WHERE id = $2;", max(0, hours), site_id)
+
+    async def delete_source_website(self, site_id: int) -> None:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            await conn.execute("UPDATE source_websites SET is_active = FALSE WHERE id = $1;", site_id)
+
     async def delete_target(self, target_id: int) -> None:
         pool = await self._get_pool()
         async with pool.acquire() as conn:
@@ -142,6 +265,15 @@ class Repository:
                 custom_prompt, target_id
             )
 
+    async def update_target_dispatch_order(self, target_id: int, dispatch_order: str) -> None:
+        order = "random" if str(dispatch_order).strip().lower() == "random" else "order"
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            await conn.execute(
+                "UPDATE targets SET dispatch_order = $1 WHERE id = $2;",
+                order, target_id
+            )
+
 
     async def update_target_schedule(
         self,