نسخهٔ وب مستقل — پلتفرم نیروی کار هوش مصنوعی

مخزن اختصاصی برنامهٔ وب؛ محتوای پوشهٔ web در ریشهٔ مخزن قرار گرفت. تاریخچهٔ پروژهٔ ربات (agy-bot) در این مخزن وجود ندارد و هیچ وابستگی runtime به آن نیست.
2026-10-13 00:39:28 +03:30
commit 717e53a7ce
185 changed files with 39910 additions and 0 deletions
@@ -0,0 +1,202 @@
+"""Knowledge Base use cases: ingest text, chunk it, and retrieve it safely.
+
+Two things matter here:
+
+* **Chunking is deterministic.** The same text always produces the same chunks,
+  which makes ingestion idempotent-ish and the tests meaningful.
+* **Retrieval is permission-aware.** Search is always scoped to one organization,
+  and optionally narrowed to one project. An org-wide source is visible to every
+  project; a project-scoped source is not visible from another project.
+
+Embeddings are computed through ``VectorStorePort`` when a real adapter is
+configured. With the mock adapter no vectors are produced and retrieval falls
+back to keyword matching — the same fallback the plan calls for at MVP.
+"""
+
+from __future__ import annotations
+
+from app.application.ports import VectorStorePort
+from app.core.errors import NotFoundError, ValidationError
+from app.domain.knowledge.models import (
+    Document,
+    DocumentChunk,
+    EmbeddingJob,
+    EmbeddingJobStatus,
+    KnowledgeSource,
+    RetrievalPolicy,
+    SourceKind,
+)
+from app.infrastructure.repositories.knowledge import (
+    DocumentChunkRepository,
+    DocumentRepository,
+    EmbeddingJobRepository,
+    KnowledgeSourceRepository,
+    RetrievalPolicyRepository,
+)
+
+#: Chunk size in words. Small enough to keep a retrieved passage focused.
+CHUNK_WORDS = 180
+CHUNK_OVERLAP_WORDS = 30
+MAX_INGEST_CHARS = 400_000
+
+
+def chunk_text(
+    text: str, *, size: int = CHUNK_WORDS, overlap: int = CHUNK_OVERLAP_WORDS
+) -> list[str]:
+    """Split text into overlapping word windows.
+
+    Pure and deterministic — no regex surprises, no dependence on punctuation.
+    """
+    words = text.split()
+    if not words:
+        return []
+    if size <= 0:
+        raise ValueError("chunk size must be positive")
+    step = max(1, size - max(0, overlap))
+
+    chunks: list[str] = []
+    for start in range(0, len(words), step):
+        window = words[start : start + size]
+        if not window:
+            break
+        chunks.append(" ".join(window))
+        if start + size >= len(words):
+            break
+    return chunks
+
+
+class KnowledgeService:
+    def __init__(
+        self,
+        *,
+        sources: KnowledgeSourceRepository,
+        documents: DocumentRepository,
+        chunks: DocumentChunkRepository,
+        jobs: EmbeddingJobRepository,
+        policies: RetrievalPolicyRepository,
+        vector_store: VectorStorePort,
+    ) -> None:
+        self.sources = sources
+        self.documents = documents
+        self.chunks = chunks
+        self.jobs = jobs
+        self.policies = policies
+        self.vector_store = vector_store
+
+    # -- ingestion -----------------------------------------------------------
+
+    async def create_text_source(
+        self,
+        *,
+        organization_id: str,
+        name: str,
+        content: str,
+        project_id: str | None = None,
+        title: str = "",
+    ) -> tuple[KnowledgeSource, int]:
+        if not name.strip():
+            raise ValidationError("نام منبع دانش نمی‌تواند خالی باشد.")
+        if not content.strip():
+            raise ValidationError("محتوای منبع دانش نمی‌تواند خالی باشد.")
+        if len(content) > MAX_INGEST_CHARS:
+            raise ValidationError("حجم محتوا بیش از حد مجاز است.")
+
+        source = await self.sources.create(
+            KnowledgeSource(
+                organization_id=organization_id,
+                name=name.strip(),
+                kind=SourceKind.TEXT,
+                project_id=project_id,
+            )
+        )
+        document = await self.documents.create(
+            Document(source_id=source.id, title=title.strip() or source.name, content=content)
+        )
+
+        pieces = chunk_text(content)
+        chunk_rows = [
+            DocumentChunk(document_id=document.id, index=index, content=piece)
+            for index, piece in enumerate(pieces)
+        ]
+        created = await self.chunks.create_many(chunk_rows)
+
+        job = await self.jobs.create(EmbeddingJob(source_id=source.id))
+        try:
+            # The mock vector store is a no-op that accepts and discards; a real
+            # adapter embeds here. Keyword search below works either way.
+            await self.vector_store.upsert(
+                organization_id=organization_id,
+                chunks=[(chunk.id, chunk.content, []) for chunk in chunk_rows],
+                metadata={"source_id": source.id, "project_id": project_id},
+            )
+            await self.jobs.set_status(job.id, EmbeddingJobStatus.DONE)
+        except Exception as exc:  # noqa: BLE001 - embedding is best-effort at MVP
+            # Ingestion still succeeded; only the vector index is degraded, and
+            # keyword search keeps working. Record why rather than failing hard.
+            await self.jobs.set_status(job.id, EmbeddingJobStatus.FAILED, error=str(exc))
+
+        return source, created
+
+    # -- reads ---------------------------------------------------------------
+
+    async def list_sources(
+        self, organization_id: str, *, limit: int = 100, offset: int = 0
+    ) -> list[KnowledgeSource]:
+        return await self.sources.list_for_org(organization_id, limit=limit, offset=offset)
+
+    async def get_source_or_404(
+        self, *, organization_id: str, source_id: str
+    ) -> KnowledgeSource:
+        source = await self.sources.get_for_org(organization_id, source_id)
+        if source is None:
+            raise NotFoundError("منبع دانش یافت نشد.")
+        return source
+
+    async def source_detail(self, *, source_id: str) -> dict:
+        documents = await self.documents.list_for_source(source_id)
+        return {
+            "documents": documents,
+            "chunk_count": await self.chunks.count_for_source(source_id),
+            "job": await self.jobs.latest_for_source(source_id),
+        }
+
+    async def chunk_counts(self, source_ids: list[str]) -> dict[str, int]:
+        """Chunk counts for many sources at once — see ``count_for_sources``."""
+        return await self.chunks.count_for_sources(source_ids)
+
+    async def search(
+        self,
+        *,
+        organization_id: str,
+        query: str,
+        project_id: str | None = None,
+        limit: int = 5,
+    ) -> list[dict]:
+        return await self.chunks.keyword_search(
+            organization_id=organization_id,
+            query=query,
+            project_id=project_id,
+            limit=limit,
+        )
+
+    async def delete_source(self, *, organization_id: str, source_id: str) -> None:
+        source = await self.get_source_or_404(
+            organization_id=organization_id, source_id=source_id
+        )
+        await self.sources.delete(source.id)
+
+    # -- retrieval policies --------------------------------------------------
+
+    async def list_policies(self, organization_id: str) -> list[RetrievalPolicy]:
+        return await self.policies.list_for_org(organization_id)
+
+    async def create_policy(
+        self, *, organization_id: str, name: str, scope: dict | None = None
+    ) -> RetrievalPolicy:
+        if not name.strip():
+            raise ValidationError("نام سیاست بازیابی نمی‌تواند خالی باشد.")
+        return await self.policies.create(
+            RetrievalPolicy(
+                organization_id=organization_id, name=name.strip(), scope_json=dict(scope or {})
+            )
+        )