smart-org/apps/api/app/workers/ai_runtime.py

"""AI runtime worker — drives queued tasks through the run engine.

The HTTP API can start a run inline (that is what the "اجرا" button does), but
a platform that only ever runs work on the request thread has two problems: a
slow model call holds a connection open, and a task created by anything other
than a button press never runs at all. This worker is the second driver.

It deliberately owns **no** orchestration logic. It finds runnable tasks and
hands each one to ``WorkforceService.start_run`` — the same call the router
makes. Model routing, usage accounting, the timeline and the artifact are all
the service's business.
"""

from __future__ import annotations

from app.application.container import build_workforce_service
from app.application.container_engine import build_scheduler
from app.application.ports.realtime import emit_worker_status
from app.core.errors import AppError
from app.domain.audit.models import ErrorSeverity, ErrorSource
from app.domain.workforce.models import TaskStatus
from app.infrastructure.error_log import record_error
from app.infrastructure.repositories.workforce import TaskRepository
from app.workers.base import WorkerOutcome, worker_session


class AiRuntimeWorker:
    """Drains the queue of tasks that nobody has run yet.

    The worker drives **two** engines in one pass, oldest first:

    * the legacy run engine — runnable tasks handed to
      ``WorkforceService.start_run``;
    * the workforce engine (Phase 4) — one ``Scheduler.tick()``, which moves
      every ready task one stage forward (a turn, a gate, or a platform merge).

    Both run on the same session, so a run started here and a stage advanced
    here commit together. ``run_once`` (the CLI default) makes one pass — the
    cron shape; ``--loop`` keeps ticking (set the interval to
    ``ENGINE_TICK_SECONDS``).
    """

    name = "ai-runtime"

    async def drain(self, *, limit: int = 10) -> WorkerOutcome:
        """Run up to ``limit`` queued tasks, then one engine tick.

        A task that fails is recorded and the pass continues: one poisoned task
        must not block the queue behind it.
        """
        outcome = WorkerOutcome(name=self.name)

        async with worker_session() as session:
            await self._drain_runs(session, outcome, limit=limit)
            await self._tick_engine(session, outcome)

        return outcome

    async def _drain_runs(self, session, outcome: WorkerOutcome, *, limit: int) -> None:
        tasks = TaskRepository(session)
        workforce = build_workforce_service(session)

        for task in await tasks.list_runnable(limit=limit):
            # Re-read through the service so the tenant check and the
            # "already closed" guard both apply. The worker gets no
            # privileged bypass.
            try:
                current = await workforce.get_task_or_404(
                    organization_id=task.organization_id, task_id=task.id
                )
            except AppError:
                outcome.skipped += 1
                outcome.note(f"task {task.id}: vanished before the run")
                continue

            if current.status not in (TaskStatus.NEW, TaskStatus.TRIAGED):
                outcome.skipped += 1
                continue

            try:
                run = await workforce.start_run(
                    organization_id=task.organization_id,
                    task_id=task.id,
                    user_id=task.created_by_user_id or "",
                    employee_id=task.assigned_employee_id,
                )
            except Exception as exc:  # noqa: BLE001 - keep draining
                outcome.failed += 1
                outcome.note(f"task {task.id}: {type(exc).__name__}: {exc}")
                # Persist with the tenant attached: a task that fails to run
                # is a tenant-visible problem, and the console must be able
                # to show that organization what happened.
                await record_error(
                    kind="task_run_failed",
                    message=str(exc) or type(exc).__name__,
                    exc=exc,
                    source=ErrorSource.WORKER,
                    severity=ErrorSeverity.ERROR,
                    organization_id=task.organization_id,
                    user_id=task.created_by_user_id,
                    metadata={
                        "worker": self.name,
                        "task_id": task.id,
                        "employee_id": task.assigned_employee_id,
                    },
                )
                continue

            outcome.processed += 1
            outcome.note(f"task {task.id} -> run {run.id} ({run.status})")
            # This worker is its own process: without a push here, a run
            # started by the queue is invisible until someone reloads.
            await emit_worker_status(
                organization_id=task.organization_id,
                worker=self.name,
                message=f"کار «{current.title}» در حال اجراست (اجرا {run.id}).",
            )

    async def _tick_engine(self, session, outcome: WorkerOutcome) -> None:
        """One scheduler tick: move every ready task one step.

        A tick failure is recorded against the worker and the pass ends — the
        run drain has already committed its own work, and a broken tick must
        not look like a clean one.
        """
        try:
            scheduler = build_scheduler(session)
            # A wide read: states that cannot run this pass (automation off, or nobody free
            # for their stage) must not fill the window and starve the other organizations.
            # Scheduler._select still bounds the real work per organization and per pass.
            report = await scheduler.tick(limit=500)
        except Exception as exc:  # noqa: BLE001 - a worker must survive a bad tick
            outcome.failed += 1
            outcome.note(f"engine tick: {type(exc).__name__}: {exc}")
            await record_error(
                kind="engine_tick_failed",
                message=str(exc) or type(exc).__name__,
                exc=exc,
                source=ErrorSource.WORKER,
                severity=ErrorSeverity.ERROR,
                metadata={"worker": self.name},
            )
            return

        outcome.processed += report.acted + report.merged
        outcome.skipped += report.skipped
        outcome.failed += report.failed
        if report.selected or report.merged or report.paused:
            outcome.note(
                f"engine tick: selected={report.selected} acted={report.acted} "
                f"merged={report.merged} paused={report.paused} failed={report.failed}"
            )
        for note in report.notes:
            outcome.note(note)


__all__ = ["AiRuntimeWorker"]