"""AI runtime worker — drives queued tasks through the run engine. The HTTP API can start a run inline (that is what the "اجرا" button does), but a platform that only ever runs work on the request thread has two problems: a slow model call holds a connection open, and a task created by anything other than a button press never runs at all. This worker is the second driver. It deliberately owns **no** orchestration logic. It finds runnable tasks and hands each one to ``WorkforceService.start_run`` — the same call the router makes. Model routing, usage accounting, the timeline and the artifact are all the service's business. """ from __future__ import annotations from app.application.container import build_workforce_service from app.application.container_engine import build_scheduler from app.application.ports.realtime import emit_worker_status from app.core.errors import AppError from app.domain.audit.models import ErrorSeverity, ErrorSource from app.domain.workforce.models import TaskStatus from app.infrastructure.error_log import record_error from app.infrastructure.repositories.workforce import TaskRepository from app.workers.base import WorkerOutcome, worker_session class AiRuntimeWorker: """Drains the queue of tasks that nobody has run yet. The worker drives **two** engines in one pass, oldest first: * the legacy run engine — runnable tasks handed to ``WorkforceService.start_run``; * the workforce engine (Phase 4) — one ``Scheduler.tick()``, which moves every ready task one stage forward (a turn, a gate, or a platform merge). Both run on the same session, so a run started here and a stage advanced here commit together. ``run_once`` (the CLI default) makes one pass — the cron shape; ``--loop`` keeps ticking (set the interval to ``ENGINE_TICK_SECONDS``). """ name = "ai-runtime" async def drain(self, *, limit: int = 10) -> WorkerOutcome: """Run up to ``limit`` queued tasks, then one engine tick. A task that fails is recorded and the pass continues: one poisoned task must not block the queue behind it. """ outcome = WorkerOutcome(name=self.name) async with worker_session() as session: await self._drain_runs(session, outcome, limit=limit) await self._tick_engine(session, outcome) return outcome async def _drain_runs(self, session, outcome: WorkerOutcome, *, limit: int) -> None: tasks = TaskRepository(session) workforce = build_workforce_service(session) for task in await tasks.list_runnable(limit=limit): # Re-read through the service so the tenant check and the # "already closed" guard both apply. The worker gets no # privileged bypass. try: current = await workforce.get_task_or_404( organization_id=task.organization_id, task_id=task.id ) except AppError: outcome.skipped += 1 outcome.note(f"task {task.id}: vanished before the run") continue if current.status not in (TaskStatus.NEW, TaskStatus.TRIAGED): outcome.skipped += 1 continue try: run = await workforce.start_run( organization_id=task.organization_id, task_id=task.id, user_id=task.created_by_user_id or "", employee_id=task.assigned_employee_id, ) except Exception as exc: # noqa: BLE001 - keep draining outcome.failed += 1 outcome.note(f"task {task.id}: {type(exc).__name__}: {exc}") # Persist with the tenant attached: a task that fails to run # is a tenant-visible problem, and the console must be able # to show that organization what happened. await record_error( kind="task_run_failed", message=str(exc) or type(exc).__name__, exc=exc, source=ErrorSource.WORKER, severity=ErrorSeverity.ERROR, organization_id=task.organization_id, user_id=task.created_by_user_id, metadata={ "worker": self.name, "task_id": task.id, "employee_id": task.assigned_employee_id, }, ) continue outcome.processed += 1 outcome.note(f"task {task.id} -> run {run.id} ({run.status})") # This worker is its own process: without a push here, a run # started by the queue is invisible until someone reloads. await emit_worker_status( organization_id=task.organization_id, worker=self.name, message=f"کار «{current.title}» در حال اجراست (اجرا {run.id}).", ) async def _tick_engine(self, session, outcome: WorkerOutcome) -> None: """One scheduler tick: move every ready task one step. A tick failure is recorded against the worker and the pass ends — the run drain has already committed its own work, and a broken tick must not look like a clean one. """ try: scheduler = build_scheduler(session) # A wide read: states that cannot run this pass (automation off, or nobody free # for their stage) must not fill the window and starve the other organizations. # Scheduler._select still bounds the real work per organization and per pass. report = await scheduler.tick(limit=500) except Exception as exc: # noqa: BLE001 - a worker must survive a bad tick outcome.failed += 1 outcome.note(f"engine tick: {type(exc).__name__}: {exc}") await record_error( kind="engine_tick_failed", message=str(exc) or type(exc).__name__, exc=exc, source=ErrorSource.WORKER, severity=ErrorSeverity.ERROR, metadata={"worker": self.name}, ) return outcome.processed += report.acted + report.merged outcome.skipped += report.skipped outcome.failed += report.failed if report.selected or report.merged or report.paused: outcome.note( f"engine tick: selected={report.selected} acted={report.acted} " f"merged={report.merged} paused={report.paused} failed={report.failed}" ) for note in report.notes: outcome.note(note) __all__ = ["AiRuntimeWorker"]