feat(engine): Phase 4 — scheduler tick, gate chain, platform git, budget

The AI workforce engine's driver. One tick moves every ready task one step: a turn, a gate, or a platform merge. The AI never runs git. - scheduler.py: tick() over TURN_STAGES/MERGE_STAGES; selection caps (per-org 3, global 8); optimistic advance so two workers cannot both win; worktree created for code pipelines, released on completion/cancel/failure (kept during error_review); budget pause before any turn is spent; merge conflict back-off; transient-failure retries then the manager's error turn. - gates.py: diff guard (empty/.git/CI/symlink/oversize), no-progress hash, tests in the sandbox, green gate that only trusts a green tests row or an explicit waiver. Every verdict is a gate_results row + a task.gate event. Fixed forbidden_paths: lstrip("./") stripped all leading dots, hiding .git and .github from the guard. - container_engine.build_scheduler + worker drives the engine tick. - ManagerOutcome.error_kind; TaskSpendRepository.spend_for_task. - New settings engine_max_concurrent_turns, engine_max_turns_per_org, engine_tests_timeout_seconds, engine_max_merge_conflicts, engine_max_task_credits (+ .env.example). - tests: test_engine_gates.py (13), test_engine_flow.py (11). Checks: pytest 746 passed/1 skipped, ruff clean, mypy clean (160 files).
2026-10-14 19:02:58 +03:30
parent 581f703910
commit 36564e3f2b
13 changed files with 2563 additions and 67 deletions
@@ -32,8 +32,11 @@ def _build_parser() -> argparse.ArgumentParser:
    parser.add_argument(
        "--interval",
        type=float,
-        default=300.0,
-        help="seconds between passes when --loop is set (default: 300)",
+        default=None,
+        help=(
+            "seconds between passes when --loop is set "
+            "(default: 300, or the engine tick for ai-runtime)"
+        ),
    )
    parser.add_argument("--limit", type=int, default=10, help="max rows per pass (default: 10)")
    parser.add_argument(
@@ -121,9 +124,20 @@ async def _main_async(args: argparse.Namespace) -> int:
    if args.loop:
        install_signal_handlers(stop)

+    interval = args.interval
+    if interval is None:
+        # The engine's driver wants to tick often; every other worker is a
+        # slower rollup and the historical 300s default suits it.
+        if args.worker == "ai-runtime":
+            from app.core.config import settings
+
+            interval = settings.engine_tick_seconds
+        else:
+            interval = 300.0
+
    outcomes = await run_periodically(
        one_pass,
-        interval_seconds=max(1.0, args.interval),
+        interval_seconds=max(1.0, interval),
        stop=stop,
        run_once=not args.loop,
        worker_name=args.worker,
@@ -14,6 +14,7 @@ the service's business.
 from __future__ import annotations

 from app.application.container import build_workforce_service
+from app.application.container_engine import build_scheduler
 from app.application.ports.realtime import emit_worker_status
 from app.core.errors import AppError
 from app.domain.audit.models import ErrorSeverity, ErrorSource
@@ -24,12 +25,25 @@ from app.workers.base import WorkerOutcome, worker_session


 class AiRuntimeWorker:
-    """Drains the queue of tasks that nobody has run yet."""
+    """Drains the queue of tasks that nobody has run yet.
+
+    The worker drives **two** engines in one pass, oldest first:
+
+    * the legacy run engine — runnable tasks handed to
+      ``WorkforceService.start_run``;
+    * the workforce engine (Phase 4) — one ``Scheduler.tick()``, which moves
+      every ready task one stage forward (a turn, a gate, or a platform merge).
+
+    Both run on the same session, so a run started here and a stage advanced
+    here commit together. ``run_once`` (the CLI default) makes one pass — the
+    cron shape; ``--loop`` keeps ticking (set the interval to
+    ``ENGINE_TICK_SECONDS``).
+    """

    name = "ai-runtime"

    async def drain(self, *, limit: int = 10) -> WorkerOutcome:
-        """Run up to ``limit`` queued tasks, oldest first.
+        """Run up to ``limit`` queued tasks, then one engine tick.

        A task that fails is recorded and the pass continues: one poisoned task
        must not block the queue behind it.
@@ -37,66 +51,104 @@ class AiRuntimeWorker:
        outcome = WorkerOutcome(name=self.name)

        async with worker_session() as session:
-            tasks = TaskRepository(session)
-            workforce = build_workforce_service(session)
-
-            for task in await tasks.list_runnable(limit=limit):
-                # Re-read through the service so the tenant check and the
-                # "already closed" guard both apply. The worker gets no
-                # privileged bypass.
-                try:
-                    current = await workforce.get_task_or_404(
-                        organization_id=task.organization_id, task_id=task.id
-                    )
-                except AppError:
-                    outcome.skipped += 1
-                    outcome.note(f"task {task.id}: vanished before the run")
-                    continue
-
-                if current.status not in (TaskStatus.NEW, TaskStatus.TRIAGED):
-                    outcome.skipped += 1
-                    continue
-
-                try:
-                    run = await workforce.start_run(
-                        organization_id=task.organization_id,
-                        task_id=task.id,
-                        user_id=task.created_by_user_id or "",
-                        employee_id=task.assigned_employee_id,
-                    )
-                except Exception as exc:  # noqa: BLE001 - keep draining
-                    outcome.failed += 1
-                    outcome.note(f"task {task.id}: {type(exc).__name__}: {exc}")
-                    # Persist with the tenant attached: a task that fails to run
-                    # is a tenant-visible problem, and the console must be able
-                    # to show that organization what happened.
-                    await record_error(
-                        kind="task_run_failed",
-                        message=str(exc) or type(exc).__name__,
-                        exc=exc,
-                        source=ErrorSource.WORKER,
-                        severity=ErrorSeverity.ERROR,
-                        organization_id=task.organization_id,
-                        user_id=task.created_by_user_id,
-                        metadata={
-                            "worker": self.name,
-                            "task_id": task.id,
-                            "employee_id": task.assigned_employee_id,
-                        },
-                    )
-                    continue
-
-                outcome.processed += 1
-                outcome.note(f"task {task.id} -> run {run.id} ({run.status})")
-                # This worker is its own process: without a push here, a run
-                # started by the queue is invisible until someone reloads.
-                await emit_worker_status(
-                    organization_id=task.organization_id,
-                    worker=self.name,
-                    message=f"کار «{current.title}» در حال اجراست (اجرا {run.id}).",
-                )
+            await self._drain_runs(session, outcome, limit=limit)
+            await self._tick_engine(session, outcome, limit=limit)

        return outcome

+    async def _drain_runs(self, session, outcome: WorkerOutcome, *, limit: int) -> None:
+        tasks = TaskRepository(session)
+        workforce = build_workforce_service(session)
+
+        for task in await tasks.list_runnable(limit=limit):
+            # Re-read through the service so the tenant check and the
+            # "already closed" guard both apply. The worker gets no
+            # privileged bypass.
+            try:
+                current = await workforce.get_task_or_404(
+                    organization_id=task.organization_id, task_id=task.id
+                )
+            except AppError:
+                outcome.skipped += 1
+                outcome.note(f"task {task.id}: vanished before the run")
+                continue
+
+            if current.status not in (TaskStatus.NEW, TaskStatus.TRIAGED):
+                outcome.skipped += 1
+                continue
+
+            try:
+                run = await workforce.start_run(
+                    organization_id=task.organization_id,
+                    task_id=task.id,
+                    user_id=task.created_by_user_id or "",
+                    employee_id=task.assigned_employee_id,
+                )
+            except Exception as exc:  # noqa: BLE001 - keep draining
+                outcome.failed += 1
+                outcome.note(f"task {task.id}: {type(exc).__name__}: {exc}")
+                # Persist with the tenant attached: a task that fails to run
+                # is a tenant-visible problem, and the console must be able
+                # to show that organization what happened.
+                await record_error(
+                    kind="task_run_failed",
+                    message=str(exc) or type(exc).__name__,
+                    exc=exc,
+                    source=ErrorSource.WORKER,
+                    severity=ErrorSeverity.ERROR,
+                    organization_id=task.organization_id,
+                    user_id=task.created_by_user_id,
+                    metadata={
+                        "worker": self.name,
+                        "task_id": task.id,
+                        "employee_id": task.assigned_employee_id,
+                    },
+                )
+                continue
+
+            outcome.processed += 1
+            outcome.note(f"task {task.id} -> run {run.id} ({run.status})")
+            # This worker is its own process: without a push here, a run
+            # started by the queue is invisible until someone reloads.
+            await emit_worker_status(
+                organization_id=task.organization_id,
+                worker=self.name,
+                message=f"کار «{current.title}» در حال اجراست (اجرا {run.id}).",
+            )
+
+    async def _tick_engine(self, session, outcome: WorkerOutcome, *, limit: int) -> None:
+        """One scheduler tick: move every ready task one step.
+
+        A tick failure is recorded against the worker and the pass ends — the
+        run drain has already committed its own work, and a broken tick must
+        not look like a clean one.
+        """
+        try:
+            scheduler = build_scheduler(session)
+            report = await scheduler.tick(limit=max(limit, 10))
+        except Exception as exc:  # noqa: BLE001 - a worker must survive a bad tick
+            outcome.failed += 1
+            outcome.note(f"engine tick: {type(exc).__name__}: {exc}")
+            await record_error(
+                kind="engine_tick_failed",
+                message=str(exc) or type(exc).__name__,
+                exc=exc,
+                source=ErrorSource.WORKER,
+                severity=ErrorSeverity.ERROR,
+                metadata={"worker": self.name},
+            )
+            return
+
+        outcome.processed += report.acted + report.merged
+        outcome.skipped += report.skipped
+        outcome.failed += report.failed
+        if report.selected or report.merged or report.paused:
+            outcome.note(
+                f"engine tick: selected={report.selected} acted={report.acted} "
+                f"merged={report.merged} paused={report.paused} failed={report.failed}"
+            )
+        for note in report.notes:
+            outcome.note(note)
+

 __all__ = ["AiRuntimeWorker"]