"""One employee, one turn: the loop every agent in the platform runs. ``run_turn`` is the whole agent in one function: call the model, parse its tags, execute the tools it asked for, feed the results back, repeat — until a terminal protocol tag appears, the tool-call cap is hit, the turn times out, or somebody cancels the run. The rules that make it safe to let a model drive work: * **The employee's model is a granted model.** ``ModelGrantPort.is_granted`` is checked *before* the first call; an ungranted model fails with ``error_kind=auth`` and a message for the manager, never a silent retry. * **Every model call is billed.** Usage events are written through the same ``record_model_usage`` path ``WorkforceService`` uses, so nothing the engine spends escapes the ledger — manager chat included. * **Errors are classified, not swallowed.** ``rate_limit``, ``unavailable``, ``network`` and ``timeout`` are transient and the scheduler retries them; ``auth``, ``model`` and ``protocol`` are not, and they go to the manager's error turn. The vocabulary lives in the domain (``app/domain/workforce/engine.py``), not here. * **A cancellation is honoured between steps.** The loop asks the run (and the caller's hook) before every model call and every tool batch, so ``cancel_run`` stops work in progress instead of after it. * **The turn leaves a record.** ``engine_turns`` (who ran, what came back, what it cost), ``run_steps`` (the timeline the UI renders) and ``tool_calls`` (the executions the approvals UI shows) are all written here. Only tool tags are executed here. Every other tag (``ASSIGN``, ``COMPLETE``, ``CREATE_TASK``, ``KNOWLEDGE``…) is a *decision*: the caller interprets it against the pipeline, which is what keeps this loop free of orchestration. """ from __future__ import annotations import asyncio import time from collections.abc import Awaitable, Callable from dataclasses import dataclass, field from app.application.engine.events import RunFeed from app.application.engine.prompt import ( TurnContext, TurnMessage, build_turn_prompt, protocol_error_feedback, ) from app.application.engine.protocol import TOOL_TAGS, Tag, TagName, parse_reply from app.application.engine.tools import ToolExecutor, ToolOutcome, tool_key_for from app.application.ports import ModelProviderPort, ModelResponse from app.application.ports.memory import MemoryPort, NullMemory from app.application.ports.model_grants import ModelGrantPort from app.application.ports.sandbox_provider import SandboxProviderPort from app.application.services.workforce_service import record_model_usage from app.core.config import settings from app.core.errors import ProviderUnavailableError from app.domain.ai_control.models import AiModel, RateUnit from app.domain.workforce.engine import EngineTurn, ErrorKind, is_transient from app.domain.workforce.models import ( Employee, RoleKey, Run, RunStatus, Task, ToolCall, ToolCallStatus, ) from app.domain.workforce.pipeline import Mode, Stage from app.infrastructure.repositories.ai_control import ( AiModelRepository, ModelCredentialRepository, RateCardRepository, ) from app.infrastructure.repositories.audit import UsageLedgerRepository from app.infrastructure.repositories.engine import ( EngineTurnRepository, PipelineStateRepository, ) from app.infrastructure.repositories.workforce import ( RunRepository, RunStepRepository, ToolCallRepository, ) #: The tag that ends each role's turn. A role whose reply has none of these #: keeps working: the caller decides what a reply without a decision means. TERMINAL_TAGS: dict[tuple[RoleKey, Mode], tuple[TagName, ...]] = { (RoleKey.MANAGER, Mode.TRIAGE): (TagName.ASSIGN, TagName.REJECT, TagName.ASK_USER), (RoleKey.MANAGER, Mode.FINAL): (TagName.COMPLETE, TagName.REVISE), (RoleKey.MANAGER, Mode.ERROR): (TagName.RESOLVE_ERROR, TagName.CANCEL), # Chat has no worktree: the manager's reply is consumed by the chat use # case, which is why these are the tags the chat layer looks for. (RoleKey.MANAGER, Mode.CHAT): ( TagName.CREATE_TASK, TagName.PAUSE_TASK, TagName.RESUME_TASK, TagName.CANCEL_TASK, TagName.HIRE, TagName.UPDATE_EMPLOYEE, TagName.FIRE, ), (RoleKey.PLANNER, Mode.WORK): (TagName.PLAN_SPEC,), (RoleKey.DEVELOPER, Mode.WORK): (TagName.WORK_REPORT,), (RoleKey.REVIEWER, Mode.WORK): (TagName.REVIEW_PASS, TagName.REVIEW_CHANGES), (RoleKey.TESTER, Mode.WORK): (TagName.TEST_PASS, TagName.TEST_FAIL), } @dataclass(slots=True, kw_only=True) class TurnResult: """What one turn produced — or why it could not. ``retryable`` is the scheduler's answer to "try again or escalate": it reads the domain's transient set, so a rate limit and a bad key are never treated the same way twice. """ ok: bool error_kind: str = "" error: str = "" reply_text: str = "" tags: tuple[Tag, ...] = () tool_calls: int = 0 model_calls: int = 0 #: True when the turn ended on a timeout, a cancellation or the tool cap #: rather than on a protocol decision. incomplete: bool = False turn_id: str = "" @property def retryable(self) -> bool: return bool(self.error_kind) and is_transient(self.error_kind) @dataclass(slots=True, kw_only=True) class TurnRequest: """Everything ``run_turn`` needs. One object, no hidden state.""" organization_id: str task: Task role: RoleKey mode: Mode stage: Stage run_id: str employee: Employee | None = None #: The granted model this employee runs on (``employee_profiles.ai_model_id``). model_id: str | None = None #: The worktree this turn works in, as the sandbox sees it. sandbox_root: str = "/workspace" session_external_id: str | None = None extra_context: str = "" history: list[TurnMessage] = field(default_factory=list) #: Checked between steps; returning True ends the turn as cancelled. should_cancel: Callable[[], Awaitable[bool]] | None = None user_id: str | None = None def classify_error(exc: BaseException) -> ErrorKind: """Sort a failure into the engine's error vocabulary. The providers raise one broad error type, so the classification reads the message rather than pretending each provider has its own exception tree. A failure that cannot be classified counts as ``unavailable`` (transient): retrying a genuine outage is right, and a permanent failure reveals itself on the second attempt. """ if isinstance(exc, asyncio.TimeoutError): return ErrorKind.TIMEOUT text = str(exc).lower() if "timeout" in text or "timed out" in text: return ErrorKind.TIMEOUT if "rate" in text or "429" in text or "quota" in text: return ErrorKind.RATE_LIMIT if any(word in text for word in ("auth", "credential", "unauthor", "401", "403")): return ErrorKind.AUTH if "model" in text: return ErrorKind.MODEL if any(word in text for word in ("connection", "network", "unreachable", "dns")): return ErrorKind.NETWORK return ErrorKind.UNAVAILABLE @dataclass(slots=True, kw_only=True) class BillingScope: """What a model call is billed to. One scope per call, never a default.""" organization_id: str project_id: str | None = None employee_id: str | None = None user_id: str | None = None run_id: str | None = None async def call_model( *, provider: ModelProviderPort, credentials: ModelCredentialRepository, rate_cards: RateCardRepository, ledger: UsageLedgerRepository, model: AiModel, prompt, scope: BillingScope, ) -> ModelResponse: """Call the model and bill it. **The** single model entry point. Every model call in the platform goes through here — a task turn, a triage turn, a final verdict and the manager's chat alike. Billing reads the internal ledger, never the provider's invoice, so a call that skips this function is a call the platform never charges for (agybot's manager chat bug). """ pool = await credentials.list_active_for_provider(model.provider_id) credential = pool[0] if pool else None response = await provider.complete( model=model, credential=credential, credentials=pool or None, messages=prompt.messages, system=prompt.system, ) for unit, tokens in ( (RateUnit.MODEL_INPUT_TOKENS, float(response.input_tokens)), (RateUnit.MODEL_OUTPUT_TOKENS, float(response.output_tokens)), ): await record_model_usage( ledger=ledger, rate_cards=rate_cards, organization_id=scope.organization_id, unit=unit, quantity=tokens, project_id=scope.project_id, user_id=scope.user_id, employee_id=scope.employee_id, run_id=scope.run_id, ) return response class TurnRunner: """Runs one turn at a time. One instance per process is enough.""" def __init__( self, *, provider: ModelProviderPort, models: AiModelRepository, credentials: ModelCredentialRepository, rate_cards: RateCardRepository, ledger: UsageLedgerRepository, grants: ModelGrantPort, sandbox: SandboxProviderPort | None = None, memory: MemoryPort | None = None, turns: EngineTurnRepository | None = None, run_steps: RunStepRepository | None = None, tool_calls: ToolCallRepository | None = None, runs: RunRepository | None = None, states: PipelineStateRepository | None = None, feed: RunFeed | None = None, ) -> None: self.provider = provider self.models = models self.credentials = credentials self.rate_cards = rate_cards self.ledger = ledger self.grants = grants self.sandbox = sandbox self.memory = memory or NullMemory() self.turns = turns self.run_steps = run_steps self.tool_calls = tool_calls self.runs = runs self.states = states self.feed = feed or RunFeed() # ------------------------------------------------------------------ # async def run_turn(self, request: TurnRequest) -> TurnResult: """Run one employee turn to completion (or to a clean failure).""" started = time.monotonic() deadline = started + settings.engine_turn_timeout_seconds history: list[TurnMessage] = list(request.history) tool_calls_made = 0 model_calls = 0 protocol_errors = 0 reply_text = "" tags: tuple[Tag, ...] = () turn_row = await self._open_turn_row(request) try: model = await self._resolve_model(request) except _ModelUnavailable as exc: return await self._fail( request, turn_row, exc.kind.value, str(exc), tool_calls_made, model_calls ) while True: if await self._cancelled(request): return await self._cancelled_result(request, turn_row, tool_calls_made, model_calls) if time.monotonic() > deadline: return await self._fail( request, turn_row, ErrorKind.TIMEOUT.value, "مهلت نوبت تمام شد.", tool_calls_made, model_calls, incomplete=True, ) prompt = await self._build_prompt(request, history) try: response = await self._complete(request, model, prompt) except ProviderUnavailableError as exc: return await self._fail( request, turn_row, classify_error(exc).value, str(exc), tool_calls_made, model_calls, ) model_calls += 1 reply_text = response.content self.feed.log( run_id=request.run_id, task_id=request.task.id, organization_id=request.organization_id, line=f"{request.role.value}/{request.mode.value}: {response.content[:400]}", ) parsed = parse_reply(response.content, role=request.role, mode=request.mode) if parsed.errors: protocol_errors += 1 if protocol_errors > settings.engine_max_protocol_retries: return await self._fail( request, turn_row, ErrorKind.PROTOCOL.value, "پاسخ مدل مکرراً ناقض پروتکل بود.", tool_calls_made, model_calls, incomplete=True, ) # The model learns the protocol from what it just got wrong. history.append( TurnMessage(kind="result", text=protocol_error_feedback(parsed.errors)) ) self.feed.log( run_id=request.run_id, task_id=request.task.id, organization_id=request.organization_id, line=f"خطای پروتکل: {parsed.errors[0]}", ) await self.feed.flush(run_id=request.run_id) continue requested_tools = [tag for tag in parsed.tags if tag.name in TOOL_TAGS] tags = self._terminal_tags(request, parsed.tags) if not requested_tools: # Nothing left to execute: the reply is a decision, and the # caller reads it out of ``TurnResult.tags``. break for tag in requested_tools: if tool_calls_made >= settings.engine_max_tool_calls_per_turn: return await self._fail( request, turn_row, ErrorKind.PROTOCOL.value, "سقف فراخوانی ابزار در این نوبت پر شد.", tool_calls_made, model_calls, incomplete=True, ) if await self._cancelled(request): return await self._cancelled_result( request, turn_row, tool_calls_made, model_calls ) outcome = await self._execute_tool(request, tag) tool_calls_made += 1 history.append( TurnMessage( kind="result", text=f"[[{outcome.tool}]] → {outcome.output}" ) ) await self._step( request, kind="tool", title=f"{outcome.tool}: {'موفق' if outcome.ok else 'ناموفق'}", payload={"tool": outcome.tool, "ok": outcome.ok}, ) await self.feed.flush(run_id=request.run_id) await self.feed.flush(run_id=request.run_id, force=True) await self._close_turn_row( turn_row, ok=True, tool_calls=tool_calls_made, reply=reply_text, tags=tags ) return TurnResult( ok=True, reply_text=reply_text, tags=tags, tool_calls=tool_calls_made, model_calls=model_calls, turn_id=turn_row.id if turn_row else "", ) # -- model ------------------------------------------------------------- # async def _resolve_model(self, request: TurnRequest) -> AiModel: """The employee's granted model, or a clean auth failure.""" model_id = request.model_id if not model_id: raise _ModelUnavailable( ErrorKind.AUTH, "برای این کارمند مدلی انتخاب نشده است." ) granted = await self.grants.is_granted( organization_id=request.organization_id, ai_model_id=model_id ) if not granted: raise _ModelUnavailable( ErrorKind.AUTH, "مدل این کارمند به سازمان شما اختصاص داده نشده است.", ) model = await self.models.get(model_id) if model is None or not model.is_active: raise _ModelUnavailable(ErrorKind.MODEL, "مدل این کارمند موجود نیست.") return model async def _complete(self, request: TurnRequest, model: AiModel, prompt): """One model call, billed through the one shared model entry point.""" return await call_model( provider=self.provider, credentials=self.credentials, rate_cards=self.rate_cards, ledger=self.ledger, model=model, prompt=prompt, scope=BillingScope( organization_id=request.organization_id, project_id=request.task.project_id, employee_id=request.employee.id if request.employee else None, user_id=request.user_id, run_id=request.run_id, ), ) # -- tools ------------------------------------------------------------- # async def _execute_tool(self, request: TurnRequest, tag: Tag) -> ToolOutcome: """Run one tool tag in the sandbox and record the call.""" if self.sandbox is None or not request.session_external_id: return ToolOutcome( tool=tag.name.value, ok=False, output="سندباکسی برای این نوبت در دسترس نیست.", refused=True, ) executor = ToolExecutor( sandbox=self.sandbox, session_external_id=request.session_external_id, root=request.sandbox_root, ) outcome = await executor.execute(tag) await self._record_tool_call(request, tag, outcome) await self.feed.tool( run_id=request.run_id, task_id=request.task.id, organization_id=request.organization_id, tool=outcome.tool, ok=outcome.ok, summary=outcome.output, ) return outcome async def _record_tool_call(self, request: TurnRequest, tag: Tag, outcome: ToolOutcome) -> None: """Persist the execution so the approvals UI shows real work.""" if self.tool_calls is None: return call = await self.tool_calls.create( ToolCall( run_id=request.run_id, tool_key=tool_key_for(tag), arguments_json={"tag": tag.name.value, "attrs": dict(tag.attrs)}, result_json={"ok": outcome.ok, "output": outcome.output[:4000]}, status=ToolCallStatus.SUCCEEDED if outcome.ok else ToolCallStatus.FAILED, requires_approval=False, ) ) outcome.tool_call_id = call.id # -- records ----------------------------------------------------------- # async def _open_turn_row(self, request: TurnRequest): prompt = await self._build_prompt(request, list(request.history)) if self.turns is None: return None return await self.turns.create( EngineTurn( task_id=request.task.id, run_id=request.run_id, employee_id=request.employee.id if request.employee else None, role=request.role, mode=request.mode, stage=request.stage, prompt_chars=prompt.total_chars, ) ) async def _close_turn_row( self, turn_row, *, ok: bool, tool_calls: int, reply: str, tags: tuple[Tag, ...] ) -> None: if turn_row is None or self.turns is None: return counts: dict[str, int] = {} for tag in tags: counts[tag.name.value] = counts.get(tag.name.value, 0) + 1 turn_row.ok = ok turn_row.tool_calls = tool_calls turn_row.reply_excerpt = reply[: settings.engine_reply_excerpt_chars] turn_row.tags = counts await self.turns.finish( turn_row.id, ok=ok, tool_calls=tool_calls, error_kind="", reply_excerpt=turn_row.reply_excerpt, tags=counts, ) async def _fail( self, request: TurnRequest, turn_row, error_kind: str, message: str, tool_calls: int, model_calls: int, *, incomplete: bool = False, ) -> TurnResult: await self.feed.flush(run_id=request.run_id, force=True) if turn_row is not None and self.turns is not None: turn_row.ok = False await self.turns.finish( turn_row.id, ok=False, tool_calls=tool_calls, error_kind=error_kind or ErrorKind.UNAVAILABLE.value, ) await self._step( request, kind="error", title="نوبت ناموفق بود.", payload={"error_kind": error_kind, "message": message[:2000]}, ) return TurnResult( ok=False, error_kind=error_kind, error=message, tool_calls=tool_calls, model_calls=model_calls, incomplete=incomplete, turn_id=turn_row.id if turn_row else "", ) async def _cancelled_result( self, request: TurnRequest, turn_row, tool_calls: int, model_calls: int ) -> TurnResult: return await self._fail( request, turn_row, ErrorKind.CANCELLED.value, "اجرا لغو شد.", tool_calls, model_calls, incomplete=True, ) # -- helpers ----------------------------------------------------------- # async def _build_prompt(self, request: TurnRequest, history: list[TurnMessage]): memory_context = await self.memory.build_context( organization_id=request.organization_id, role=request.role.value, query=f"{request.task.title}\n{request.task.description}", budget_chars=2500, ) return build_turn_prompt( TurnContext( organization_id=request.organization_id, task_id=request.task.id, project_id=request.task.project_id, task_title=request.task.title, task_description=request.task.description, stage=request.stage, role=request.role, mode=request.mode, memory_context=memory_context, stage_context=request.extra_context, history=history, ) ) async def _cancelled(self, request: TurnRequest) -> bool: """Ask the caller's hook and the run itself whether to stop.""" if request.should_cancel is not None and await request.should_cancel(): return True if self.runs is None: return False run: Run | None = await self.runs.get(request.run_id) return run is not None and run.status in ( RunStatus.CANCELLED, RunStatus.FAILED, RunStatus.TIMED_OUT, ) async def _step( self, request: TurnRequest, *, kind: str, title: str, payload: dict ) -> None: """Append to the run timeline the UI renders.""" if self.run_steps is None: return await self.run_steps.append( run_id=request.run_id, kind=kind, title=title, payload=payload ) @staticmethod def _terminal_tags(request: TurnRequest, tags: tuple[Tag, ...]) -> tuple[Tag, ...]: wanted = TERMINAL_TAGS.get((request.role, request.mode), ()) return tuple(tag for tag in tags if tag.name in wanted) class _ModelUnavailable(Exception): """The employee's model is missing or not granted to this organization.""" def __init__(self, kind: ErrorKind, message: str) -> None: super().__init__(message) self.kind = kind __all__ = [ "TERMINAL_TAGS", "BillingScope", "TurnRequest", "TurnResult", "TurnRunner", "call_model", "classify_error", ]