feat: realtime WebSocket layer for live events and streaming logs

- Backend: ConnectionManager, Redis-backed bus with in-process fallback, channel authorization, and typed emit helpers wired into workers (ai_runtime, sandbox_gateway), workforce/sandbox/billing/audit/error services. - Frontend: single shared WS client with backoff + heartbeat, RealtimeProvider, useLiveChannel/useLiveEvent hooks, live indicators on task detail, manager chat, dashboard, traces, sandbox terminal, admin errors, and billing. - Contracts: WsEvent/WsServerFrame/WsClientFrame types mirroring the port. All 255 backend tests pass; web typecheck and build pass.
2026-10-13 12:56:36 +03:30
parent afdef4b444
commit f1e5df29ce
31 changed files with 2981 additions and 28 deletions
@@ -0,0 +1,504 @@
+"""Realtime push contract: the event envelope and the typed publishers.
+
+This module is an **application port**. Services depend on it; the WebSocket
+adapter in ``app.interfaces.realtime`` implements it. That direction matters:
+the run engine must be able to publish without knowing that a browser exists,
+and a worker process that never serves a socket still publishes the same way.
+
+Two parts:
+
+* **The envelope** (:class:`RealtimeEvent`) and the channel naming scheme.
+  Framework-free, like every other port.
+* **The typed publishers** — ``emit_run_step`` and friends. They own the
+  fan-out channels and the one rule that makes live feeds safe to call from
+  anywhere: *publishing never raises*. A feed must not be able to fail a task
+  run, a sandbox command or a billing write.
+
+The publisher is process-wide state, set once by a composition root
+(``app.main`` for the API, ``app.workers.__main__`` for the workers). Until
+then it is a :class:`NullPublisher` — publishing is a no-op rather than a
+missing-import crash.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Protocol, runtime_checkable
+
+from app.domain.base import new_id, utcnow
+
+logger = logging.getLogger("agy.realtime")
+
+#: The platform-owner console stream. Authorized against ``is_platform_owner``.
+ADMIN_CHANNEL = "admin"
+
+
+# ---------------------------------------------------------------------------
+# Channels
+# ---------------------------------------------------------------------------
+
+
+def org_channel(organization_id: str) -> str:
+    """Everything happening inside one tenant."""
+    return f"org:{organization_id}"
+
+
+def task_channel(task_id: str) -> str:
+    return f"task:{task_id}"
+
+
+def run_channel(run_id: str) -> str:
+    return f"run:{run_id}"
+
+
+def thread_channel(thread_id: str) -> str:
+    return f"thread:{thread_id}"
+
+
+def session_channel(session_id: str) -> str:
+    """One sandbox session — the live command terminal subscribes here."""
+    return f"session:{session_id}"
+
+
+def user_channel(user_id: str) -> str:
+    return f"user:{user_id}"
+
+
+def admin_channel() -> str:
+    return ADMIN_CHANNEL
+
+
+def dedupe_channels(channels: Iterable[str]) -> tuple[str, ...]:
+    """Preserve order, drop duplicates — a publisher may overlap channels."""
+    seen: dict[str, None] = {}
+    for channel in channels:
+        if channel and channel not in seen:
+            seen[channel] = None
+    return tuple(seen)
+
+
+def with_org_channel(channels: Sequence[str], organization_id: str | None) -> tuple[str, ...]:
+    """Append the tenant channel unless the caller already listed it."""
+    if not organization_id:
+        return dedupe_channels(channels)
+    return dedupe_channels([*channels, org_channel(organization_id)])
+
+
+# ---------------------------------------------------------------------------
+# Envelope
+# ---------------------------------------------------------------------------
+
+
+@dataclass(slots=True, kw_only=True)
+class RealtimeEvent:
+    """One push to one or more channels.
+
+    ``channels`` is the fan-out list — the same event typically goes to the
+    narrow channel (``run:…``) and the wide one (``org:…``), so a page watching
+    a single run and a dashboard watching the whole tenant both stay live
+    without the publisher caring who is listening.
+    """
+
+    type: str
+    channels: tuple[str, ...]
+    data: dict = field(default_factory=dict)
+    organization_id: str | None = None
+    id: str = field(default_factory=new_id)
+    at: datetime = field(default_factory=utcnow)
+
+    def envelope(self) -> dict:
+        """The JSON object a browser receives.
+
+        ``at`` is serialised as an ISO-8601 UTC string: the client renders it
+        with ``toLocaleString`` like every other timestamp in the product.
+        """
+        return {
+            "id": self.id,
+            "type": self.type,
+            "data": self.data,
+            "organization_id": self.organization_id,
+            "at": self.at.isoformat(),
+        }
+
+    def to_json(self) -> str:
+        return json.dumps(self.envelope(), ensure_ascii=False, default=str)
+
+    @classmethod
+    def from_json(cls, raw: str | bytes) -> RealtimeEvent:
+        payload = json.loads(raw)
+        return cls(
+            type=str(payload["type"]),
+            channels=tuple(payload.get("channels") or ()),
+            data=dict(payload.get("data") or {}),
+            organization_id=payload.get("organization_id"),
+            id=str(payload["id"]),
+            at=_parse_at(payload.get("at")),
+        )
+
+
+def _parse_at(value: object) -> datetime:
+    if isinstance(value, str) and value:
+        try:
+            return datetime.fromisoformat(value)
+        except ValueError:
+            return utcnow()
+    return utcnow()
+
+
+# ---------------------------------------------------------------------------
+# Publisher plumbing
+# ---------------------------------------------------------------------------
+
+
+@runtime_checkable
+class EventPublisher(Protocol):
+    async def publish(self, event: RealtimeEvent) -> None: ...
+
+
+class NullPublisher:
+    """Default before a composition root wires the real bus."""
+
+    async def publish(self, event: RealtimeEvent) -> None:
+        return None
+
+
+_PUBLISHER: EventPublisher = NullPublisher()
+
+
+def set_publisher(publisher: EventPublisher) -> None:
+    """Install the process-wide publisher. Called by composition roots only."""
+    global _PUBLISHER
+    _PUBLISHER = publisher
+
+
+def get_publisher() -> EventPublisher:
+    return _PUBLISHER
+
+
+async def publish(event: RealtimeEvent) -> None:
+    """Publish through the installed publisher. Never raises."""
+    try:
+        await _PUBLISHER.publish(event)
+    except Exception:  # noqa: BLE001 - the feed must never fail a caller
+        logger.exception("realtime: publish failed for %s", event.type)
+
+
+async def _emit(
+    *,
+    type: str,
+    channels: tuple[str, ...],
+    data: dict,
+    organization_id: str | None = None,
+) -> None:
+    await publish(
+        RealtimeEvent(
+            type=type,
+            channels=with_org_channel(channels, organization_id),
+            data=data,
+            organization_id=organization_id,
+        )
+    )
+
+
+# ---------------------------------------------------------------------------
+# Typed publishers
+#
+# Payload shape is the product contract with the browser: the client-side types
+# in ``packages/contracts`` mirror ``data`` key-for-key. Keep the two in sync.
+# ---------------------------------------------------------------------------
+
+#: Long text fields are clipped in the live feed. The full record stays
+#: available through the REST endpoints — the feed is a signal, not an archive.
+MAX_LIVE_TEXT_CHARS = 8000
+
+
+def _clip(text: str) -> str:
+    return text[:MAX_LIVE_TEXT_CHARS]
+
+
+async def emit_run_step(
+    *,
+    run_id: str,
+    task_id: str,
+    organization_id: str,
+    step_id: str,
+    index: int,
+    kind: str,
+    title: str,
+    payload: dict | None = None,
+) -> None:
+    await _emit(
+        type="run.step",
+        channels=(run_channel(run_id), task_channel(task_id)),
+        organization_id=organization_id,
+        data={
+            "run_id": run_id,
+            "task_id": task_id,
+            "step": {
+                "id": step_id,
+                "index": index,
+                "kind": kind,
+                "title": title,
+                "payload": payload or {},
+            },
+        },
+    )
+
+
+async def emit_run_status(
+    *,
+    run_id: str,
+    task_id: str,
+    organization_id: str,
+    status: str,
+    error: str | None = None,
+) -> None:
+    await _emit(
+        type="run.status",
+        channels=(run_channel(run_id), task_channel(task_id)),
+        organization_id=organization_id,
+        data={"run_id": run_id, "task_id": task_id, "status": status, "error": error},
+    )
+
+
+async def emit_task_status(*, task_id: str, organization_id: str, status: str) -> None:
+    await _emit(
+        type="task.status",
+        channels=(task_channel(task_id),),
+        organization_id=organization_id,
+        data={"task_id": task_id, "status": status},
+    )
+
+
+async def emit_task_created(
+    *, task_id: str, organization_id: str, title: str, status: str
+) -> None:
+    await _emit(
+        type="task.created",
+        channels=(task_channel(task_id),),
+        organization_id=organization_id,
+        data={"task_id": task_id, "title": title, "status": status},
+    )
+
+
+async def emit_chat_pending(*, thread_id: str, organization_id: str) -> None:
+    """The manager is thinking — the client shows a typing indicator."""
+    await _emit(
+        type="chat.pending",
+        channels=(thread_channel(thread_id),),
+        organization_id=organization_id,
+        data={"thread_id": thread_id},
+    )
+
+
+async def emit_message_created(
+    *,
+    thread_id: str,
+    organization_id: str,
+    message_id: str,
+    role: str,
+    content: str,
+    employee_id: str | None = None,
+) -> None:
+    await _emit(
+        type="message.created",
+        channels=(thread_channel(thread_id),),
+        organization_id=organization_id,
+        data={
+            "thread_id": thread_id,
+            "message": {
+                "id": message_id,
+                "role": role,
+                "content": _clip(content),
+                "employee_id": employee_id,
+            },
+        },
+    )
+
+
+async def emit_approval_pending(
+    *,
+    run_id: str,
+    task_id: str,
+    organization_id: str,
+    tool_call_id: str,
+    tool_key: str,
+    requires_approval: bool,
+) -> None:
+    await _emit(
+        type="approval.pending",
+        channels=(run_channel(run_id), task_channel(task_id)),
+        organization_id=organization_id,
+        data={
+            "run_id": run_id,
+            "task_id": task_id,
+            "tool_call_id": tool_call_id,
+            "tool_key": tool_key,
+            "requires_approval": requires_approval,
+        },
+    )
+
+
+async def emit_artifact_created(
+    *, run_id: str, task_id: str, organization_id: str, artifact_id: str, name: str
+) -> None:
+    await _emit(
+        type="artifact.created",
+        channels=(run_channel(run_id), task_channel(task_id)),
+        organization_id=organization_id,
+        data={
+            "run_id": run_id,
+            "task_id": task_id,
+            "artifact_id": artifact_id,
+            "name": name,
+        },
+    )
+
+
+async def emit_sandbox_command(
+    *,
+    session_id: str,
+    organization_id: str,
+    command_run_id: str,
+    command: str,
+    exit_code: int | None,
+    stdout: str,
+    stderr: str,
+    duration_ms: int,
+    truncated: bool,
+    status: str = "finished",
+) -> None:
+    """One command's output, pushed live to the session's terminal panel."""
+    await _emit(
+        type="sandbox.command",
+        channels=(session_channel(session_id),),
+        organization_id=organization_id,
+        data={
+            "session_id": session_id,
+            "run_id": command_run_id,
+            "command": _clip(command),
+            "status": status,
+            "exit_code": exit_code,
+            "stdout": _clip(stdout),
+            "stderr": _clip(stderr),
+            "duration_ms": duration_ms,
+            "truncated": truncated,
+        },
+    )
+
+
+async def emit_sandbox_session(
+    *,
+    session_id: str,
+    organization_id: str,
+    status: str,
+    workspace_id: str | None = None,
+) -> None:
+    await _emit(
+        type="sandbox.session",
+        channels=(session_channel(session_id),),
+        organization_id=organization_id,
+        data={
+            "session_id": session_id,
+            "status": status,
+            "workspace_id": workspace_id,
+        },
+    )
+
+
+async def emit_sandbox_reaped(
+    *, organization_id: str | None, session_id: str, workspace_id: str
+) -> None:
+    await _emit(
+        type="sandbox.reaped",
+        channels=(session_channel(session_id),),
+        organization_id=organization_id,
+        data={"session_id": session_id, "workspace_id": workspace_id},
+    )
+
+
+async def emit_usage_recorded(
+    *,
+    organization_id: str,
+    unit: str,
+    quantity: float,
+    run_id: str | None = None,
+    employee_id: str | None = None,
+) -> None:
+    await _emit(
+        type="usage.recorded",
+        channels=(),
+        organization_id=organization_id,
+        data={
+            "unit": unit,
+            "quantity": quantity,
+            "run_id": run_id,
+            "employee_id": employee_id,
+        },
+    )
+
+
+async def emit_wallet_changed(*, organization_id: str, balance: float) -> None:
+    await _emit(
+        type="billing.wallet",
+        channels=(),
+        organization_id=organization_id,
+        data={"organization_id": organization_id, "balance": balance},
+    )
+
+
+async def emit_error_recorded(
+    *,
+    error_id: str | None,
+    kind: str,
+    message: str,
+    severity: str,
+    source: str,
+    organization_id: str | None = None,
+) -> None:
+    """Feed the platform-owner console; tenants see their own through the org
+    channel the envelope carries."""
+    await _emit(
+        type="error.recorded",
+        channels=("admin",),
+        organization_id=organization_id,
+        data={
+            "error_id": error_id,
+            "kind": kind,
+            "message": _clip(message),
+            "severity": severity,
+            "source": source,
+            "organization_id": organization_id,
+        },
+    )
+
+
+async def emit_audit_recorded(
+    *, organization_id: str | None, action: str, target_type: str, target_id: str
+) -> None:
+    if not organization_id:
+        return
+    await _emit(
+        type="audit.recorded",
+        channels=(),
+        organization_id=organization_id,
+        data={"action": action, "target_type": target_type, "target_id": target_id},
+    )
+
+
+async def emit_worker_status(
+    *,
+    organization_id: str,
+    worker: str,
+    message: str,
+) -> None:
+    await _emit(
+        type="worker.status",
+        channels=(),
+        organization_id=organization_id,
+        data={"worker": worker, "message": _clip(message)},
+    )
@@ -8,6 +8,7 @@ from __future__ import annotations

 from sqlalchemy.ext.asyncio import AsyncSession

+from app.application.ports.realtime import emit_audit_recorded
 from app.domain.audit.models import AuditEvent
 from app.infrastructure.repositories.audit import AuditRepository

@@ -34,4 +35,13 @@ async def record_audit(
        ip_address=ip,
        user_agent=user_agent,
    )
-    return await AuditRepository(session).record(event=event)
+    recorded = await AuditRepository(session).record(event=event)
+    # Live push: the audit timelines (dashboard, traces) update as actions
+    # happen instead of waiting for the next reload.
+    await emit_audit_recorded(
+        organization_id=organization_id,
+        action=action,
+        target_type=target_type,
+        target_id=target_id,
+    )
+    return recorded
@@ -14,6 +14,7 @@ from __future__ import annotations
 from datetime import datetime

 from app.application.ports import PaymentProviderPort
+from app.application.ports.realtime import emit_wallet_changed
 from app.core.errors import NotFoundError, PermissionDeniedError, ValidationError
 from app.domain.billing.models import (
    CreditWallet,
@@ -121,6 +122,11 @@ class BillingService:
            # Refuse to leave a wallet in debt; undo and surface the conflict.
            await self.wallets.adjust(organization_id=organization_id, delta=-amount)
            raise ValidationError("اعتبار کافی برای این کسر وجود ندارد.")
+        # Live push: the sidebar credit badge and the billing page follow the
+        # wallet as it moves, whoever moved it.
+        await emit_wallet_changed(
+            organization_id=organization_id, balance=wallet.balance
+        )
        return wallet

    async def start_checkout(
@@ -177,8 +183,13 @@ class BillingService:
            return payment

        if succeeded:
-            await self.wallets.adjust(organization_id=organization_id, delta=payment.amount)
+            wallet = await self.wallets.adjust(
+                organization_id=organization_id, delta=payment.amount
+            )
            final = await self.payments.set_status(payment_id, PaymentStatus.SUCCEEDED)
+            await emit_wallet_changed(
+                organization_id=organization_id, balance=wallet.balance
+            )
        else:
            final = await self.payments.set_status(payment_id, PaymentStatus.FAILED)
        return final or payment
@@ -18,6 +18,10 @@ import contextlib
 from datetime import datetime

 from app.application.ports import CommandResult, SandboxProviderPort, UsageRecord
+from app.application.ports.realtime import (
+    emit_sandbox_command,
+    emit_sandbox_session,
+)
 from app.core.config import settings
 from app.core.errors import ConflictError, NotFoundError, ProviderUnavailableError, ValidationError
 from app.domain.ai_control.models import RateUnit
@@ -210,6 +214,12 @@ class SandboxService:
            )
        )
        await self.workspaces.set_status(workspace.id, WorkspaceStatus.READY)
+        await emit_sandbox_session(
+            session_id=session.id,
+            organization_id=organization_id,
+            status=SessionStatus.RUNNING.value,
+            workspace_id=workspace.id,
+        )
        return session

    async def get_session_or_404(
@@ -229,6 +239,11 @@ class SandboxService:
        await self.provider.destroy(session_external_id=session.external_session_id)
        await self.sessions.set_status(session.id, SessionStatus.STOPPED, stopped=True)
        refreshed = await self.sessions.get(session.id)
+        await emit_sandbox_session(
+            session_id=session.id,
+            organization_id=organization_id,
+            status=SessionStatus.STOPPED.value,
+        )
        return refreshed or session

    # -- commands ------------------------------------------------------------
@@ -285,6 +300,20 @@ class SandboxService:
        )
        await self.sessions.set_status(session.id, SessionStatus.RUNNING)

+        # Live push: the terminal panel on the project page streams the command
+        # and its output as the row is written, not on the next poll.
+        await emit_sandbox_command(
+            session_id=session.id,
+            organization_id=organization_id,
+            command_run_id=run.id,
+            command=cleaned,
+            exit_code=result.exit_code,
+            stdout=run.stdout_ref,
+            stderr=run.stderr_ref,
+            duration_ms=result.duration_ms,
+            truncated=result.truncated,
+        )
+
        # CPU seconds are billed. 1 second of wall time ~= 1 CPU second at MVP.
        cpu_seconds = max(0.0, result.duration_ms / 1000.0)
        if cpu_seconds > 0:
@@ -24,6 +24,17 @@ from app.application.ports import (
    ModelProviderPort,
    UsageRecord,
 )
+from app.application.ports.realtime import (
+    emit_approval_pending,
+    emit_artifact_created,
+    emit_chat_pending,
+    emit_message_created,
+    emit_run_status,
+    emit_run_step,
+    emit_task_created,
+    emit_task_status,
+    emit_usage_recorded,
+)
 from app.core.errors import (
    ConflictError,
    NotFoundError,
@@ -324,6 +335,10 @@ class WorkforceService:
        else:
            employee = await self._default_manager(organization_id)

+        # The manager is about to call a model. Tell the thread so the UI can
+        # show a thinking indicator immediately.
+        await emit_chat_pending(thread_id=thread.id, organization_id=organization_id)
+
        history = await self.messages.list_for_thread(thread.id)
        system_prompt = await self._system_prompt_for(employee)
        reply = await self._complete(
@@ -347,6 +362,16 @@ class WorkforceService:
                employee_id=employee.id if employee else None,
            )
        )
+        # Live push: any browser on this thread sees the reply arrive, and the
+        # pending indicator clears, without waiting for a reload.
+        await emit_message_created(
+            thread_id=thread.id,
+            organization_id=organization_id,
+            message_id=assistant_message.id,
+            role=MessageRole.ASSISTANT.value,
+            content=reply,
+            employee_id=employee.id if employee else None,
+        )
        return user_message, assistant_message

    # ------------------------------------------------------------------ #
@@ -371,7 +396,7 @@ class WorkforceService:
            await self.get_employee_or_404(
                organization_id=organization_id, employee_id=assigned_employee_id
            )
-        return await self.tasks.create(
+        task = await self.tasks.create(
            Task(
                organization_id=organization_id,
                title=title.strip(),
@@ -382,6 +407,15 @@ class WorkforceService:
                created_by_user_id=created_by_user_id,
            )
        )
+        # Live push: the tasks list and the manager-chat sidebar update the
+        # moment the task exists, whoever created it and from where.
+        await emit_task_created(
+            task_id=task.id,
+            organization_id=organization_id,
+            title=task.title,
+            status=task.status.value,
+        )
+        return task

    async def list_tasks(
        self,
@@ -435,21 +469,39 @@ class WorkforceService:
            Run(task_id=task.id, employee_id=employee.id if employee else None)
        )
        await self.tasks.set_status(task.id, TaskStatus.RUNNING)
+        await emit_task_status(
+            task_id=task.id, organization_id=organization_id, status=TaskStatus.RUNNING.value
+        )

-        await self.run_steps.append(
-            run_id=run.id, kind="queued", title="کار در صف اجرا قرار گرفت.",
-            payload={"task_id": task.id},
+        await self._publish_step(
+            organization_id=organization_id,
+            run_id=run.id,
+            task_id=task.id,
+            step=await self.run_steps.append(
+                run_id=run.id, kind="queued", title="کار در صف اجرا قرار گرفت.",
+                payload={"task_id": task.id},
+            ),
        )

        try:
            run = await self._advance(run, RunStatus.PLANNING)
-            await self.run_steps.append(
-                run_id=run.id, kind="planning", title="برنامه‌ریزی مرحلهٔ اجرا."
+            await self._publish_step(
+                organization_id=organization_id,
+                run_id=run.id,
+                task_id=task.id,
+                step=await self.run_steps.append(
+                    run_id=run.id, kind="planning", title="برنامه‌ریزی مرحلهٔ اجرا."
+                ),
            )

            run = await self._advance(run, RunStatus.RUNNING)
-            await self.run_steps.append(
-                run_id=run.id, kind="running", title="اجرای کار آغاز شد."
+            await self._publish_step(
+                organization_id=organization_id,
+                run_id=run.id,
+                task_id=task.id,
+                step=await self.run_steps.append(
+                    run_id=run.id, kind="running", title="اجرای کار آغاز شد."
+                ),
            )

            reply = await self._complete(
@@ -463,9 +515,14 @@ class WorkforceService:
                    ChatMessage(role="user", content=self._task_brief(task)),
                ],
            )
-            await self.run_steps.append(
-                run_id=run.id, kind="output", title="خروجی آماده شد.",
-                payload={"chars": len(reply)},
+            await self._publish_step(
+                organization_id=organization_id,
+                run_id=run.id,
+                task_id=task.id,
+                step=await self.run_steps.append(
+                    run_id=run.id, kind="output", title="خروجی آماده شد.",
+                    payload={"chars": len(reply)},
+                ),
            )

            artifact = await self.artifacts.create(
@@ -479,30 +536,106 @@ class WorkforceService:
                    size_bytes=len(reply.encode("utf-8")),
                )
            )
-            await self.run_steps.append(
-                run_id=run.id, kind="artifact", title="سند خروجی ساخته شد.",
-                payload={"artifact_id": artifact.id},
+            await self._publish_step(
+                organization_id=organization_id,
+                run_id=run.id,
+                task_id=task.id,
+                step=await self.run_steps.append(
+                    run_id=run.id, kind="artifact", title="سند خروجی ساخته شد.",
+                    payload={"artifact_id": artifact.id},
+                ),
+            )
+            await emit_artifact_created(
+                run_id=run.id,
+                task_id=task.id,
+                organization_id=organization_id,
+                artifact_id=artifact.id,
+                name=artifact.name,
            )

            run = await self._finish(run, RunStatus.COMPLETED)
            await self.tasks.set_status(task.id, TaskStatus.DONE)
+            await emit_run_status(
+                run_id=run.id,
+                task_id=task.id,
+                organization_id=organization_id,
+                status=RunStatus.COMPLETED.value,
+            )
+            await emit_task_status(
+                task_id=task.id, organization_id=organization_id, status=TaskStatus.DONE.value
+            )
        except ProviderUnavailableError as exc:
            run = await self._finish(run, RunStatus.FAILED, error=str(exc))
            await self.tasks.set_status(task.id, TaskStatus.FAILED)
-            await self.run_steps.append(
-                run_id=run.id, kind="error", title="اجرا ناموفق بود.",
-                payload={"error": str(exc)},
+            await self._publish_step(
+                organization_id=organization_id,
+                run_id=run.id,
+                task_id=task.id,
+                step=await self.run_steps.append(
+                    run_id=run.id, kind="error", title="اجرا ناموفق بود.",
+                    payload={"error": str(exc)},
+                ),
+            )
+            await emit_run_status(
+                run_id=run.id,
+                task_id=task.id,
+                organization_id=organization_id,
+                status=RunStatus.FAILED.value,
+                error=str(exc),
+            )
+            await emit_task_status(
+                task_id=task.id, organization_id=organization_id, status=TaskStatus.FAILED.value
            )
        except Exception as exc:  # noqa: BLE001 - the run must always terminate
            run = await self._finish(run, RunStatus.FAILED, error=str(exc))
            await self.tasks.set_status(task.id, TaskStatus.FAILED)
-            await self.run_steps.append(
-                run_id=run.id, kind="error", title="خطای غیرمنتظره در اجرا.",
-                payload={"error": str(exc)},
+            await self._publish_step(
+                organization_id=organization_id,
+                run_id=run.id,
+                task_id=task.id,
+                step=await self.run_steps.append(
+                    run_id=run.id, kind="error", title="خطای غیرمنتظره در اجرا.",
+                    payload={"error": str(exc)},
+                ),
+            )
+            await emit_run_status(
+                run_id=run.id,
+                task_id=task.id,
+                organization_id=organization_id,
+                status=RunStatus.FAILED.value,
+                error=str(exc),
+            )
+            await emit_task_status(
+                task_id=task.id, organization_id=organization_id, status=TaskStatus.FAILED.value
            )

        return run

+    async def _publish_step(
+        self,
+        *,
+        organization_id: str,
+        run_id: str,
+        task_id: str,
+        step: RunStep,
+    ) -> None:
+        """Push one timeline step to the run, the task and the tenant feed.
+
+        The step is already persisted by the repository; this only broadcasts
+        it. Kept as a helper so the run engine above reads as the plain
+        sequence of transitions it is.
+        """
+        await emit_run_step(
+            run_id=run_id,
+            task_id=task_id,
+            organization_id=organization_id,
+            step_id=step.id,
+            index=step.index,
+            kind=step.kind,
+            title=step.title,
+            payload=step.payload_json,
+        )
+
    async def get_run_or_404(
        self, *, organization_id: str, run_id: str
    ) -> tuple[Run, list[RunStep], list[ToolCall], list[Artifact]]:
@@ -530,12 +663,28 @@ class WorkforceService:
            raise ConflictError("از وضعیت فعلی امکان لغو وجود ندارد.")

        run = await self._finish(run, RunStatus.CANCELLED)
-        await self.run_steps.append(
-            run_id=run.id, kind="cancelled", title="اجرا توسط کاربر لغو شد."
+        await self._publish_step(
+            organization_id=organization_id,
+            run_id=run.id,
+            task_id=run.task_id,
+            step=await self.run_steps.append(
+                run_id=run.id, kind="cancelled", title="اجرا توسط کاربر لغو شد."
+            ),
        )
        task = await self.tasks.get(run.task_id)
        if task is not None and task.status not in (TaskStatus.DONE, TaskStatus.CANCELLED):
            await self.tasks.set_status(task.id, TaskStatus.CANCELLED)
+            await emit_task_status(
+                task_id=task.id,
+                organization_id=organization_id,
+                status=TaskStatus.CANCELLED.value,
+            )
+        await emit_run_status(
+            run_id=run.id,
+            task_id=run.task_id,
+            organization_id=organization_id,
+            status=RunStatus.CANCELLED.value,
+        )
        return run

    async def list_pending_approvals(self, organization_id: str) -> list[ToolCall]:
@@ -627,6 +776,14 @@ class WorkforceService:
            ),
            payload={"tool_call_id": call.id, "requires_approval": needs_approval},
        )
+        await emit_approval_pending(
+            run_id=run.id,
+            task_id=run.task_id,
+            organization_id=organization_id,
+            tool_call_id=call.id,
+            tool_key=tool_key,
+            requires_approval=needs_approval,
+        )
        return call

    async def decide_approval(
@@ -655,12 +812,25 @@ class WorkforceService:
            result=result,
            approved_by_user_id=decided_by_user_id,
        )
+        run = await self.runs.get(call.run_id)
        await self.run_steps.append(
            run_id=call.run_id,
            kind="approval",
            title=("درخواست تأیید شد." if approve else "درخواست رد شد."),
            payload={"tool_call_id": call.id, "tool_key": call.tool_key},
        )
+        # Live push: the task page clears the pending card and the run timeline
+        # gains the decision without a reload.
+        await emit_run_step(
+            run_id=call.run_id,
+            task_id=run.task_id if run else "",
+            organization_id=organization_id,
+            step_id=call.id,
+            index=0,
+            kind="approval",
+            title=("درخواست تأیید شد." if approve else "درخواست رد شد."),
+            payload={"tool_call_id": call.id, "tool_key": call.tool_key},
+        )
        return updated or call

    # ------------------------------------------------------------------ #
@@ -776,6 +946,15 @@ class WorkforceService:
                metadata_json={"priced": card is not None},
            )
        )
+        # Live push: the billing page and the sidebar credit badge react to
+        # consumption as it happens instead of at the next page load.
+        await emit_usage_recorded(
+            organization_id=organization_id,
+            unit=unit.value if hasattr(unit, "value") else str(unit),
+            quantity=quantity,
+            run_id=run_id,
+            employee_id=employee_id,
+        )

    async def _advance(self, run: Run, target: RunStatus) -> Run:
        if not can_transition_run(run.status, target):