Merge origin/dev: لایهٔ realtime (WebSocket/RPC) + فلو دعوت عضو
ادغام ۱۲ کامیت ریموت (لایهٔ realtime روی WebSocket، حذف REST عمومی، رفع
باگهای صفحهٔ مدلها) با ۸ کامیت محلی (ارائهدهندهٔ agy + رفع فلو دعوت).
تنها کانفلیکت واقعی در `app/main.py` بود و ماهوی بود، نه متنی:
ریموت اپ عمومی را به «فقط /health + /api/ws» کاهش داده و همهٔ هندلرهای HTTP
را به اپ داخلی `interfaces/realtime/rpc_app.py` منتقل کرده است. بنابراین
روتر `invites` من نباید در اپ عمومی mount شود.
راهحل:
- `app/main.py`: فقط `health` و `realtime_router` میمانند؛ ایمپورتهای اضافی
حذف شدند (نسخهٔ ریموت).
- `interfaces/realtime/rpc_app.py`: روتر `invites` به اپ داخلی اضافه شد تا
`GET /api/invites/{token}` و `POST /api/invites/accept` از مسیر WS-RPC
قابل دسترس باشند.
`api.ts` و `contracts/index.ts` خودکار و بدون کانفلیکت ادغام شدند: کلاینت
`previewInvite`/`acceptInvite` حالا از `getRealtimeClient().rpc()` رد میشود،
پس هیچ تغییری در فراخوانیها لازم نبود.
رفع تست شکستهٔ ریموت: `src/test/version.test.tsx` مقدار `AppShell` را بدون
`RealtimeProvider` رندر میکرد و `useRealtime` استثنا میداد. حالا اسکلت
همانطور که در `app/layout.tsx` سیمکشی شده رندر میشود و سوکت در تست stub
شده است.
اعتبارسنجی روی درخت ادغامشده:
- بکاند ۲۷۰ تست سبز (شامل ۶ تست دعوت روی معماری جدید)؛ mypy تمیز.
- فرانت ۶۱ تست سبز؛ tsc و eslint تمیز.
- تأیید شد روترهای invite روی اپ داخلی RPC حاضرند و سطح عمومی تغییر نکرده.
نکته: ۵ خطای ruff در `tests/conftest.py`، `tests/test_error_log.py` و
`tests/test_realtime.py` از خود `origin/dev` میآیند و به این ادغام ربطی
ندارند؛ دستنخورده رها شدند.
This commit is contained in:
@@ -0,0 +1,504 @@
|
||||
"""Realtime push contract: the event envelope and the typed publishers.
|
||||
|
||||
This module is an **application port**. Services depend on it; the WebSocket
|
||||
adapter in ``app.interfaces.realtime`` implements it. That direction matters:
|
||||
the run engine must be able to publish without knowing that a browser exists,
|
||||
and a worker process that never serves a socket still publishes the same way.
|
||||
|
||||
Two parts:
|
||||
|
||||
* **The envelope** (:class:`RealtimeEvent`) and the channel naming scheme.
|
||||
Framework-free, like every other port.
|
||||
* **The typed publishers** — ``emit_run_step`` and friends. They own the
|
||||
fan-out channels and the one rule that makes live feeds safe to call from
|
||||
anywhere: *publishing never raises*. A feed must not be able to fail a task
|
||||
run, a sandbox command or a billing write.
|
||||
|
||||
The publisher is process-wide state, set once by a composition root
|
||||
(``app.main`` for the API, ``app.workers.__main__`` for the workers). Until
|
||||
then it is a :class:`NullPublisher` — publishing is a no-op rather than a
|
||||
missing-import crash.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
from app.domain.base import new_id, utcnow
|
||||
|
||||
logger = logging.getLogger("agy.realtime")
|
||||
|
||||
#: The platform-owner console stream. Authorized against ``is_platform_owner``.
|
||||
ADMIN_CHANNEL = "admin"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Channels
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def org_channel(organization_id: str) -> str:
|
||||
"""Everything happening inside one tenant."""
|
||||
return f"org:{organization_id}"
|
||||
|
||||
|
||||
def task_channel(task_id: str) -> str:
|
||||
return f"task:{task_id}"
|
||||
|
||||
|
||||
def run_channel(run_id: str) -> str:
|
||||
return f"run:{run_id}"
|
||||
|
||||
|
||||
def thread_channel(thread_id: str) -> str:
|
||||
return f"thread:{thread_id}"
|
||||
|
||||
|
||||
def session_channel(session_id: str) -> str:
|
||||
"""One sandbox session — the live command terminal subscribes here."""
|
||||
return f"session:{session_id}"
|
||||
|
||||
|
||||
def user_channel(user_id: str) -> str:
|
||||
return f"user:{user_id}"
|
||||
|
||||
|
||||
def admin_channel() -> str:
|
||||
return ADMIN_CHANNEL
|
||||
|
||||
|
||||
def dedupe_channels(channels: Iterable[str]) -> tuple[str, ...]:
|
||||
"""Preserve order, drop duplicates — a publisher may overlap channels."""
|
||||
seen: dict[str, None] = {}
|
||||
for channel in channels:
|
||||
if channel and channel not in seen:
|
||||
seen[channel] = None
|
||||
return tuple(seen)
|
||||
|
||||
|
||||
def with_org_channel(channels: Sequence[str], organization_id: str | None) -> tuple[str, ...]:
|
||||
"""Append the tenant channel unless the caller already listed it."""
|
||||
if not organization_id:
|
||||
return dedupe_channels(channels)
|
||||
return dedupe_channels([*channels, org_channel(organization_id)])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Envelope
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(slots=True, kw_only=True)
|
||||
class RealtimeEvent:
|
||||
"""One push to one or more channels.
|
||||
|
||||
``channels`` is the fan-out list — the same event typically goes to the
|
||||
narrow channel (``run:…``) and the wide one (``org:…``), so a page watching
|
||||
a single run and a dashboard watching the whole tenant both stay live
|
||||
without the publisher caring who is listening.
|
||||
"""
|
||||
|
||||
type: str
|
||||
channels: tuple[str, ...]
|
||||
data: dict = field(default_factory=dict)
|
||||
organization_id: str | None = None
|
||||
id: str = field(default_factory=new_id)
|
||||
at: datetime = field(default_factory=utcnow)
|
||||
|
||||
def envelope(self) -> dict:
|
||||
"""The JSON object a browser receives.
|
||||
|
||||
``at`` is serialised as an ISO-8601 UTC string: the client renders it
|
||||
with ``toLocaleString`` like every other timestamp in the product.
|
||||
"""
|
||||
return {
|
||||
"id": self.id,
|
||||
"type": self.type,
|
||||
"data": self.data,
|
||||
"organization_id": self.organization_id,
|
||||
"at": self.at.isoformat(),
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
return json.dumps(self.envelope(), ensure_ascii=False, default=str)
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, raw: str | bytes) -> RealtimeEvent:
|
||||
payload = json.loads(raw)
|
||||
return cls(
|
||||
type=str(payload["type"]),
|
||||
channels=tuple(payload.get("channels") or ()),
|
||||
data=dict(payload.get("data") or {}),
|
||||
organization_id=payload.get("organization_id"),
|
||||
id=str(payload["id"]),
|
||||
at=_parse_at(payload.get("at")),
|
||||
)
|
||||
|
||||
|
||||
def _parse_at(value: object) -> datetime:
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
return datetime.fromisoformat(value)
|
||||
except ValueError:
|
||||
return utcnow()
|
||||
return utcnow()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Publisher plumbing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class EventPublisher(Protocol):
|
||||
async def publish(self, event: RealtimeEvent) -> None: ...
|
||||
|
||||
|
||||
class NullPublisher:
|
||||
"""Default before a composition root wires the real bus."""
|
||||
|
||||
async def publish(self, event: RealtimeEvent) -> None:
|
||||
return None
|
||||
|
||||
|
||||
_PUBLISHER: EventPublisher = NullPublisher()
|
||||
|
||||
|
||||
def set_publisher(publisher: EventPublisher) -> None:
|
||||
"""Install the process-wide publisher. Called by composition roots only."""
|
||||
global _PUBLISHER
|
||||
_PUBLISHER = publisher
|
||||
|
||||
|
||||
def get_publisher() -> EventPublisher:
|
||||
return _PUBLISHER
|
||||
|
||||
|
||||
async def publish(event: RealtimeEvent) -> None:
|
||||
"""Publish through the installed publisher. Never raises."""
|
||||
try:
|
||||
await _PUBLISHER.publish(event)
|
||||
except Exception: # noqa: BLE001 - the feed must never fail a caller
|
||||
logger.exception("realtime: publish failed for %s", event.type)
|
||||
|
||||
|
||||
async def _emit(
|
||||
*,
|
||||
type: str,
|
||||
channels: tuple[str, ...],
|
||||
data: dict,
|
||||
organization_id: str | None = None,
|
||||
) -> None:
|
||||
await publish(
|
||||
RealtimeEvent(
|
||||
type=type,
|
||||
channels=with_org_channel(channels, organization_id),
|
||||
data=data,
|
||||
organization_id=organization_id,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Typed publishers
|
||||
#
|
||||
# Payload shape is the product contract with the browser: the client-side types
|
||||
# in ``packages/contracts`` mirror ``data`` key-for-key. Keep the two in sync.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Long text fields are clipped in the live feed. The full record stays
|
||||
#: available through the REST endpoints — the feed is a signal, not an archive.
|
||||
MAX_LIVE_TEXT_CHARS = 8000
|
||||
|
||||
|
||||
def _clip(text: str) -> str:
|
||||
return text[:MAX_LIVE_TEXT_CHARS]
|
||||
|
||||
|
||||
async def emit_run_step(
|
||||
*,
|
||||
run_id: str,
|
||||
task_id: str,
|
||||
organization_id: str,
|
||||
step_id: str,
|
||||
index: int,
|
||||
kind: str,
|
||||
title: str,
|
||||
payload: dict | None = None,
|
||||
) -> None:
|
||||
await _emit(
|
||||
type="run.step",
|
||||
channels=(run_channel(run_id), task_channel(task_id)),
|
||||
organization_id=organization_id,
|
||||
data={
|
||||
"run_id": run_id,
|
||||
"task_id": task_id,
|
||||
"step": {
|
||||
"id": step_id,
|
||||
"index": index,
|
||||
"kind": kind,
|
||||
"title": title,
|
||||
"payload": payload or {},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def emit_run_status(
|
||||
*,
|
||||
run_id: str,
|
||||
task_id: str,
|
||||
organization_id: str,
|
||||
status: str,
|
||||
error: str | None = None,
|
||||
) -> None:
|
||||
await _emit(
|
||||
type="run.status",
|
||||
channels=(run_channel(run_id), task_channel(task_id)),
|
||||
organization_id=organization_id,
|
||||
data={"run_id": run_id, "task_id": task_id, "status": status, "error": error},
|
||||
)
|
||||
|
||||
|
||||
async def emit_task_status(*, task_id: str, organization_id: str, status: str) -> None:
|
||||
await _emit(
|
||||
type="task.status",
|
||||
channels=(task_channel(task_id),),
|
||||
organization_id=organization_id,
|
||||
data={"task_id": task_id, "status": status},
|
||||
)
|
||||
|
||||
|
||||
async def emit_task_created(
|
||||
*, task_id: str, organization_id: str, title: str, status: str
|
||||
) -> None:
|
||||
await _emit(
|
||||
type="task.created",
|
||||
channels=(task_channel(task_id),),
|
||||
organization_id=organization_id,
|
||||
data={"task_id": task_id, "title": title, "status": status},
|
||||
)
|
||||
|
||||
|
||||
async def emit_chat_pending(*, thread_id: str, organization_id: str) -> None:
|
||||
"""The manager is thinking — the client shows a typing indicator."""
|
||||
await _emit(
|
||||
type="chat.pending",
|
||||
channels=(thread_channel(thread_id),),
|
||||
organization_id=organization_id,
|
||||
data={"thread_id": thread_id},
|
||||
)
|
||||
|
||||
|
||||
async def emit_message_created(
|
||||
*,
|
||||
thread_id: str,
|
||||
organization_id: str,
|
||||
message_id: str,
|
||||
role: str,
|
||||
content: str,
|
||||
employee_id: str | None = None,
|
||||
) -> None:
|
||||
await _emit(
|
||||
type="message.created",
|
||||
channels=(thread_channel(thread_id),),
|
||||
organization_id=organization_id,
|
||||
data={
|
||||
"thread_id": thread_id,
|
||||
"message": {
|
||||
"id": message_id,
|
||||
"role": role,
|
||||
"content": _clip(content),
|
||||
"employee_id": employee_id,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def emit_approval_pending(
|
||||
*,
|
||||
run_id: str,
|
||||
task_id: str,
|
||||
organization_id: str,
|
||||
tool_call_id: str,
|
||||
tool_key: str,
|
||||
requires_approval: bool,
|
||||
) -> None:
|
||||
await _emit(
|
||||
type="approval.pending",
|
||||
channels=(run_channel(run_id), task_channel(task_id)),
|
||||
organization_id=organization_id,
|
||||
data={
|
||||
"run_id": run_id,
|
||||
"task_id": task_id,
|
||||
"tool_call_id": tool_call_id,
|
||||
"tool_key": tool_key,
|
||||
"requires_approval": requires_approval,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def emit_artifact_created(
|
||||
*, run_id: str, task_id: str, organization_id: str, artifact_id: str, name: str
|
||||
) -> None:
|
||||
await _emit(
|
||||
type="artifact.created",
|
||||
channels=(run_channel(run_id), task_channel(task_id)),
|
||||
organization_id=organization_id,
|
||||
data={
|
||||
"run_id": run_id,
|
||||
"task_id": task_id,
|
||||
"artifact_id": artifact_id,
|
||||
"name": name,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def emit_sandbox_command(
|
||||
*,
|
||||
session_id: str,
|
||||
organization_id: str,
|
||||
command_run_id: str,
|
||||
command: str,
|
||||
exit_code: int | None,
|
||||
stdout: str,
|
||||
stderr: str,
|
||||
duration_ms: int,
|
||||
truncated: bool,
|
||||
status: str = "finished",
|
||||
) -> None:
|
||||
"""One command's output, pushed live to the session's terminal panel."""
|
||||
await _emit(
|
||||
type="sandbox.command",
|
||||
channels=(session_channel(session_id),),
|
||||
organization_id=organization_id,
|
||||
data={
|
||||
"session_id": session_id,
|
||||
"run_id": command_run_id,
|
||||
"command": _clip(command),
|
||||
"status": status,
|
||||
"exit_code": exit_code,
|
||||
"stdout": _clip(stdout),
|
||||
"stderr": _clip(stderr),
|
||||
"duration_ms": duration_ms,
|
||||
"truncated": truncated,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def emit_sandbox_session(
|
||||
*,
|
||||
session_id: str,
|
||||
organization_id: str,
|
||||
status: str,
|
||||
workspace_id: str | None = None,
|
||||
) -> None:
|
||||
await _emit(
|
||||
type="sandbox.session",
|
||||
channels=(session_channel(session_id),),
|
||||
organization_id=organization_id,
|
||||
data={
|
||||
"session_id": session_id,
|
||||
"status": status,
|
||||
"workspace_id": workspace_id,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def emit_sandbox_reaped(
|
||||
*, organization_id: str | None, session_id: str, workspace_id: str
|
||||
) -> None:
|
||||
await _emit(
|
||||
type="sandbox.reaped",
|
||||
channels=(session_channel(session_id),),
|
||||
organization_id=organization_id,
|
||||
data={"session_id": session_id, "workspace_id": workspace_id},
|
||||
)
|
||||
|
||||
|
||||
async def emit_usage_recorded(
|
||||
*,
|
||||
organization_id: str,
|
||||
unit: str,
|
||||
quantity: float,
|
||||
run_id: str | None = None,
|
||||
employee_id: str | None = None,
|
||||
) -> None:
|
||||
await _emit(
|
||||
type="usage.recorded",
|
||||
channels=(),
|
||||
organization_id=organization_id,
|
||||
data={
|
||||
"unit": unit,
|
||||
"quantity": quantity,
|
||||
"run_id": run_id,
|
||||
"employee_id": employee_id,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def emit_wallet_changed(*, organization_id: str, balance: float) -> None:
|
||||
await _emit(
|
||||
type="billing.wallet",
|
||||
channels=(),
|
||||
organization_id=organization_id,
|
||||
data={"organization_id": organization_id, "balance": balance},
|
||||
)
|
||||
|
||||
|
||||
async def emit_error_recorded(
|
||||
*,
|
||||
error_id: str | None,
|
||||
kind: str,
|
||||
message: str,
|
||||
severity: str,
|
||||
source: str,
|
||||
organization_id: str | None = None,
|
||||
) -> None:
|
||||
"""Feed the platform-owner console; tenants see their own through the org
|
||||
channel the envelope carries."""
|
||||
await _emit(
|
||||
type="error.recorded",
|
||||
channels=("admin",),
|
||||
organization_id=organization_id,
|
||||
data={
|
||||
"error_id": error_id,
|
||||
"kind": kind,
|
||||
"message": _clip(message),
|
||||
"severity": severity,
|
||||
"source": source,
|
||||
"organization_id": organization_id,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def emit_audit_recorded(
|
||||
*, organization_id: str | None, action: str, target_type: str, target_id: str
|
||||
) -> None:
|
||||
if not organization_id:
|
||||
return
|
||||
await _emit(
|
||||
type="audit.recorded",
|
||||
channels=(),
|
||||
organization_id=organization_id,
|
||||
data={"action": action, "target_type": target_type, "target_id": target_id},
|
||||
)
|
||||
|
||||
|
||||
async def emit_worker_status(
|
||||
*,
|
||||
organization_id: str,
|
||||
worker: str,
|
||||
message: str,
|
||||
) -> None:
|
||||
await _emit(
|
||||
type="worker.status",
|
||||
channels=(),
|
||||
organization_id=organization_id,
|
||||
data={"worker": worker, "message": _clip(message)},
|
||||
)
|
||||
@@ -8,6 +8,7 @@ from __future__ import annotations
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.application.ports.realtime import emit_audit_recorded
|
||||
from app.domain.audit.models import AuditEvent
|
||||
from app.infrastructure.repositories.audit import AuditRepository
|
||||
|
||||
@@ -34,4 +35,13 @@ async def record_audit(
|
||||
ip_address=ip,
|
||||
user_agent=user_agent,
|
||||
)
|
||||
return await AuditRepository(session).record(event=event)
|
||||
recorded = await AuditRepository(session).record(event=event)
|
||||
# Live push: the audit timelines (dashboard, traces) update as actions
|
||||
# happen instead of waiting for the next reload.
|
||||
await emit_audit_recorded(
|
||||
organization_id=organization_id,
|
||||
action=action,
|
||||
target_type=target_type,
|
||||
target_id=target_id,
|
||||
)
|
||||
return recorded
|
||||
|
||||
@@ -14,6 +14,7 @@ from __future__ import annotations
|
||||
from datetime import datetime
|
||||
|
||||
from app.application.ports import PaymentProviderPort
|
||||
from app.application.ports.realtime import emit_wallet_changed
|
||||
from app.core.errors import NotFoundError, PermissionDeniedError, ValidationError
|
||||
from app.domain.billing.models import (
|
||||
CreditWallet,
|
||||
@@ -121,6 +122,11 @@ class BillingService:
|
||||
# Refuse to leave a wallet in debt; undo and surface the conflict.
|
||||
await self.wallets.adjust(organization_id=organization_id, delta=-amount)
|
||||
raise ValidationError("اعتبار کافی برای این کسر وجود ندارد.")
|
||||
# Live push: the sidebar credit badge and the billing page follow the
|
||||
# wallet as it moves, whoever moved it.
|
||||
await emit_wallet_changed(
|
||||
organization_id=organization_id, balance=wallet.balance
|
||||
)
|
||||
return wallet
|
||||
|
||||
async def start_checkout(
|
||||
@@ -177,8 +183,13 @@ class BillingService:
|
||||
return payment
|
||||
|
||||
if succeeded:
|
||||
await self.wallets.adjust(organization_id=organization_id, delta=payment.amount)
|
||||
wallet = await self.wallets.adjust(
|
||||
organization_id=organization_id, delta=payment.amount
|
||||
)
|
||||
final = await self.payments.set_status(payment_id, PaymentStatus.SUCCEEDED)
|
||||
await emit_wallet_changed(
|
||||
organization_id=organization_id, balance=wallet.balance
|
||||
)
|
||||
else:
|
||||
final = await self.payments.set_status(payment_id, PaymentStatus.FAILED)
|
||||
return final or payment
|
||||
|
||||
@@ -18,6 +18,10 @@ import contextlib
|
||||
from datetime import datetime
|
||||
|
||||
from app.application.ports import CommandResult, SandboxProviderPort, UsageRecord
|
||||
from app.application.ports.realtime import (
|
||||
emit_sandbox_command,
|
||||
emit_sandbox_session,
|
||||
)
|
||||
from app.core.config import settings
|
||||
from app.core.errors import ConflictError, NotFoundError, ProviderUnavailableError, ValidationError
|
||||
from app.domain.ai_control.models import RateUnit
|
||||
@@ -210,6 +214,12 @@ class SandboxService:
|
||||
)
|
||||
)
|
||||
await self.workspaces.set_status(workspace.id, WorkspaceStatus.READY)
|
||||
await emit_sandbox_session(
|
||||
session_id=session.id,
|
||||
organization_id=organization_id,
|
||||
status=SessionStatus.RUNNING.value,
|
||||
workspace_id=workspace.id,
|
||||
)
|
||||
return session
|
||||
|
||||
async def get_session_or_404(
|
||||
@@ -229,6 +239,11 @@ class SandboxService:
|
||||
await self.provider.destroy(session_external_id=session.external_session_id)
|
||||
await self.sessions.set_status(session.id, SessionStatus.STOPPED, stopped=True)
|
||||
refreshed = await self.sessions.get(session.id)
|
||||
await emit_sandbox_session(
|
||||
session_id=session.id,
|
||||
organization_id=organization_id,
|
||||
status=SessionStatus.STOPPED.value,
|
||||
)
|
||||
return refreshed or session
|
||||
|
||||
# -- commands ------------------------------------------------------------
|
||||
@@ -285,6 +300,20 @@ class SandboxService:
|
||||
)
|
||||
await self.sessions.set_status(session.id, SessionStatus.RUNNING)
|
||||
|
||||
# Live push: the terminal panel on the project page streams the command
|
||||
# and its output as the row is written, not on the next poll.
|
||||
await emit_sandbox_command(
|
||||
session_id=session.id,
|
||||
organization_id=organization_id,
|
||||
command_run_id=run.id,
|
||||
command=cleaned,
|
||||
exit_code=result.exit_code,
|
||||
stdout=run.stdout_ref,
|
||||
stderr=run.stderr_ref,
|
||||
duration_ms=result.duration_ms,
|
||||
truncated=result.truncated,
|
||||
)
|
||||
|
||||
# CPU seconds are billed. 1 second of wall time ~= 1 CPU second at MVP.
|
||||
cpu_seconds = max(0.0, result.duration_ms / 1000.0)
|
||||
if cpu_seconds > 0:
|
||||
|
||||
@@ -24,6 +24,17 @@ from app.application.ports import (
|
||||
ModelProviderPort,
|
||||
UsageRecord,
|
||||
)
|
||||
from app.application.ports.realtime import (
|
||||
emit_approval_pending,
|
||||
emit_artifact_created,
|
||||
emit_chat_pending,
|
||||
emit_message_created,
|
||||
emit_run_status,
|
||||
emit_run_step,
|
||||
emit_task_created,
|
||||
emit_task_status,
|
||||
emit_usage_recorded,
|
||||
)
|
||||
from app.core.errors import (
|
||||
ConflictError,
|
||||
NotFoundError,
|
||||
@@ -324,6 +335,10 @@ class WorkforceService:
|
||||
else:
|
||||
employee = await self._default_manager(organization_id)
|
||||
|
||||
# The manager is about to call a model. Tell the thread so the UI can
|
||||
# show a thinking indicator immediately.
|
||||
await emit_chat_pending(thread_id=thread.id, organization_id=organization_id)
|
||||
|
||||
history = await self.messages.list_for_thread(thread.id)
|
||||
system_prompt = await self._system_prompt_for(employee)
|
||||
reply = await self._complete(
|
||||
@@ -347,6 +362,16 @@ class WorkforceService:
|
||||
employee_id=employee.id if employee else None,
|
||||
)
|
||||
)
|
||||
# Live push: any browser on this thread sees the reply arrive, and the
|
||||
# pending indicator clears, without waiting for a reload.
|
||||
await emit_message_created(
|
||||
thread_id=thread.id,
|
||||
organization_id=organization_id,
|
||||
message_id=assistant_message.id,
|
||||
role=MessageRole.ASSISTANT.value,
|
||||
content=reply,
|
||||
employee_id=employee.id if employee else None,
|
||||
)
|
||||
return user_message, assistant_message
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
@@ -371,7 +396,7 @@ class WorkforceService:
|
||||
await self.get_employee_or_404(
|
||||
organization_id=organization_id, employee_id=assigned_employee_id
|
||||
)
|
||||
return await self.tasks.create(
|
||||
task = await self.tasks.create(
|
||||
Task(
|
||||
organization_id=organization_id,
|
||||
title=title.strip(),
|
||||
@@ -382,6 +407,15 @@ class WorkforceService:
|
||||
created_by_user_id=created_by_user_id,
|
||||
)
|
||||
)
|
||||
# Live push: the tasks list and the manager-chat sidebar update the
|
||||
# moment the task exists, whoever created it and from where.
|
||||
await emit_task_created(
|
||||
task_id=task.id,
|
||||
organization_id=organization_id,
|
||||
title=task.title,
|
||||
status=task.status.value,
|
||||
)
|
||||
return task
|
||||
|
||||
async def list_tasks(
|
||||
self,
|
||||
@@ -435,21 +469,39 @@ class WorkforceService:
|
||||
Run(task_id=task.id, employee_id=employee.id if employee else None)
|
||||
)
|
||||
await self.tasks.set_status(task.id, TaskStatus.RUNNING)
|
||||
await emit_task_status(
|
||||
task_id=task.id, organization_id=organization_id, status=TaskStatus.RUNNING.value
|
||||
)
|
||||
|
||||
await self.run_steps.append(
|
||||
run_id=run.id, kind="queued", title="کار در صف اجرا قرار گرفت.",
|
||||
payload={"task_id": task.id},
|
||||
await self._publish_step(
|
||||
organization_id=organization_id,
|
||||
run_id=run.id,
|
||||
task_id=task.id,
|
||||
step=await self.run_steps.append(
|
||||
run_id=run.id, kind="queued", title="کار در صف اجرا قرار گرفت.",
|
||||
payload={"task_id": task.id},
|
||||
),
|
||||
)
|
||||
|
||||
try:
|
||||
run = await self._advance(run, RunStatus.PLANNING)
|
||||
await self.run_steps.append(
|
||||
run_id=run.id, kind="planning", title="برنامهریزی مرحلهٔ اجرا."
|
||||
await self._publish_step(
|
||||
organization_id=organization_id,
|
||||
run_id=run.id,
|
||||
task_id=task.id,
|
||||
step=await self.run_steps.append(
|
||||
run_id=run.id, kind="planning", title="برنامهریزی مرحلهٔ اجرا."
|
||||
),
|
||||
)
|
||||
|
||||
run = await self._advance(run, RunStatus.RUNNING)
|
||||
await self.run_steps.append(
|
||||
run_id=run.id, kind="running", title="اجرای کار آغاز شد."
|
||||
await self._publish_step(
|
||||
organization_id=organization_id,
|
||||
run_id=run.id,
|
||||
task_id=task.id,
|
||||
step=await self.run_steps.append(
|
||||
run_id=run.id, kind="running", title="اجرای کار آغاز شد."
|
||||
),
|
||||
)
|
||||
|
||||
reply = await self._complete(
|
||||
@@ -463,9 +515,14 @@ class WorkforceService:
|
||||
ChatMessage(role="user", content=self._task_brief(task)),
|
||||
],
|
||||
)
|
||||
await self.run_steps.append(
|
||||
run_id=run.id, kind="output", title="خروجی آماده شد.",
|
||||
payload={"chars": len(reply)},
|
||||
await self._publish_step(
|
||||
organization_id=organization_id,
|
||||
run_id=run.id,
|
||||
task_id=task.id,
|
||||
step=await self.run_steps.append(
|
||||
run_id=run.id, kind="output", title="خروجی آماده شد.",
|
||||
payload={"chars": len(reply)},
|
||||
),
|
||||
)
|
||||
|
||||
artifact = await self.artifacts.create(
|
||||
@@ -479,30 +536,106 @@ class WorkforceService:
|
||||
size_bytes=len(reply.encode("utf-8")),
|
||||
)
|
||||
)
|
||||
await self.run_steps.append(
|
||||
run_id=run.id, kind="artifact", title="سند خروجی ساخته شد.",
|
||||
payload={"artifact_id": artifact.id},
|
||||
await self._publish_step(
|
||||
organization_id=organization_id,
|
||||
run_id=run.id,
|
||||
task_id=task.id,
|
||||
step=await self.run_steps.append(
|
||||
run_id=run.id, kind="artifact", title="سند خروجی ساخته شد.",
|
||||
payload={"artifact_id": artifact.id},
|
||||
),
|
||||
)
|
||||
await emit_artifact_created(
|
||||
run_id=run.id,
|
||||
task_id=task.id,
|
||||
organization_id=organization_id,
|
||||
artifact_id=artifact.id,
|
||||
name=artifact.name,
|
||||
)
|
||||
|
||||
run = await self._finish(run, RunStatus.COMPLETED)
|
||||
await self.tasks.set_status(task.id, TaskStatus.DONE)
|
||||
await emit_run_status(
|
||||
run_id=run.id,
|
||||
task_id=task.id,
|
||||
organization_id=organization_id,
|
||||
status=RunStatus.COMPLETED.value,
|
||||
)
|
||||
await emit_task_status(
|
||||
task_id=task.id, organization_id=organization_id, status=TaskStatus.DONE.value
|
||||
)
|
||||
except ProviderUnavailableError as exc:
|
||||
run = await self._finish(run, RunStatus.FAILED, error=str(exc))
|
||||
await self.tasks.set_status(task.id, TaskStatus.FAILED)
|
||||
await self.run_steps.append(
|
||||
run_id=run.id, kind="error", title="اجرا ناموفق بود.",
|
||||
payload={"error": str(exc)},
|
||||
await self._publish_step(
|
||||
organization_id=organization_id,
|
||||
run_id=run.id,
|
||||
task_id=task.id,
|
||||
step=await self.run_steps.append(
|
||||
run_id=run.id, kind="error", title="اجرا ناموفق بود.",
|
||||
payload={"error": str(exc)},
|
||||
),
|
||||
)
|
||||
await emit_run_status(
|
||||
run_id=run.id,
|
||||
task_id=task.id,
|
||||
organization_id=organization_id,
|
||||
status=RunStatus.FAILED.value,
|
||||
error=str(exc),
|
||||
)
|
||||
await emit_task_status(
|
||||
task_id=task.id, organization_id=organization_id, status=TaskStatus.FAILED.value
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - the run must always terminate
|
||||
run = await self._finish(run, RunStatus.FAILED, error=str(exc))
|
||||
await self.tasks.set_status(task.id, TaskStatus.FAILED)
|
||||
await self.run_steps.append(
|
||||
run_id=run.id, kind="error", title="خطای غیرمنتظره در اجرا.",
|
||||
payload={"error": str(exc)},
|
||||
await self._publish_step(
|
||||
organization_id=organization_id,
|
||||
run_id=run.id,
|
||||
task_id=task.id,
|
||||
step=await self.run_steps.append(
|
||||
run_id=run.id, kind="error", title="خطای غیرمنتظره در اجرا.",
|
||||
payload={"error": str(exc)},
|
||||
),
|
||||
)
|
||||
await emit_run_status(
|
||||
run_id=run.id,
|
||||
task_id=task.id,
|
||||
organization_id=organization_id,
|
||||
status=RunStatus.FAILED.value,
|
||||
error=str(exc),
|
||||
)
|
||||
await emit_task_status(
|
||||
task_id=task.id, organization_id=organization_id, status=TaskStatus.FAILED.value
|
||||
)
|
||||
|
||||
return run
|
||||
|
||||
async def _publish_step(
|
||||
self,
|
||||
*,
|
||||
organization_id: str,
|
||||
run_id: str,
|
||||
task_id: str,
|
||||
step: RunStep,
|
||||
) -> None:
|
||||
"""Push one timeline step to the run, the task and the tenant feed.
|
||||
|
||||
The step is already persisted by the repository; this only broadcasts
|
||||
it. Kept as a helper so the run engine above reads as the plain
|
||||
sequence of transitions it is.
|
||||
"""
|
||||
await emit_run_step(
|
||||
run_id=run_id,
|
||||
task_id=task_id,
|
||||
organization_id=organization_id,
|
||||
step_id=step.id,
|
||||
index=step.index,
|
||||
kind=step.kind,
|
||||
title=step.title,
|
||||
payload=step.payload_json,
|
||||
)
|
||||
|
||||
async def get_run_or_404(
|
||||
self, *, organization_id: str, run_id: str
|
||||
) -> tuple[Run, list[RunStep], list[ToolCall], list[Artifact]]:
|
||||
@@ -530,12 +663,28 @@ class WorkforceService:
|
||||
raise ConflictError("از وضعیت فعلی امکان لغو وجود ندارد.")
|
||||
|
||||
run = await self._finish(run, RunStatus.CANCELLED)
|
||||
await self.run_steps.append(
|
||||
run_id=run.id, kind="cancelled", title="اجرا توسط کاربر لغو شد."
|
||||
await self._publish_step(
|
||||
organization_id=organization_id,
|
||||
run_id=run.id,
|
||||
task_id=run.task_id,
|
||||
step=await self.run_steps.append(
|
||||
run_id=run.id, kind="cancelled", title="اجرا توسط کاربر لغو شد."
|
||||
),
|
||||
)
|
||||
task = await self.tasks.get(run.task_id)
|
||||
if task is not None and task.status not in (TaskStatus.DONE, TaskStatus.CANCELLED):
|
||||
await self.tasks.set_status(task.id, TaskStatus.CANCELLED)
|
||||
await emit_task_status(
|
||||
task_id=task.id,
|
||||
organization_id=organization_id,
|
||||
status=TaskStatus.CANCELLED.value,
|
||||
)
|
||||
await emit_run_status(
|
||||
run_id=run.id,
|
||||
task_id=run.task_id,
|
||||
organization_id=organization_id,
|
||||
status=RunStatus.CANCELLED.value,
|
||||
)
|
||||
return run
|
||||
|
||||
async def list_pending_approvals(self, organization_id: str) -> list[ToolCall]:
|
||||
@@ -627,6 +776,14 @@ class WorkforceService:
|
||||
),
|
||||
payload={"tool_call_id": call.id, "requires_approval": needs_approval},
|
||||
)
|
||||
await emit_approval_pending(
|
||||
run_id=run.id,
|
||||
task_id=run.task_id,
|
||||
organization_id=organization_id,
|
||||
tool_call_id=call.id,
|
||||
tool_key=tool_key,
|
||||
requires_approval=needs_approval,
|
||||
)
|
||||
return call
|
||||
|
||||
async def decide_approval(
|
||||
@@ -655,12 +812,25 @@ class WorkforceService:
|
||||
result=result,
|
||||
approved_by_user_id=decided_by_user_id,
|
||||
)
|
||||
run = await self.runs.get(call.run_id)
|
||||
await self.run_steps.append(
|
||||
run_id=call.run_id,
|
||||
kind="approval",
|
||||
title=("درخواست تأیید شد." if approve else "درخواست رد شد."),
|
||||
payload={"tool_call_id": call.id, "tool_key": call.tool_key},
|
||||
)
|
||||
# Live push: the task page clears the pending card and the run timeline
|
||||
# gains the decision without a reload.
|
||||
await emit_run_step(
|
||||
run_id=call.run_id,
|
||||
task_id=run.task_id if run else "",
|
||||
organization_id=organization_id,
|
||||
step_id=call.id,
|
||||
index=0,
|
||||
kind="approval",
|
||||
title=("درخواست تأیید شد." if approve else "درخواست رد شد."),
|
||||
payload={"tool_call_id": call.id, "tool_key": call.tool_key},
|
||||
)
|
||||
return updated or call
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
@@ -776,6 +946,15 @@ class WorkforceService:
|
||||
metadata_json={"priced": card is not None},
|
||||
)
|
||||
)
|
||||
# Live push: the billing page and the sidebar credit badge react to
|
||||
# consumption as it happens instead of at the next page load.
|
||||
await emit_usage_recorded(
|
||||
organization_id=organization_id,
|
||||
unit=unit.value if hasattr(unit, "value") else str(unit),
|
||||
quantity=quantity,
|
||||
run_id=run_id,
|
||||
employee_id=employee_id,
|
||||
)
|
||||
|
||||
async def _advance(self, run: Run, target: RunStatus) -> Run:
|
||||
if not can_transition_run(run.status, target):
|
||||
|
||||
@@ -53,6 +53,15 @@ class Settings(BaseSettings):
|
||||
database_url: str = "sqlite+aiosqlite:///./agy_web.db"
|
||||
redis_url: str = "redis://localhost:6379/0"
|
||||
|
||||
# --- realtime push -------------------------------------------------------
|
||||
#: How live events travel between processes. ``auto`` mirrors published
|
||||
#: events onto Redis pub/sub so the standalone workers' pushes reach the
|
||||
#: API process's browser sockets; if no broker is reachable the bus falls
|
||||
#: back to in-process fan-out and every API-driven feed stays live.
|
||||
#: ``local`` pins the bus to in-process delivery (tests, single-process
|
||||
#: deployments without a broker).
|
||||
realtime_transport: str = "auto"
|
||||
|
||||
# --- security ------------------------------------------------------------
|
||||
jwt_secret: str = "change-me"
|
||||
jwt_algorithm: str = "HS256"
|
||||
|
||||
@@ -31,12 +31,16 @@ import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlencode, urlparse
|
||||
|
||||
import httpx
|
||||
@@ -106,9 +110,8 @@ EFFORT_THINKING_BUDGETS: dict[str, int] = {
|
||||
#: a model that does not support it — that is a hard 400 from the backend.
|
||||
_THINKING_MARKERS: tuple[str, ...] = (
|
||||
"thinking",
|
||||
"gemini-3",
|
||||
"gemini-2.5-pro",
|
||||
"gemini-2.5-flash",
|
||||
"tiered",
|
||||
"gemini-",
|
||||
)
|
||||
|
||||
|
||||
@@ -118,6 +121,7 @@ def _supports_thinking(model_name: str) -> bool:
|
||||
return False
|
||||
return any(marker in lowered for marker in _THINKING_MARKERS)
|
||||
|
||||
|
||||
_StatusFn = Callable[..., Awaitable[None]]
|
||||
_LoaderFn = Callable[[], Awaitable[list[ModelCredential]]]
|
||||
|
||||
@@ -209,6 +213,18 @@ class AgyLoginResult:
|
||||
refresh_token: str
|
||||
|
||||
|
||||
@dataclass(slots=True, kw_only=True)
|
||||
class AgyAccountModelInfo:
|
||||
"""Model catalogue item as reported by an agy account."""
|
||||
|
||||
id: str
|
||||
display_name: str
|
||||
supports_thinking: bool = False
|
||||
supported_efforts: list[str] = field(default_factory=list)
|
||||
default_effort: str | None = None
|
||||
supports_images: bool = False
|
||||
|
||||
|
||||
class AgyOAuthError(ValidationError):
|
||||
"""The pasted URL or the token exchange was rejected by Google."""
|
||||
|
||||
@@ -309,9 +325,7 @@ class AgyAuthClient:
|
||||
self._fetch_email(tokens.get("access_token", "")),
|
||||
self._fetch_project_id(tokens.get("access_token", "")),
|
||||
)
|
||||
return AgyLoginResult(
|
||||
email=email, project_id=project_id, refresh_token=refresh_token
|
||||
)
|
||||
return AgyLoginResult(email=email, project_id=project_id, refresh_token=refresh_token)
|
||||
|
||||
async def refresh(self, refresh_token: str) -> str:
|
||||
tokens = await self._post_token(
|
||||
@@ -436,9 +450,7 @@ async def redeem_login(login_id: str, callback_url: str) -> AgyLoginResult:
|
||||
return await AgyAuthClient().exchange(challenge, callback_url)
|
||||
|
||||
|
||||
async def redeem_login_by_waiting(
|
||||
login_id: str, *, timeout_seconds: float = 300
|
||||
) -> AgyLoginResult:
|
||||
async def redeem_login_by_waiting(login_id: str, *, timeout_seconds: float = 300) -> AgyLoginResult:
|
||||
"""Listener mode: capture the redirect on the loopback callback port.
|
||||
|
||||
Used by the operator-side script (and by a same-machine deployment): Google
|
||||
@@ -453,9 +465,7 @@ async def redeem_login_by_waiting(
|
||||
loop = asyncio.get_running_loop()
|
||||
future: asyncio.Future[str] = loop.create_future()
|
||||
|
||||
async def _handle(
|
||||
reader: asyncio.StreamReader, writer: asyncio.StreamWriter
|
||||
) -> None:
|
||||
async def _handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
|
||||
try:
|
||||
request_line = await reader.readline()
|
||||
parts = request_line.decode("latin-1").split(" ")
|
||||
@@ -579,8 +589,10 @@ class AgyModelAdapter:
|
||||
temperature: float = 0.2,
|
||||
max_tokens: int = 2048,
|
||||
) -> ModelResponse:
|
||||
pool = credentials if credentials is not None else (
|
||||
[credential] if credential is not None else await self._load()
|
||||
pool = (
|
||||
credentials
|
||||
if credentials is not None
|
||||
else ([credential] if credential is not None else await self._load())
|
||||
)
|
||||
accounts = [a for a in (_account_of(c) for c in pool) if a.refresh_token]
|
||||
if not accounts:
|
||||
@@ -593,8 +605,12 @@ class AgyModelAdapter:
|
||||
for account in ordered:
|
||||
try:
|
||||
response = await self._complete_with(
|
||||
account, model=model, messages=messages, system=system,
|
||||
temperature=temperature, max_tokens=max_tokens,
|
||||
account,
|
||||
model=model,
|
||||
messages=messages,
|
||||
system=system,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
except _AgyRateLimited as exc:
|
||||
failures.append(f"{account.email or account.credential_id}: محدودیت نرخ")
|
||||
@@ -630,24 +646,30 @@ class AgyModelAdapter:
|
||||
accounts = [a for a in (_account_of(c) for c in await self._load()) if a.refresh_token]
|
||||
if not accounts:
|
||||
return []
|
||||
return await self._fetch_models(accounts[0])
|
||||
items = await self._fetch_models(accounts[0])
|
||||
return [m.id for m in items]
|
||||
|
||||
async def available_models_for(self, credential: ModelCredential) -> list[str]:
|
||||
async def available_models_for(self, credential: ModelCredential) -> list[AgyAccountModelInfo]:
|
||||
"""Live model catalog of exactly one account.
|
||||
|
||||
Called per account selection in the admin console, so the list is
|
||||
fetched fresh from Google every time (the access token is cached, the
|
||||
``fetchAvailableModels`` round-trip is not) and reflects what *this*
|
||||
subscription is entitled to.
|
||||
subscription is entitled to along with each model's supported efforts.
|
||||
"""
|
||||
account = _account_of(credential)
|
||||
if not account.refresh_token:
|
||||
raise ProviderUnavailableError(
|
||||
"این اکانت اطلاعات ورود ندارد؛ یک بار دیگر لاگین کنید."
|
||||
)
|
||||
raise ProviderUnavailableError("این اکانت اطلاعات ورود ندارد؛ یک بار دیگر لاگین کنید.")
|
||||
return await self._fetch_models(account)
|
||||
|
||||
async def _fetch_models(self, account: AgyAccount) -> list[str]:
|
||||
async def _fetch_models(self, account: AgyAccount) -> list[AgyAccountModelInfo]:
|
||||
# Priority 1: Query models dynamically using headless agy CLI for this account
|
||||
if not os.environ.get("PYTEST_CURRENT_TEST") or os.environ.get("AGY_TEST_CLI"):
|
||||
cli_models = await self._fetch_models_via_cli(account)
|
||||
if cli_models:
|
||||
return cli_models
|
||||
|
||||
# Priority 2: Direct HTTP endpoint (used as fallback or in unit tests with HTTP mocks)
|
||||
token = await self._access_token(account)
|
||||
for base in BASE_URLS:
|
||||
try:
|
||||
@@ -663,11 +685,218 @@ class AgyModelAdapter:
|
||||
)
|
||||
if response.is_success:
|
||||
models = response.json().get("models") or {}
|
||||
return sorted(models)
|
||||
return self._parse_model_catalog(models)
|
||||
except httpx.HTTPError:
|
||||
continue
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
async def _fetch_models_via_cli(cls, account: AgyAccount) -> list[AgyAccountModelInfo]:
|
||||
"""Query models live from the headless `agy` CLI application for this account."""
|
||||
agy_bin = shutil.which("agy") or (
|
||||
"/root/.local/bin/agy" if os.path.exists("/root/.local/bin/agy") else None
|
||||
)
|
||||
if not agy_bin or not account.refresh_token:
|
||||
return []
|
||||
|
||||
# Prepare isolated app data directory with this account's oauth credentials
|
||||
account_data_dir = tempfile.mkdtemp(prefix=f"agy_acc_{account.credential_id[:8]}_")
|
||||
token_path = os.path.join(account_data_dir, "antigravity-oauth-token")
|
||||
try:
|
||||
with open(token_path, "w", encoding="utf-8") as f:
|
||||
json.dump(
|
||||
{
|
||||
"token": {
|
||||
"access_token": "",
|
||||
"token_type": "Bearer",
|
||||
"refresh_token": account.refresh_token,
|
||||
"expiry": "2000-01-01T00:00:00Z",
|
||||
},
|
||||
"auth_method": "oauth",
|
||||
},
|
||||
f,
|
||||
)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["ANTIGRAVITY_APP_DATA_DIR"] = account_data_dir
|
||||
env["HOME"] = "/root"
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
agy_bin,
|
||||
"models",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env=env,
|
||||
)
|
||||
try:
|
||||
stdout_bytes, _ = await asyncio.wait_for(proc.communicate(), timeout=20.0)
|
||||
except TimeoutError:
|
||||
proc.kill()
|
||||
return []
|
||||
|
||||
if proc.returncode != 0:
|
||||
return []
|
||||
|
||||
return cls._parse_cli_models_output(stdout_bytes.decode("utf-8", errors="replace"))
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to fetch models via agy CLI, falling back to HTTP", exc_info=True
|
||||
)
|
||||
return []
|
||||
finally:
|
||||
shutil.rmtree(account_data_dir, ignore_errors=True)
|
||||
|
||||
@classmethod
|
||||
def _parse_cli_models_output(cls, output: str) -> list[AgyAccountModelInfo]:
|
||||
items: list[AgyAccountModelInfo] = []
|
||||
for line in output.splitlines():
|
||||
line = line.strip()
|
||||
if not line or "Fetching" in line:
|
||||
continue
|
||||
# Output format is tab-separated: <model_id>\t<display_name>
|
||||
parts = line.split("\t", 1)
|
||||
if len(parts) == 2:
|
||||
model_id = parts[0].strip()
|
||||
display_name = parts[1].strip()
|
||||
else:
|
||||
tokens = line.split(None, 1)
|
||||
model_id = tokens[0].strip()
|
||||
display_name = tokens[1].strip() if len(tokens) > 1 else model_id
|
||||
|
||||
lowered = model_id.lower()
|
||||
dn_lowered = display_name.lower()
|
||||
|
||||
supports_thinking = (
|
||||
"thinking" in lowered
|
||||
or "thinking" in dn_lowered
|
||||
or "-high" in lowered
|
||||
or "-medium" in lowered
|
||||
or "-low" in lowered
|
||||
)
|
||||
|
||||
supported_efforts: list[str] = []
|
||||
default_effort: str | None = None
|
||||
if "-high" in lowered or "(high)" in dn_lowered:
|
||||
supported_efforts = ["high"]
|
||||
default_effort = "high"
|
||||
elif "-medium" in lowered or "(medium)" in dn_lowered:
|
||||
supported_efforts = ["medium"]
|
||||
default_effort = "medium"
|
||||
elif "-low" in lowered or "(low)" in dn_lowered:
|
||||
supported_efforts = ["low"]
|
||||
default_effort = "low"
|
||||
elif "thinking" in lowered or "thinking" in dn_lowered:
|
||||
if "gemini" in lowered:
|
||||
supported_efforts = ["low", "medium", "high"]
|
||||
default_effort = "high"
|
||||
else:
|
||||
supported_efforts = []
|
||||
default_effort = None
|
||||
|
||||
items.append(
|
||||
AgyAccountModelInfo(
|
||||
id=model_id,
|
||||
display_name=display_name,
|
||||
supports_thinking=supports_thinking,
|
||||
supported_efforts=supported_efforts,
|
||||
default_effort=default_effort,
|
||||
supports_images=True,
|
||||
)
|
||||
)
|
||||
return items
|
||||
|
||||
@classmethod
|
||||
def _parse_model_catalog(cls, raw_models: dict[str, Any]) -> list[AgyAccountModelInfo]:
|
||||
items: list[AgyAccountModelInfo] = []
|
||||
for model_id, details in sorted(raw_models.items()):
|
||||
if not isinstance(details, dict):
|
||||
details = {}
|
||||
# Omit internal non-generative / tab-completion preview models
|
||||
if details.get("isInternal") and not details.get("displayName"):
|
||||
continue
|
||||
if model_id.startswith("tab_") or model_id.startswith("chat_"):
|
||||
continue
|
||||
|
||||
raw_dn = details.get("displayName")
|
||||
if raw_dn and str(raw_dn).strip():
|
||||
display_name = str(raw_dn).strip()
|
||||
else:
|
||||
display_name = cls._format_model_name(model_id)
|
||||
supports_images = bool(details.get("supportsImages"))
|
||||
supports_thinking = bool(details.get("supportsThinking"))
|
||||
if not supports_thinking and _supports_thinking(model_id):
|
||||
supports_thinking = True
|
||||
|
||||
supported_efforts = cls._supported_efforts_for(model_id, details)
|
||||
default_effort = cls._default_effort_for(supported_efforts)
|
||||
|
||||
items.append(
|
||||
AgyAccountModelInfo(
|
||||
id=model_id,
|
||||
display_name=display_name,
|
||||
supports_thinking=supports_thinking,
|
||||
supported_efforts=supported_efforts,
|
||||
default_effort=default_effort,
|
||||
supports_images=supports_images,
|
||||
)
|
||||
)
|
||||
return items
|
||||
|
||||
@classmethod
|
||||
def _format_model_name(cls, model_id: str) -> str:
|
||||
"""Dynamic fallback formatter for models without a displayName from agy."""
|
||||
parts = model_id.split("-")
|
||||
words: list[str] = []
|
||||
for p in parts:
|
||||
if p == "tiered":
|
||||
words.append("(Tiered)")
|
||||
elif p in ("agent", "thinking", "image", "preview"):
|
||||
words.append(f"({p.title()})")
|
||||
elif p.isdigit() or any(c.isdigit() for c in p):
|
||||
words.append(p)
|
||||
else:
|
||||
words.append(p.title())
|
||||
return " ".join(words)
|
||||
|
||||
@classmethod
|
||||
def _supported_efforts_for(cls, model_id: str, details: dict[str, Any]) -> list[str]:
|
||||
lowered = model_id.lower()
|
||||
if "image" in lowered:
|
||||
return []
|
||||
if "claude" in lowered:
|
||||
return []
|
||||
|
||||
supports_thinking = details.get("supportsThinking")
|
||||
if supports_thinking is False:
|
||||
return []
|
||||
if supports_thinking is None and not _supports_thinking(model_id):
|
||||
return []
|
||||
|
||||
if "gpt" in lowered:
|
||||
return ["medium"]
|
||||
if "pro" in lowered and "3.1" in lowered:
|
||||
return ["low", "high"]
|
||||
|
||||
budget = details.get("thinkingBudget")
|
||||
if isinstance(budget, int) and budget > 0:
|
||||
if budget <= 4096:
|
||||
return ["low"]
|
||||
if budget <= 8192:
|
||||
return ["medium"]
|
||||
return ["high"]
|
||||
|
||||
return ["low", "medium", "high"]
|
||||
|
||||
@staticmethod
|
||||
def _default_effort_for(supported: list[str]) -> str | None:
|
||||
if not supported:
|
||||
return None
|
||||
if "high" in supported:
|
||||
return "high"
|
||||
if "medium" in supported:
|
||||
return "medium"
|
||||
return supported[0]
|
||||
|
||||
# -- internals ----------------------------------------------------------
|
||||
|
||||
async def _load(self) -> list[ModelCredential]:
|
||||
@@ -758,7 +987,12 @@ class AgyModelAdapter:
|
||||
) -> ModelResponse:
|
||||
token = await self._access_token(account)
|
||||
payload = _build_generate_payload(
|
||||
account, model.internal_name, messages, system, temperature, max_tokens,
|
||||
account,
|
||||
model.internal_name,
|
||||
messages,
|
||||
system,
|
||||
temperature,
|
||||
max_tokens,
|
||||
reasoning_effort=model.reasoning_effort,
|
||||
)
|
||||
headers = {
|
||||
@@ -783,9 +1017,9 @@ class AgyModelAdapter:
|
||||
# Both backends share one quota, but a 429 can also be a
|
||||
# backend hiccup — try the other base URL before benching the
|
||||
# whole account.
|
||||
retry_after = _retry_after_seconds(
|
||||
response.headers.get("Retry-After")
|
||||
) or retry_after
|
||||
retry_after = (
|
||||
_retry_after_seconds(response.headers.get("Retry-After")) or retry_after
|
||||
)
|
||||
continue
|
||||
if response.status_code in (401, 403):
|
||||
raise _AgyAuthError(
|
||||
@@ -798,12 +1032,9 @@ class AgyModelAdapter:
|
||||
# Our request shape is wrong — another account will fail the
|
||||
# same way, so surface it instead of burning the pool.
|
||||
raise ProviderUnavailableError(
|
||||
f"agy درخواست را رد کرد ({response.status_code}): "
|
||||
f"{response.text[:300]}"
|
||||
f"agy درخواست را رد کرد ({response.status_code}): {response.text[:300]}"
|
||||
)
|
||||
return _parse_generate_response(
|
||||
response.json(), model=model, started=started
|
||||
)
|
||||
return _parse_generate_response(response.json(), model=model, started=started)
|
||||
if retry_after is not None:
|
||||
raise _AgyRateLimited(retry_after)
|
||||
raise last_transient or _AgyTransientError("هیچ بکاند agy پاسخ نداد.")
|
||||
@@ -833,14 +1064,26 @@ def _build_generate_payload(
|
||||
role = "model" if message.role == "assistant" else "user"
|
||||
contents.append({"role": role, "parts": [{"text": message.content}]})
|
||||
|
||||
backend_model = model_name
|
||||
effective_effort = reasoning_effort
|
||||
if model_name == "gemini-3.8-flash-high":
|
||||
backend_model = "gemini-3.8-flash-tiered"
|
||||
effective_effort = effective_effort or "high"
|
||||
elif model_name == "gemini-3.8-flash-medium":
|
||||
backend_model = "gemini-3.8-flash-tiered"
|
||||
effective_effort = effective_effort or "medium"
|
||||
elif model_name == "gemini-3.8-flash-low":
|
||||
backend_model = "gemini-3.8-flash-tiered"
|
||||
effective_effort = effective_effort or "low"
|
||||
|
||||
generation: dict = {
|
||||
"temperature": temperature,
|
||||
"maxOutputTokens": max_tokens,
|
||||
}
|
||||
# Reasoning effort is only sent to models that can think; the backend
|
||||
# rejects ``thinkingConfig`` everywhere else with a 400.
|
||||
if reasoning_effort and _supports_thinking(model_name):
|
||||
budget = EFFORT_THINKING_BUDGETS.get(reasoning_effort, 8192)
|
||||
if effective_effort and _supports_thinking(backend_model):
|
||||
budget = EFFORT_THINKING_BUDGETS.get(effective_effort, 8192)
|
||||
generation["thinkingConfig"] = {
|
||||
"thinkingBudget": budget,
|
||||
"includeThoughts": True,
|
||||
@@ -864,16 +1107,14 @@ def _build_generate_payload(
|
||||
}
|
||||
return {
|
||||
"project": account.project_id or _fallback_project_id(),
|
||||
"model": model_name,
|
||||
"model": backend_model,
|
||||
"userAgent": "antigravity",
|
||||
"requestId": f"agent-{uuid.uuid4()}",
|
||||
"request": request,
|
||||
}
|
||||
|
||||
|
||||
def _parse_generate_response(
|
||||
data: dict, *, model: AiModel, started: float
|
||||
) -> ModelResponse:
|
||||
def _parse_generate_response(data: dict, *, model: AiModel, started: float) -> ModelResponse:
|
||||
candidates = data.get("candidates") or [{}]
|
||||
candidate = candidates[0] if isinstance(candidates[0], dict) else {}
|
||||
text_parts: list[str] = []
|
||||
@@ -914,14 +1155,10 @@ async def load_agy_accounts() -> list[ModelCredential]:
|
||||
|
||||
async with SessionFactory() as session:
|
||||
providers = await ProviderRepository(session).list()
|
||||
provider = next(
|
||||
(p for p in providers if p.type.value == "agy" and p.is_active), None
|
||||
)
|
||||
provider = next((p for p in providers if p.type.value == "agy" and p.is_active), None)
|
||||
if provider is None:
|
||||
return []
|
||||
return await ModelCredentialRepository(session).list_active_for_provider(
|
||||
provider.id
|
||||
)
|
||||
return await ModelCredentialRepository(session).list_active_for_provider(provider.id)
|
||||
|
||||
|
||||
async def report_agy_state(
|
||||
|
||||
@@ -255,6 +255,19 @@ async def record_error(
|
||||
request_id or "-",
|
||||
fingerprint,
|
||||
)
|
||||
|
||||
# Live push to the platform-owner console. Best-effort like everything here:
|
||||
# a failed publish must not turn a recorded error into an unrecorded one.
|
||||
from app.application.ports.realtime import emit_error_recorded
|
||||
|
||||
await emit_error_recorded(
|
||||
error_id=event.id,
|
||||
kind=event.kind,
|
||||
message=event.message,
|
||||
severity=str(severity),
|
||||
source=str(source),
|
||||
organization_id=event.organization_id,
|
||||
)
|
||||
return event.id
|
||||
|
||||
|
||||
|
||||
@@ -281,6 +281,14 @@ class AiModelRepository:
|
||||
await self.session.flush()
|
||||
return _model_to_domain(row)
|
||||
|
||||
async def delete(self, model_id: str) -> bool:
|
||||
row = await self.session.get(AiModelRow, model_id)
|
||||
if row is None:
|
||||
return False
|
||||
await self.session.delete(row)
|
||||
await self.session.flush()
|
||||
return True
|
||||
|
||||
async def pick_for_mode(self, mode: RoutingMode) -> AiModel | None:
|
||||
"""Choose a model for a routing mode.
|
||||
|
||||
|
||||
@@ -215,6 +215,28 @@ async def create_model(
|
||||
return _model_out(model)
|
||||
|
||||
|
||||
@router.delete("/models/{model_id}", status_code=status.HTTP_200_OK)
|
||||
async def delete_model(
|
||||
model_id: str,
|
||||
request: Request,
|
||||
owner: User = Depends(require_platform_owner),
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> dict:
|
||||
repo = AiModelRepository(session)
|
||||
existing = await repo.get(model_id)
|
||||
if existing is None:
|
||||
raise NotFoundError("مدلی با این شناسه یافت نشد.")
|
||||
await repo.delete(model_id)
|
||||
ip, ua = client_meta(request)
|
||||
await record_audit(
|
||||
session, action=AuditAction.MODEL_CHANGE, actor_user_id=owner.id,
|
||||
target_type="model", target_id=model_id,
|
||||
metadata={"internal_name": existing.internal_name, "deleted": True},
|
||||
ip=ip, user_agent=ua,
|
||||
)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Routing policies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -105,6 +105,15 @@ class AgyTestOut(BaseModel):
|
||||
output_tokens: int
|
||||
|
||||
|
||||
class AgyAccountModelOut(BaseModel):
|
||||
id: str
|
||||
display_name: str
|
||||
supports_thinking: bool = False
|
||||
supported_efforts: list[str] = Field(default_factory=list)
|
||||
default_effort: str | None = None
|
||||
supports_images: bool = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -331,16 +340,16 @@ async def test_agy_account(
|
||||
)
|
||||
|
||||
|
||||
@router.get("/accounts/{credential_id}/models", response_model=list[str])
|
||||
@router.get("/accounts/{credential_id}/models", response_model=list[AgyAccountModelOut])
|
||||
async def list_account_models(
|
||||
credential_id: str,
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> list[str]:
|
||||
) -> list[AgyAccountModelOut]:
|
||||
"""Live model catalog of one account — fetched from Google on every call.
|
||||
|
||||
The admin console calls this when an account is selected, so the model
|
||||
list always reflects that subscription's current entitlement instead of a
|
||||
cached snapshot.
|
||||
list always reflects that subscription's current entitlement along with
|
||||
the supported reasoning efforts for each model.
|
||||
"""
|
||||
from app.core.errors import ValidationError
|
||||
from app.infrastructure.adapters.agy import AgyModelAdapter
|
||||
@@ -352,7 +361,18 @@ async def list_account_models(
|
||||
|
||||
adapter = AgyModelAdapter() # stateless: one forced account
|
||||
try:
|
||||
return await adapter.available_models_for(credential)
|
||||
models = await adapter.available_models_for(credential)
|
||||
return [
|
||||
AgyAccountModelOut(
|
||||
id=m.id,
|
||||
display_name=m.display_name,
|
||||
supports_thinking=m.supports_thinking,
|
||||
supported_efforts=m.supported_efforts,
|
||||
default_effort=m.default_effort,
|
||||
supports_images=m.supports_images,
|
||||
)
|
||||
for m in models
|
||||
]
|
||||
except ValidationError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - the console wants the reason
|
||||
|
||||
@@ -42,5 +42,22 @@ async def readiness(session: AsyncSession = Depends(get_session)) -> dict:
|
||||
"sandbox": settings.sandbox_provider,
|
||||
"payment": settings.payment_provider,
|
||||
}
|
||||
|
||||
# Live-feed visibility: how many browser sockets this process is serving,
|
||||
# and whether the cross-process relay (Redis) is connected. An operator
|
||||
# wondering "is realtime working?" should not have to read the logs.
|
||||
try:
|
||||
from app.interfaces.realtime.bus import get_bus
|
||||
from app.interfaces.realtime.router import realtime_manager
|
||||
|
||||
manager = realtime_manager()
|
||||
checks["realtime"] = {
|
||||
"connections": manager.connection_count,
|
||||
"channels": manager.channel_count,
|
||||
"redis_relay": "connected" if get_bus().redis_healthy else "in_process_only",
|
||||
}
|
||||
except Exception: # noqa: BLE001 - health must answer regardless
|
||||
checks["realtime"] = "unavailable"
|
||||
|
||||
healthy = checks.get("database") == "ok"
|
||||
return {"status": "ok" if healthy else "degraded", "checks": checks}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Realtime push layer — the WebSocket *interface* adapter.
|
||||
|
||||
The adapter is transport and plumbing only: who is connected to which channel
|
||||
(:mod:`manager`), how an event travels between processes (:mod:`bus`), which
|
||||
channels a socket may subscribe to (:mod:`authz`), and the wire protocol
|
||||
(:mod:`router`).
|
||||
|
||||
The event contract and the typed publishers live one layer in, in
|
||||
``app.application.ports.realtime`` — services depend on the port, this package
|
||||
implements it. ``app.main`` is the composition root that installs the bus as
|
||||
the process publisher and starts the cross-process relay.
|
||||
"""
|
||||
|
||||
from app.interfaces.realtime.bus import get_bus, start_relay, stop_relay
|
||||
|
||||
__all__ = [
|
||||
"get_bus",
|
||||
"start_relay",
|
||||
"stop_relay",
|
||||
]
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Channel authorization — the only gate on what a socket may receive.
|
||||
|
||||
Authorization happens once, at subscribe time. After that the connection holds
|
||||
plain channel names and the manager routes on them, so there is no code path
|
||||
that can deliver an event to a connection that was never checked.
|
||||
|
||||
Every rule resolves the channel's tenant through the repositories and then the
|
||||
caller's membership in it — exactly like the HTTP layer's ``get_org_context``.
|
||||
A tenant that does not own the object gets a rejection, not an empty stream:
|
||||
the difference matters when debugging "why is my page not live?".
|
||||
|
||||
The caller's identity arrives as two primitives (id, platform-owner flag) —
|
||||
what an authenticated socket actually knows — rather than a ``User`` row, so a
|
||||
subscribe costs one fewer database read.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.infrastructure.repositories.identity import MembershipRepository
|
||||
from app.infrastructure.repositories.sandbox import (
|
||||
SandboxSessionRepository,
|
||||
WorkspaceRepository,
|
||||
)
|
||||
from app.infrastructure.repositories.workforce import (
|
||||
RunRepository,
|
||||
TaskRepository,
|
||||
ThreadRepository,
|
||||
)
|
||||
|
||||
|
||||
async def authorize_channel(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
user_id: str,
|
||||
is_platform_owner: bool,
|
||||
channel: str,
|
||||
) -> bool:
|
||||
"""May this identity subscribe to ``channel``? Deny by default."""
|
||||
kind, _, identifier = channel.partition(":")
|
||||
if not identifier and kind != "admin":
|
||||
return False
|
||||
|
||||
if kind == "admin":
|
||||
return is_platform_owner
|
||||
if kind == "user":
|
||||
return identifier == user_id
|
||||
if kind == "org":
|
||||
return await _is_member(
|
||||
session=session, user_id=user_id, organization_id=identifier
|
||||
)
|
||||
if kind == "task":
|
||||
return await _task_is_visible(
|
||||
session=session, user_id=user_id, task_id=identifier
|
||||
)
|
||||
if kind == "run":
|
||||
return await _run_is_visible(
|
||||
session=session, user_id=user_id, run_id=identifier
|
||||
)
|
||||
if kind == "thread":
|
||||
return await _thread_is_visible(
|
||||
session=session, user_id=user_id, thread_id=identifier
|
||||
)
|
||||
if kind == "session":
|
||||
return await _session_is_visible(
|
||||
session=session, user_id=user_id, session_id=identifier
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
async def _is_member(*, session: AsyncSession, user_id: str, organization_id: str) -> bool:
|
||||
membership = await MembershipRepository(session).get(organization_id, user_id)
|
||||
return membership is not None and membership.is_active
|
||||
|
||||
|
||||
async def _task_is_visible(*, session: AsyncSession, user_id: str, task_id: str) -> bool:
|
||||
task = await TaskRepository(session).get(task_id)
|
||||
if task is None:
|
||||
return False
|
||||
return await _is_member(
|
||||
session=session, user_id=user_id, organization_id=task.organization_id
|
||||
)
|
||||
|
||||
|
||||
async def _run_is_visible(*, session: AsyncSession, user_id: str, run_id: str) -> bool:
|
||||
run = await RunRepository(session).get(run_id)
|
||||
if run is None:
|
||||
return False
|
||||
return await _task_is_visible(session=session, user_id=user_id, task_id=run.task_id)
|
||||
|
||||
|
||||
async def _thread_is_visible(*, session: AsyncSession, user_id: str, thread_id: str) -> bool:
|
||||
thread = await ThreadRepository(session).get(thread_id)
|
||||
if thread is None:
|
||||
return False
|
||||
return await _is_member(
|
||||
session=session, user_id=user_id, organization_id=thread.organization_id
|
||||
)
|
||||
|
||||
|
||||
async def _session_is_visible(*, session: AsyncSession, user_id: str, session_id: str) -> bool:
|
||||
"""A sandbox session belongs to a workspace, which belongs to a tenant."""
|
||||
sandbox_session = await SandboxSessionRepository(session).get(session_id)
|
||||
if sandbox_session is None:
|
||||
return False
|
||||
workspace = await WorkspaceRepository(session).get(sandbox_session.workspace_id)
|
||||
if workspace is None:
|
||||
return False
|
||||
return await _is_member(
|
||||
session=session, user_id=user_id, organization_id=workspace.organization_id
|
||||
)
|
||||
@@ -0,0 +1,260 @@
|
||||
"""The event bus: how a published event reaches connected browsers.
|
||||
|
||||
Two transports, in this order:
|
||||
|
||||
1. **In-process.** Always on. When the API itself records the event (a task run
|
||||
driven by a button, a sandbox command, an error handler), the fan-out is a
|
||||
direct call into the ``ConnectionManager`` — microseconds, no broker.
|
||||
2. **Redis pub/sub.** Carries events published by *other* processes: the
|
||||
standalone workers (``pnpm worker:ai-runtime``) run tasks in their own
|
||||
process, and without a broker their events would die there. The API process
|
||||
relays whatever arrives on the shared channel into the same manager.
|
||||
|
||||
Redis is a bonus, never a dependency:
|
||||
|
||||
* The bus tries it once per cooldown window. If nothing is listening, the
|
||||
attempt fails quietly and in-process delivery continues — the product stays
|
||||
fully live for every API-driven action, only cross-process events are lost.
|
||||
* The relay task owns the subscription lifecycle; it starts with the app and
|
||||
stops on shutdown, and reconnects on its own after a broker restart.
|
||||
|
||||
Nothing here ever raises. Publishing is called from request handlers, from
|
||||
inside exception handlers and from worker passes; an exception in the feed
|
||||
must not be able to fail any of those.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from contextlib import suppress
|
||||
|
||||
from app.application.ports.realtime import RealtimeEvent, set_publisher
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger("agy.realtime")
|
||||
|
||||
#: Redis pub/sub channel shared by every process of one deployment.
|
||||
DEFAULT_REDIS_CHANNEL = "agy:realtime"
|
||||
|
||||
#: After a failed broker attempt, wait this long before trying again. Keeps a
|
||||
#: dev machine without Redis from adding a connection timeout to every event.
|
||||
REDIS_RETRY_COOLDOWN_SECONDS = 10.0
|
||||
|
||||
#: How long a single connection attempt may take before it is abandoned.
|
||||
REDIS_CONNECT_TIMEOUT_SECONDS = 1.0
|
||||
|
||||
#: Listener signature: ``await listener(event)``. The ConnectionManager
|
||||
#: registers one; tests register fakes. The return value is ignored so a
|
||||
#: listener may report how many connections it fanned out to.
|
||||
EventListener = Callable[[RealtimeEvent], Awaitable[object]]
|
||||
|
||||
|
||||
class RealtimeBus:
|
||||
"""Fan-out hub shared by every publisher in this process."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
redis_url: str | None = None,
|
||||
channel: str = DEFAULT_REDIS_CHANNEL,
|
||||
) -> None:
|
||||
self._listeners: set[EventListener] = set()
|
||||
self._redis_url = redis_url if redis_url is not None else settings.redis_url
|
||||
self._channel = channel
|
||||
self._redis = None
|
||||
self._pubsub = None
|
||||
self._relay_task: asyncio.Task | None = None
|
||||
self._next_redis_attempt = 0.0
|
||||
self._redis_healthy = False
|
||||
|
||||
# -- listeners (in-process fan-out) --------------------------------------
|
||||
|
||||
def attach(self, listener: EventListener) -> None:
|
||||
self._listeners.add(listener)
|
||||
|
||||
def detach(self, listener: EventListener) -> None:
|
||||
self._listeners.discard(listener)
|
||||
|
||||
# -- publishing -----------------------------------------------------------
|
||||
|
||||
async def publish(self, event: RealtimeEvent) -> None:
|
||||
"""Deliver locally, then mirror to other processes. Never raises."""
|
||||
try:
|
||||
await self._dispatch_local(event)
|
||||
except Exception: # noqa: BLE001 - the feed must never fail a caller
|
||||
logger.exception("realtime: local dispatch failed for %s", event.type)
|
||||
|
||||
if not self._redis_enabled():
|
||||
return
|
||||
try:
|
||||
await self._publish_redis(event)
|
||||
except Exception: # noqa: BLE001 - same rule
|
||||
logger.exception("realtime: redis publish failed for %s", event.type)
|
||||
|
||||
async def _dispatch_local(self, event: RealtimeEvent) -> None:
|
||||
for listener in list(self._listeners):
|
||||
try:
|
||||
await listener(event)
|
||||
except Exception: # noqa: BLE001 - one bad listener is not fatal
|
||||
logger.exception("realtime: listener failed for %s", event.type)
|
||||
|
||||
def _redis_enabled(self) -> bool:
|
||||
# "local" pins the bus to in-process delivery — used by tests and by
|
||||
# single-process deployments that have no broker on purpose.
|
||||
return getattr(settings, "realtime_transport", "auto").lower() != "local"
|
||||
|
||||
async def _publish_redis(self, event: RealtimeEvent) -> None:
|
||||
client = await self._redis_client()
|
||||
if client is None:
|
||||
return
|
||||
await client.publish(self._channel, event.to_json())
|
||||
|
||||
async def _redis_client(self):
|
||||
"""Lazily connect, with a cooldown after failure. ``None`` if down."""
|
||||
if self._redis is not None:
|
||||
return self._redis
|
||||
now = time.monotonic()
|
||||
if now < self._next_redis_attempt:
|
||||
return None
|
||||
self._next_redis_attempt = now + REDIS_RETRY_COOLDOWN_SECONDS
|
||||
try:
|
||||
import redis.asyncio as aioredis
|
||||
|
||||
client = aioredis.from_url(
|
||||
self._redis_url,
|
||||
encoding="utf-8",
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=REDIS_CONNECT_TIMEOUT_SECONDS,
|
||||
socket_timeout=REDIS_CONNECT_TIMEOUT_SECONDS,
|
||||
)
|
||||
await client.ping()
|
||||
self._redis = client
|
||||
self._redis_healthy = True
|
||||
logger.info("realtime: redis pub/sub connected (%s)", self._channel)
|
||||
except Exception as exc: # noqa: BLE001 - broker is optional
|
||||
self._redis = None
|
||||
self._redis_healthy = False
|
||||
logger.debug("realtime: redis unavailable (%s); in-process only", exc)
|
||||
return None
|
||||
return self._redis
|
||||
|
||||
# -- relay (other processes -> this one's browsers) ----------------------
|
||||
|
||||
async def start_relay(self) -> None:
|
||||
"""Subscribe to the shared channel and feed the local manager."""
|
||||
if self._relay_task is not None and not self._relay_task.done():
|
||||
return
|
||||
self._relay_task = asyncio.create_task(self._relay_loop(), name="realtime-relay")
|
||||
|
||||
async def stop_relay(self) -> None:
|
||||
task = self._relay_task
|
||||
self._relay_task = None
|
||||
if task is not None:
|
||||
task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task
|
||||
await self._close_redis()
|
||||
|
||||
async def _relay_loop(self) -> None:
|
||||
while True:
|
||||
client = await self._redis_client()
|
||||
if client is None:
|
||||
# Broker absent (or down mid-flight): wait out the cooldown and
|
||||
# try again — a Redis started after the API must still work.
|
||||
await asyncio.sleep(REDIS_RETRY_COOLDOWN_SECONDS)
|
||||
continue
|
||||
try:
|
||||
pubsub = client.pubsub()
|
||||
await pubsub.subscribe(self._channel)
|
||||
self._pubsub = pubsub
|
||||
logger.info("realtime: relay subscribed to %s", self._channel)
|
||||
async for message in pubsub.listen():
|
||||
if message.get("type") != "message":
|
||||
continue
|
||||
event = _decode_message(message.get("data"))
|
||||
if event is not None:
|
||||
# Dedupe by id happens in the manager, so an event this
|
||||
# process published itself is not delivered twice.
|
||||
await self._dispatch_local(event)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - relay must survive
|
||||
self._redis_healthy = False
|
||||
logger.warning("realtime: relay interrupted (%s); reconnecting", exc)
|
||||
finally:
|
||||
if self._pubsub is not None:
|
||||
with suppress(Exception):
|
||||
await self._pubsub.unsubscribe(self._channel)
|
||||
self._pubsub = None
|
||||
self._redis = None
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
async def _close_redis(self) -> None:
|
||||
client, self._redis = self._redis, None
|
||||
if client is not None:
|
||||
with suppress(Exception):
|
||||
await client.aclose()
|
||||
self._redis_healthy = False
|
||||
|
||||
# -- introspection -------------------------------------------------------
|
||||
|
||||
@property
|
||||
def redis_healthy(self) -> bool:
|
||||
return self._redis_healthy
|
||||
|
||||
@property
|
||||
def listener_count(self) -> int:
|
||||
return len(self._listeners)
|
||||
|
||||
|
||||
def _decode_message(raw: object) -> RealtimeEvent | None:
|
||||
if not isinstance(raw, (str, bytes)):
|
||||
return None
|
||||
try:
|
||||
return RealtimeEvent.from_json(raw)
|
||||
except Exception: # noqa: BLE001 - a bad frame is skipped, not fatal
|
||||
logger.warning("realtime: undecodable relay frame dropped")
|
||||
return None
|
||||
|
||||
|
||||
_bus: RealtimeBus | None = None
|
||||
|
||||
|
||||
def get_bus() -> RealtimeBus:
|
||||
"""Process-wide bus. Created lazily so importing never opens a socket."""
|
||||
global _bus
|
||||
if _bus is None:
|
||||
_bus = RealtimeBus()
|
||||
return _bus
|
||||
|
||||
|
||||
def install() -> RealtimeBus:
|
||||
"""Make this process's bus the application-layer publisher.
|
||||
|
||||
Called by the composition roots (``app.main`` for the API,
|
||||
``app.workers.__main__`` for the workers). Idempotent, so a worker that
|
||||
imports several modules pays nothing for calling it more than once.
|
||||
"""
|
||||
bus = get_bus()
|
||||
set_publisher(bus)
|
||||
return bus
|
||||
|
||||
|
||||
async def publish(event: RealtimeEvent) -> None:
|
||||
"""Publish through the shared bus. Never raises."""
|
||||
bus = get_bus()
|
||||
try:
|
||||
await bus.publish(event)
|
||||
except Exception: # noqa: BLE001 - belt and braces; publish is best-effort
|
||||
logger.exception("realtime: publish failed for %s", event.type)
|
||||
|
||||
|
||||
async def start_relay() -> None:
|
||||
await get_bus().start_relay()
|
||||
|
||||
|
||||
async def stop_relay() -> None:
|
||||
await get_bus().stop_relay()
|
||||
@@ -0,0 +1,220 @@
|
||||
"""Connection bookkeeping: who is connected, and to which channels.
|
||||
|
||||
The manager is deliberately transport-agnostic — it does not know it is talking
|
||||
to a WebSocket, only to an object with ``send_text``. That keeps it testable
|
||||
without a socket and lets a future transport (SSE, push bridge) reuse it.
|
||||
|
||||
Two invariants:
|
||||
|
||||
* **A connection only ever receives channels it was authorized for.** The
|
||||
authorizer runs at subscribe time; the routing table built afterwards can
|
||||
only contain pre-approved channel names.
|
||||
* **An event is delivered at most once.** ``dispatch`` de-duplicates by event
|
||||
id, because the same event reaches a process twice in the normal case: once
|
||||
in-process and once relayed from Redis.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections import deque
|
||||
from collections.abc import Awaitable, Callable
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from starlette.websockets import WebSocket, WebSocketState
|
||||
|
||||
from app.application.ports.realtime import RealtimeEvent
|
||||
|
||||
logger = logging.getLogger("agy.realtime")
|
||||
|
||||
#: Bounded dedupe memory. Large enough that a relayed duplicate is always
|
||||
#: caught, small enough that a chatty day costs a few kilobytes.
|
||||
MAX_SEEN_EVENTS = 2048
|
||||
|
||||
#: Per-connection outbox. If a browser stops reading (a frozen tab), a
|
||||
#: producer must not be able to grow this without bound. Overflow drops the
|
||||
#: oldest frame — ordering of what follows stays meaningful.
|
||||
MAX_QUEUE_DEPTH = 256
|
||||
|
||||
#: What a subscribe check looks like: ``(connection, channel) -> allowed``.
|
||||
ChannelAuthorizer = Callable[["Connection", str], Awaitable[bool]]
|
||||
|
||||
|
||||
#: ``eq=False`` on purpose: a connection is identified by object identity. The
|
||||
#: default dataclass ``__eq__`` would also make it unhashable (a ``set`` field
|
||||
#: cannot be compared safely), and the manager keys connections by identity in
|
||||
#: both its connection set and its channel routing table.
|
||||
@dataclass(slots=True, eq=False)
|
||||
class Connection:
|
||||
"""One browser socket. ``user_id`` is ``None`` for an anonymous (pre-login)
|
||||
connection: it may only call the bootstrap RPCs (login/signup) and may not
|
||||
subscribe to live channels."""
|
||||
|
||||
user_id: str | None
|
||||
is_platform_owner: bool
|
||||
websocket: WebSocket
|
||||
token: str | None = None
|
||||
channels: set[str] = field(default_factory=set)
|
||||
_queue: asyncio.Queue[str] = field(
|
||||
default_factory=lambda: asyncio.Queue(maxsize=MAX_QUEUE_DEPTH)
|
||||
)
|
||||
_sender: asyncio.Task | None = None
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Single writer task per connection.
|
||||
|
||||
One task owns the socket: concurrent ``send_text`` calls on one
|
||||
WebSocket interleave frames and corrupt the stream, so everything goes
|
||||
through this queue instead.
|
||||
"""
|
||||
if self._sender is None or self._sender.done():
|
||||
self._sender = asyncio.create_task(self._drain(), name="realtime-sender")
|
||||
|
||||
async def _drain(self) -> None:
|
||||
while True:
|
||||
payload = await self._queue.get()
|
||||
if payload is None: # shutdown sentinel
|
||||
return
|
||||
try:
|
||||
if self.websocket.client_state == WebSocketState.CONNECTED:
|
||||
await self.websocket.send_text(payload)
|
||||
except Exception: # noqa: BLE001 - a dead socket ends the connection
|
||||
logger.debug("realtime: send failed; closing connection", exc_info=True)
|
||||
return
|
||||
|
||||
def offer(self, payload: str) -> None:
|
||||
"""Queue a frame, never blocking the publisher.
|
||||
|
||||
A slow consumer loses the oldest frames rather than stalling every
|
||||
other tenant's stream behind it.
|
||||
"""
|
||||
if self._queue.full():
|
||||
with suppress(asyncio.QueueFull):
|
||||
# Drop the oldest frame to make room: ordering of what follows
|
||||
# stays meaningful, which is what the UI renders.
|
||||
self._queue.get_nowait()
|
||||
with suppress(asyncio.QueueFull): # pragma: no cover - defensive
|
||||
self._queue.put_nowait(payload)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._sender is not None:
|
||||
self._sender.cancel()
|
||||
self._sender = None
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
"""The routing table between events and open sockets."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._connections: set[Connection] = set()
|
||||
self._by_channel: dict[str, set[Connection]] = {}
|
||||
self._seen: deque[str] = deque(maxlen=MAX_SEEN_EVENTS)
|
||||
self._seen_set: set[str] = set()
|
||||
|
||||
# -- lifecycle -----------------------------------------------------------
|
||||
|
||||
async def register(self, connection: Connection) -> None:
|
||||
self._connections.add(connection)
|
||||
await connection.start()
|
||||
|
||||
async def unregister(self, connection: Connection) -> None:
|
||||
self._connections.discard(connection)
|
||||
for channel in list(connection.channels):
|
||||
subscribers = self._by_channel.get(channel)
|
||||
if subscribers is not None:
|
||||
subscribers.discard(connection)
|
||||
if not subscribers:
|
||||
self._by_channel.pop(channel, None)
|
||||
connection.channels.clear()
|
||||
await connection.close()
|
||||
|
||||
# -- subscriptions ---------------------------------------------------------
|
||||
|
||||
async def subscribe(
|
||||
self,
|
||||
connection: Connection,
|
||||
channels: list[str],
|
||||
authorizer: ChannelAuthorizer,
|
||||
) -> tuple[list[str], list[str]]:
|
||||
"""Authorize and subscribe. Returns ``(accepted, rejected)``."""
|
||||
accepted: list[str] = []
|
||||
rejected: list[str] = []
|
||||
for channel in channels:
|
||||
if channel in connection.channels:
|
||||
accepted.append(channel)
|
||||
continue
|
||||
try:
|
||||
allowed = await authorizer(connection, channel)
|
||||
except Exception: # noqa: BLE001 - a broken check denies, never allows
|
||||
logger.exception("realtime: channel authorization crashed")
|
||||
allowed = False
|
||||
if not allowed:
|
||||
rejected.append(channel)
|
||||
continue
|
||||
connection.channels.add(channel)
|
||||
self._by_channel.setdefault(channel, set()).add(connection)
|
||||
accepted.append(channel)
|
||||
return accepted, rejected
|
||||
|
||||
async def unsubscribe(self, connection: Connection, channels: list[str]) -> None:
|
||||
for channel in channels:
|
||||
connection.channels.discard(channel)
|
||||
subscribers = self._by_channel.get(channel)
|
||||
if subscribers is not None:
|
||||
subscribers.discard(connection)
|
||||
if not subscribers:
|
||||
self._by_channel.pop(channel, None)
|
||||
|
||||
# -- fan-out ---------------------------------------------------------------
|
||||
|
||||
async def dispatch(self, event: RealtimeEvent) -> int:
|
||||
"""Deliver to every subscriber of every channel. Returns fan-out count.
|
||||
|
||||
The id check is what makes "publish locally, then receive the same event
|
||||
from Redis" deliver once instead of twice.
|
||||
"""
|
||||
if event.id in self._seen_set:
|
||||
return 0
|
||||
self._remember(event.id)
|
||||
|
||||
recipients: set[Connection] = set()
|
||||
for channel in event.channels:
|
||||
recipients.update(self._by_channel.get(channel, ()))
|
||||
|
||||
envelope = event.envelope()
|
||||
# Control frames and event frames share one top-level discriminator
|
||||
# (``type``), so the client switches on one field for everything.
|
||||
payload = _dumps({"type": "event", "event": envelope})
|
||||
delivered = 0
|
||||
for connection in recipients:
|
||||
connection.offer(payload)
|
||||
delivered += 1
|
||||
return delivered
|
||||
|
||||
def _remember(self, event_id: str) -> None:
|
||||
if len(self._seen) == self._seen.maxlen:
|
||||
evicted = self._seen[0]
|
||||
self._seen_set.discard(evicted)
|
||||
self._seen.append(event_id)
|
||||
self._seen_set.add(event_id)
|
||||
|
||||
# -- introspection -------------------------------------------------------
|
||||
|
||||
@property
|
||||
def connection_count(self) -> int:
|
||||
return len(self._connections)
|
||||
|
||||
@property
|
||||
def channel_count(self) -> int:
|
||||
return len(self._by_channel)
|
||||
|
||||
def channel_load(self) -> dict[str, int]:
|
||||
return {channel: len(subs) for channel, subs in self._by_channel.items()}
|
||||
|
||||
|
||||
def _dumps(envelope: dict) -> str:
|
||||
import json
|
||||
|
||||
return json.dumps(envelope, ensure_ascii=False, default=str)
|
||||
@@ -0,0 +1,300 @@
|
||||
"""The WebSocket endpoint — one socket per browser, every live feed and every
|
||||
RPC on it.
|
||||
|
||||
``GET /api/ws?token=<jwt>``
|
||||
|
||||
The token travels as a query parameter because the browser WebSocket API cannot
|
||||
set an ``Authorization`` header. That is a deliberate, contained exception to the
|
||||
header-only rule: the token is short-lived, the socket is TLS-terminated in any
|
||||
real deployment, and the alternative (a cookie) would drag CSRF into a protocol
|
||||
that has no CSRF story.
|
||||
|
||||
Protocol — small, symmetric, JSON in both directions. Server->client frames
|
||||
share the ``type`` discriminator; client->server frames use ``action``::
|
||||
|
||||
client -> {"action": "subscribe", "channels": ["org:…"]}
|
||||
client -> {"action": "unsubscribe", "channels": ["task:…"]}
|
||||
client -> {"action": "ping"}
|
||||
client -> {"action": "rpc", "id": "…", "method": "POST",
|
||||
"path": "/api/organizations", "body": {"name": "…"}}
|
||||
server -> {"type": "hello", "user_id": "…"|null}
|
||||
server -> {"type": "subscribed", "accepted": [...], "rejected": [...]}
|
||||
server -> {"type": "event", "event": {…envelope…}}
|
||||
server -> {"type": "rpc_result", "id": "…", "status": 200, "body": …}
|
||||
server -> {"type": "pong"}
|
||||
server -> {"type": "error", "message": "…"}
|
||||
|
||||
A connection may be **anonymous** (no token): that is the bootstrap state that
|
||||
lets a brand-new visitor call ``/api/auth/login`` and ``/api/auth/signup`` over
|
||||
RPC. Every other RPC and every ``subscribe`` requires an authenticated
|
||||
connection; the token presented at connect time is re-injected as a bearer
|
||||
header when the RPC is dispatched into the internal app, so the handlers' own
|
||||
auth dependencies still run unchanged.
|
||||
|
||||
Authorization queries open their own short-lived session: a socket can stay open
|
||||
for hours, and one SQLAlchemy session per open tab would quietly exhaust the
|
||||
connection pool.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Query, WebSocket, WebSocketDisconnect
|
||||
|
||||
from app.application.ports.realtime import dedupe_channels
|
||||
from app.core.database import SessionFactory
|
||||
from app.core.errors import AuthenticationError
|
||||
from app.core.security import decode_access_token
|
||||
from app.infrastructure.repositories.identity import UserRepository
|
||||
from app.interfaces.realtime.authz import authorize_channel
|
||||
from app.interfaces.realtime.manager import Connection, ConnectionManager
|
||||
|
||||
logger = logging.getLogger("agy.realtime")
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
#: The manager instance this process fans out with. Created at import: the
|
||||
#: endpoint and the lifecycle hooks can share it without a wiring ceremony.
|
||||
_manager = ConnectionManager()
|
||||
|
||||
|
||||
def realtime_manager() -> ConnectionManager:
|
||||
"""The process-wide manager — attached to the bus by ``app.main``."""
|
||||
return _manager
|
||||
|
||||
|
||||
#: Paths an anonymous (not-yet-logged-in) connection is allowed to call. These are
|
||||
#: the only two calls a visitor can make; everything else needs a session.
|
||||
_ANONYMOUS_RPC_PATHS = frozenset({"/api/auth/login", "/api/auth/signup"})
|
||||
|
||||
#: One in-process HTTP client that dispatches RPCs into the internal app. Built
|
||||
#: lazily so importing this module never pulls the app into existence.
|
||||
_rpc_client: httpx.AsyncClient | None = None
|
||||
|
||||
|
||||
def _get_rpc_client() -> httpx.AsyncClient:
|
||||
global _rpc_client
|
||||
if _rpc_client is None:
|
||||
# The internal app holds every HTTP handler; it is never socket-bound, so
|
||||
# it is reached here via an in-process ASGI transport instead of the wire.
|
||||
from app.interfaces.realtime.rpc_app import get_rpc_app
|
||||
|
||||
transport = httpx.ASGITransport(app=get_rpc_app())
|
||||
_rpc_client = httpx.AsyncClient(transport=transport, base_url="http://internal")
|
||||
return _rpc_client
|
||||
|
||||
|
||||
@router.websocket("/ws")
|
||||
async def realtime_socket(
|
||||
websocket: WebSocket,
|
||||
token: str | None = Query(default=None),
|
||||
) -> None:
|
||||
connection = await _authenticate(websocket, token)
|
||||
if connection is None:
|
||||
return # _authenticate already closed with a policy-violation code
|
||||
|
||||
await _manager.register(connection)
|
||||
|
||||
_send_frame(
|
||||
connection,
|
||||
{"type": "hello", "user_id": connection.user_id, "server_time": _now_iso()},
|
||||
)
|
||||
|
||||
try:
|
||||
while True:
|
||||
raw = await websocket.receive_text()
|
||||
await _handle_message(connection, raw)
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
except Exception: # noqa: BLE001 - any read failure ends the socket cleanly
|
||||
logger.debug("realtime: socket loop ended", exc_info=True)
|
||||
finally:
|
||||
await _manager.unregister(connection)
|
||||
|
||||
|
||||
async def _authenticate(websocket: WebSocket, token: str | None) -> Connection | None:
|
||||
"""Accept an anonymous or an authenticated, active user.
|
||||
|
||||
* No token -> an anonymous connection (bootstrap for login/signup).
|
||||
* A token that decodes to a real, active user -> an authenticated connection
|
||||
whose raw token is kept for RPC re-injection.
|
||||
* A presented-but-invalid token -> 1008, so the client can tell "bad token"
|
||||
from "network dropped" and stop retrying.
|
||||
"""
|
||||
user = None
|
||||
raw_token: str | None = None
|
||||
if token:
|
||||
try:
|
||||
payload = decode_access_token(token)
|
||||
user_id = payload.get("sub")
|
||||
if user_id:
|
||||
# Own session: the socket must not depend on request scope.
|
||||
async with SessionFactory() as session:
|
||||
user = await UserRepository(session).get(str(user_id))
|
||||
if user is not None and user.is_active:
|
||||
raw_token = token
|
||||
except AuthenticationError:
|
||||
user = None
|
||||
|
||||
if token and user is None:
|
||||
await websocket.close(code=1008, reason="unauthorized")
|
||||
return None
|
||||
|
||||
await websocket.accept()
|
||||
return Connection(
|
||||
user_id=user.id if user else None,
|
||||
is_platform_owner=user.is_platform_owner if user else False,
|
||||
token=raw_token,
|
||||
websocket=websocket,
|
||||
)
|
||||
|
||||
|
||||
async def _handle_message(connection: Connection, raw: str) -> None:
|
||||
try:
|
||||
message = json.loads(raw)
|
||||
except (TypeError, ValueError):
|
||||
_send_frame(connection, {"type": "error", "message": "malformed JSON frame"})
|
||||
return
|
||||
if not isinstance(message, dict):
|
||||
_send_frame(connection, {"type": "error", "message": "frame must be an object"})
|
||||
return
|
||||
|
||||
action = message.get("action")
|
||||
|
||||
if action == "ping":
|
||||
_send_frame(connection, {"type": "pong"})
|
||||
return
|
||||
|
||||
if action == "rpc":
|
||||
# Run off the receive loop: a slow handler must not stall pings.
|
||||
asyncio.create_task(_handle_rpc(connection, message))
|
||||
return
|
||||
|
||||
# subscribe / unsubscribe require a logged-in user.
|
||||
if connection.user_id is None:
|
||||
_send_frame(connection, {"type": "error", "message": "authentication required"})
|
||||
return
|
||||
|
||||
channels = dedupe_channels(
|
||||
str(channel) for channel in (message.get("channels") or []) if isinstance(channel, str)
|
||||
)
|
||||
|
||||
if action == "subscribe":
|
||||
accepted, rejected = await _subscribe(connection, channels)
|
||||
_send_frame(
|
||||
connection, {"type": "subscribed", "accepted": accepted, "rejected": rejected}
|
||||
)
|
||||
return
|
||||
|
||||
if action == "unsubscribe":
|
||||
await _manager.unsubscribe(connection, list(channels))
|
||||
_send_frame(connection, {"type": "unsubscribed", "channels": channels})
|
||||
return
|
||||
|
||||
_send_frame(connection, {"type": "error", "message": f"unknown action: {action!r}"})
|
||||
|
||||
|
||||
async def _handle_rpc(connection: Connection, message: dict) -> None:
|
||||
"""Dispatch one client RPC into the internal app and return the result."""
|
||||
req_id = message.get("id")
|
||||
if not req_id:
|
||||
_send_frame(connection, {"type": "error", "message": "rpc frame needs an id"})
|
||||
return
|
||||
|
||||
method = str(message.get("method") or "GET").upper()
|
||||
path = message.get("path")
|
||||
if not isinstance(path, str) or not path:
|
||||
_send_rpc_result(
|
||||
connection,
|
||||
req_id,
|
||||
400,
|
||||
{"error": {"code": "bad_request", "message": "rpc frame needs a path"}},
|
||||
)
|
||||
return
|
||||
|
||||
# Gate unauthenticated callers to login/signup only.
|
||||
if connection.user_id is None and path not in _ANONYMOUS_RPC_PATHS:
|
||||
_send_rpc_result(
|
||||
connection,
|
||||
req_id,
|
||||
401,
|
||||
{"error": {"code": "unauthenticated", "message": "نیاز به ورود دارید."}},
|
||||
)
|
||||
return
|
||||
|
||||
body = message.get("body")
|
||||
headers = {"X-Request-ID": str(req_id), "User-Agent": "realtime-rpc"}
|
||||
if connection.token:
|
||||
headers["Authorization"] = f"Bearer {connection.token}"
|
||||
|
||||
try:
|
||||
client = _get_rpc_client()
|
||||
if isinstance(body, (dict, list)):
|
||||
response = await client.request(method, path, json=body, headers=headers)
|
||||
elif body is None:
|
||||
response = await client.request(method, path, headers=headers)
|
||||
else:
|
||||
response = await client.request(method, path, content=str(body), headers=headers)
|
||||
except Exception: # noqa: BLE001 - reported to the caller as a 500 envelope
|
||||
logger.exception("realtime: rpc dispatch to internal app failed")
|
||||
_send_rpc_result(
|
||||
connection,
|
||||
req_id,
|
||||
500,
|
||||
{"error": {"code": "internal_error", "message": "خطای داخلی سرور."}},
|
||||
)
|
||||
return
|
||||
|
||||
status = response.status_code
|
||||
try:
|
||||
payload = response.json()
|
||||
except Exception:
|
||||
payload = None
|
||||
_send_rpc_result(connection, req_id, status, payload)
|
||||
|
||||
|
||||
def _send_rpc_result(
|
||||
connection: Connection, req_id: object, status: int, body: object
|
||||
) -> None:
|
||||
_send_frame(
|
||||
connection,
|
||||
{"type": "rpc_result", "id": req_id, "status": status, "body": body},
|
||||
)
|
||||
|
||||
|
||||
async def _subscribe(
|
||||
connection: Connection, channels: tuple[str, ...]
|
||||
) -> tuple[list[str], list[str]]:
|
||||
if not channels:
|
||||
return [], []
|
||||
|
||||
async def _authorize(conn: Connection, channel: str) -> bool:
|
||||
async with SessionFactory() as session:
|
||||
return await authorize_channel(
|
||||
session,
|
||||
user_id=conn.user_id or "",
|
||||
is_platform_owner=conn.is_platform_owner,
|
||||
channel=channel,
|
||||
)
|
||||
|
||||
return await _manager.subscribe(connection, list(channels), _authorize)
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
from app.domain.base import utcnow
|
||||
|
||||
return utcnow().isoformat()
|
||||
|
||||
|
||||
def _send_frame(connection: Connection, payload: dict) -> None:
|
||||
"""Queue a frame on the connection's single-writer outbox.
|
||||
|
||||
Every frame — control, event and rpc_result — goes through the same queue so
|
||||
a slow RPC task can never interleave bytes with the receive loop's own
|
||||
frames and corrupt the stream.
|
||||
"""
|
||||
connection.offer(json.dumps(payload, ensure_ascii=False, default=str))
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Internal, never-socket-bound FastAPI app that holds every HTTP handler.
|
||||
|
||||
The public app (``app.main.create_app``) exposes only ``/health`` and the
|
||||
``/api/ws`` WebSocket. All request/response logic lives here and is reached in
|
||||
exactly one of two ways:
|
||||
|
||||
* from the WS-RPC proxy (``app.interfaces.realtime.router``), which dispatches
|
||||
client RPCs into this app over an in-process ASGI transport, and
|
||||
* from the test-suite's ``AsyncClient``/``TestClient``.
|
||||
|
||||
It is intentionally never bound to a socket, so including the routers here adds no
|
||||
public REST surface — the only public routes are the ones ``create_app``
|
||||
registers. Keeping the handlers in one FastAPI app also means they run through the
|
||||
real middleware, dependencies, validation and error handling, unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from app.core.config import settings
|
||||
from app.interfaces.http.errors import register_exception_handlers
|
||||
from app.interfaces.http.request_context import RequestContextMiddleware
|
||||
from app.interfaces.http.routers import (
|
||||
admin,
|
||||
agy,
|
||||
auth,
|
||||
billing,
|
||||
health,
|
||||
invites,
|
||||
knowledge,
|
||||
organizations,
|
||||
projects,
|
||||
sandbox,
|
||||
workforce,
|
||||
)
|
||||
from app.interfaces.realtime.router import router as realtime_router
|
||||
from app.main import lifespan # shared lifespan installs the realtime bus
|
||||
|
||||
#: Module-level singleton so the RPC proxy and the tests share one app instance.
|
||||
_rpc_app: FastAPI | None = None
|
||||
|
||||
|
||||
def build_rpc_app() -> FastAPI:
|
||||
"""Construct the full handler app. Called once per process."""
|
||||
application = FastAPI(
|
||||
title=settings.app_name,
|
||||
version="0.1.0",
|
||||
description="AI Workforce Platform — internal RPC dispatch surface.",
|
||||
lifespan=lifespan,
|
||||
# This app is reached in-process, never over the network, so the
|
||||
# interactive docs would only be a liability.
|
||||
docs_url=None,
|
||||
openapi_url=None,
|
||||
)
|
||||
application.add_middleware(RequestContextMiddleware)
|
||||
register_exception_handlers(application)
|
||||
|
||||
# Same mounting as the old public app: health at the root, everything else
|
||||
# under /api, and the live socket alongside the RPC surface.
|
||||
application.include_router(health.router)
|
||||
application.include_router(auth.router, prefix="/api")
|
||||
application.include_router(invites.router, prefix="/api")
|
||||
application.include_router(organizations.router, prefix="/api")
|
||||
application.include_router(projects.router, prefix="/api")
|
||||
application.include_router(workforce.router, prefix="/api")
|
||||
application.include_router(billing.router, prefix="/api")
|
||||
application.include_router(knowledge.router, prefix="/api")
|
||||
application.include_router(sandbox.router, prefix="/api")
|
||||
application.include_router(admin.router, prefix="/api")
|
||||
application.include_router(agy.router, prefix="/api")
|
||||
application.include_router(realtime_router, prefix="/api")
|
||||
return application
|
||||
|
||||
|
||||
def get_rpc_app() -> FastAPI:
|
||||
"""Return the process-wide internal app, building it on first use."""
|
||||
global _rpc_app
|
||||
if _rpc_app is None:
|
||||
_rpc_app = build_rpc_app()
|
||||
return _rpc_app
|
||||
+36
-26
@@ -15,19 +15,11 @@ from app.core.config import settings
|
||||
from app.core.database import Base, engine
|
||||
from app.interfaces.http.errors import register_exception_handlers
|
||||
from app.interfaces.http.request_context import RequestContextMiddleware
|
||||
from app.interfaces.http.routers import (
|
||||
admin,
|
||||
agy,
|
||||
auth,
|
||||
billing,
|
||||
health,
|
||||
invites,
|
||||
knowledge,
|
||||
organizations,
|
||||
projects,
|
||||
sandbox,
|
||||
workforce,
|
||||
)
|
||||
from app.interfaces.http.routers import health
|
||||
from app.interfaces.realtime.bus import install as install_realtime_bus
|
||||
from app.interfaces.realtime.bus import start_relay, stop_relay
|
||||
from app.interfaces.realtime.router import realtime_manager
|
||||
from app.interfaces.realtime.router import router as realtime_router
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
@@ -63,8 +55,19 @@ async def lifespan(_: FastAPI):
|
||||
# loudly — a half-started API must not accept traffic.
|
||||
await _record_startup_failure(exc)
|
||||
raise
|
||||
|
||||
# --- realtime wiring ---------------------------------------------------
|
||||
# This process is the publisher for API-driven events and the relay for
|
||||
# anything the standalone workers publish. Both are best-effort: if Redis
|
||||
# is not reachable the in-process fan-out keeps every HTTP-driven feed live.
|
||||
bus = install_realtime_bus()
|
||||
bus.attach(realtime_manager().dispatch)
|
||||
await start_relay()
|
||||
|
||||
yield
|
||||
|
||||
await stop_relay()
|
||||
|
||||
|
||||
async def _record_startup_failure(exc: BaseException) -> None:
|
||||
"""Persist a boot failure. Never raises; the caller re-raises regardless."""
|
||||
@@ -85,13 +88,22 @@ async def _record_startup_failure(exc: BaseException) -> None:
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
"""Build the *public* app.
|
||||
|
||||
This is the only app uvicorn binds. It deliberately exposes no REST API: the
|
||||
application surface is reachable only over the ``/api/ws`` WebSocket (live
|
||||
feeds + RPC). ``/health`` stays as an HTTP liveness probe because deploy
|
||||
orchestration and the browser's status check both need a plain GET, and a
|
||||
probe is not an application endpoint.
|
||||
"""
|
||||
application = FastAPI(
|
||||
title=settings.app_name,
|
||||
version="0.1.0",
|
||||
description="AI Workforce Platform — standalone multi-tenant SaaS.",
|
||||
lifespan=lifespan,
|
||||
docs_url="/docs",
|
||||
openapi_url="/openapi.json",
|
||||
# No REST API on the public surface, so the interactive docs are off.
|
||||
docs_url=None,
|
||||
openapi_url=None,
|
||||
)
|
||||
|
||||
# Starlette applies middleware outermost-first in reverse registration
|
||||
@@ -114,25 +126,23 @@ def create_app() -> FastAPI:
|
||||
|
||||
register_exception_handlers(application)
|
||||
|
||||
# The only public routes: a liveness probe and the single WebSocket that
|
||||
# carries every application call (live events + RPC). All HTTP handlers live
|
||||
# in the internal app reached through that socket.
|
||||
application.include_router(health.router)
|
||||
application.include_router(auth.router, prefix="/api")
|
||||
application.include_router(invites.router, prefix="/api")
|
||||
application.include_router(organizations.router, prefix="/api")
|
||||
application.include_router(projects.router, prefix="/api")
|
||||
application.include_router(workforce.router, prefix="/api")
|
||||
application.include_router(billing.router, prefix="/api")
|
||||
application.include_router(knowledge.router, prefix="/api")
|
||||
application.include_router(sandbox.router, prefix="/api")
|
||||
application.include_router(admin.router, prefix="/api")
|
||||
application.include_router(agy.router, prefix="/api")
|
||||
application.include_router(realtime_router, prefix="/api")
|
||||
|
||||
@application.get("/", tags=["meta"])
|
||||
async def root() -> dict:
|
||||
return {
|
||||
"name": settings.app_name,
|
||||
"version": "0.1.0",
|
||||
"docs": "/docs",
|
||||
"health": "/health",
|
||||
"websocket": "/api/ws",
|
||||
"note": (
|
||||
"All application calls travel over the WebSocket at /api/ws "
|
||||
"(live feeds + RPC). There is no REST API on this server."
|
||||
),
|
||||
}
|
||||
|
||||
return application
|
||||
|
||||
@@ -125,6 +125,11 @@ def main(argv: list[str] | None = None) -> int:
|
||||
format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
|
||||
)
|
||||
args = _build_parser().parse_args(argv)
|
||||
# A worker is its own process: publishing here is how a run it starts, or a
|
||||
# session it reaps, reaches the browsers connected to the API process.
|
||||
from app.interfaces.realtime.bus import install as install_realtime
|
||||
|
||||
install_realtime()
|
||||
return asyncio.run(_main_async(args))
|
||||
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ the service's business.
|
||||
from __future__ import annotations
|
||||
|
||||
from app.application.container import build_workforce_service
|
||||
from app.application.ports.realtime import emit_worker_status
|
||||
from app.core.errors import AppError
|
||||
from app.domain.audit.models import ErrorSeverity, ErrorSource
|
||||
from app.domain.workforce.models import TaskStatus
|
||||
@@ -87,6 +88,13 @@ class AiRuntimeWorker:
|
||||
|
||||
outcome.processed += 1
|
||||
outcome.note(f"task {task.id} -> run {run.id} ({run.status})")
|
||||
# This worker is its own process: without a push here, a run
|
||||
# started by the queue is invisible until someone reloads.
|
||||
await emit_worker_status(
|
||||
organization_id=task.organization_id,
|
||||
worker=self.name,
|
||||
message=f"کار «{current.title}» در حال اجراست (اجرا {run.id}).",
|
||||
)
|
||||
|
||||
return outcome
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ from __future__ import annotations
|
||||
from datetime import timedelta
|
||||
|
||||
from app.application.container import build_sandbox_service
|
||||
from app.application.ports.realtime import emit_sandbox_reaped
|
||||
from app.core.config import settings
|
||||
from app.domain.audit.models import AuditAction, AuditEvent
|
||||
from app.domain.base import utcnow
|
||||
@@ -72,6 +73,13 @@ class SandboxGatewayWorker:
|
||||
)
|
||||
outcome.processed += 1
|
||||
outcome.note(f"stopped session {sandbox_session.id}")
|
||||
# A reaped sandbox is a tenant-visible fact: push it so the
|
||||
# project page's terminal stops showing a live session.
|
||||
await emit_sandbox_reaped(
|
||||
organization_id=workspace.organization_id if workspace else None,
|
||||
session_id=sandbox_session.id,
|
||||
workspace_id=sandbox_session.workspace_id,
|
||||
)
|
||||
|
||||
if not stopped:
|
||||
outcome.skipped += 1
|
||||
|
||||
Reference in New Issue
Block a user