feat(delivery): docker sandbox adapter + owner-only infrastructure (v2 phase A3-1)
The execution boundary for v2. Adds the local docker/podman sandbox adapter and the hardened host subprocess runner every other delivery adapter will use. - adapters/process.py: the one place the platform shells out. argv-only (no shell), start_new_session + killpg on timeout, concurrent stdout/stderr drains with a byte cap, explicit minimal env (never os.environ), and a missing binary reported as spawn_failed rather than an exception. - adapters/docker_sandbox.py: DockerSandboxAdapter over SandboxProviderPort. --user 1000:1000, --cap-drop ALL, no-new-privileges, pids/cpu/memory limits, --read-only with tmpfs scratch, one bind mount (the work tree, never the git dir), labels, no published ports. Path-safe file access; secrets refused in `docker exec -e`. Extras for later phases: start_demo_container (loopback publish only), list_managed_containers, remove_container. - registry: SANDBOX_PROVIDER=docker, cached singleton so the workspace registry survives create_workspace -> start_session. - sandbox_gateway worker: reap_orphans() removes labelled containers no live session points at, audited as sandbox.reap; run_pass() does both passes. - sandbox router: every mutating route now requires the platform owner (PLAN-v2 §7.7); read-only status routes stay open to org members. - sandbox-panel: canManageInfra prop (fails closed) hides the create form, the session panel and the action buttons from non-owners. Tests: 346 API (+37), 67 web (+3); ruff, mypy, tsc and lint clean. Three existing tests now drive the sandbox as the platform owner, since that is who may touch it. Report: docs/plan/reports/agent-3-phase-1.md
This commit is contained in:
mjalmoudy94
@@ -0,0 +1,641 @@
|
||||
"""Local Docker/podman sandbox adapter — the hardened execution boundary (D5).
|
||||
|
||||
One *workspace* container per application is where the AI employees edit files
|
||||
and run commands; a separate *demo* container (created by the delivery service)
|
||||
runs the latest green build. Both are created by this adapter.
|
||||
|
||||
The hardening is not decoration — it is the only thing standing between an LLM
|
||||
that writes arbitrary shell and the host:
|
||||
|
||||
* non-root (``--user 1000:1000``) and ``--cap-drop ALL`` + ``no-new-privileges``
|
||||
so a setuid binary cannot be used to escalate;
|
||||
* ``--read-only`` root filesystem with two ``tmpfs`` scratch mounts, so a
|
||||
compromised process cannot persist anything outside the bind-mounted work dir;
|
||||
* ``--pids-limit`` / ``--cpus`` / ``--memory`` so a fork bomb or a runaway build
|
||||
cannot take the host down;
|
||||
* the **git dir is never mounted** — only the work tree. The platform runs git
|
||||
on the host (see ``git_workspace.py``), so repository credentials never enter
|
||||
the container;
|
||||
* ports are published on ``127.0.0.1`` only, never ``0.0.0.0``.
|
||||
|
||||
The provider binary is configurable (``SANDBOX_DOCKER_BIN``) and the flags used
|
||||
here are the common subset of ``docker`` and ``podman``, so podman works as a
|
||||
drop-in.
|
||||
|
||||
``create_workspace``/``start_session`` are split by the port, but the workspace
|
||||
identity is *encoded in the external id* (``ws-<org>-<project>``) so a restart
|
||||
can rebuild the host paths without a database round-trip. Only the image is
|
||||
kept in a process-local registry; a cold registry falls back to the profile
|
||||
default for the language, which is what a fresh process would want anyway.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
from collections.abc import Awaitable, Callable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from app.application.ports import (
|
||||
CommandResult,
|
||||
SandboxProviderPort, # noqa: F401 - documented as the implemented protocol
|
||||
SessionInfo,
|
||||
SnapshotInfo,
|
||||
WorkspaceInfo,
|
||||
)
|
||||
from app.core.config import settings
|
||||
from app.core.errors import ValidationError
|
||||
from app.domain.platform.rules import assert_not_protected_port
|
||||
from app.infrastructure.adapters.process import ProcessResult, run_process
|
||||
|
||||
logger = logging.getLogger("agy.sandbox.docker")
|
||||
|
||||
#: Language → base image. The plan fixes these names; the mapping lives here so
|
||||
#: the delivery service can ask for an image without knowing the table.
|
||||
LANGUAGE_IMAGES: dict[str, str] = {
|
||||
"node": "node:20-bookworm",
|
||||
"javascript": "node:20-bookworm",
|
||||
"typescript": "node:20-bookworm",
|
||||
"python": "python:3.13-bookworm",
|
||||
"php": "php:8.3-cli",
|
||||
"go": "golang:1.23",
|
||||
"golang": "golang:1.23",
|
||||
"static": "debian:bookworm",
|
||||
"html": "debian:bookworm",
|
||||
}
|
||||
FALLBACK_IMAGE = "debian:bookworm"
|
||||
|
||||
#: Role → label value. The reaper and the admin console filter on these.
|
||||
ROLE_WORKSPACE = "workspace"
|
||||
ROLE_DEMO = "demo"
|
||||
ROLE_BUILD = "build"
|
||||
|
||||
LABEL_ORG = "smart-org.org"
|
||||
LABEL_PROJECT = "smart-org.project"
|
||||
LABEL_ROLE = "smart-org.role"
|
||||
|
||||
#: In-container mount point of the working tree.
|
||||
SANDBOX_WORKSPACE_PATH = "/workspace"
|
||||
|
||||
_SAFE_ID = re.compile(r"^[A-Za-z0-9_.-]+$")
|
||||
_SNAPSHOT_NAME = re.compile(r"[^a-z0-9-]+")
|
||||
|
||||
#: Command used to keep a container alive. ``sleep infinity`` is present in
|
||||
#: every image we use (coreutils in debian, busybox in alpine variants).
|
||||
_KEEPALIVE = "sleep infinity"
|
||||
|
||||
|
||||
class PathEscapeError(ValidationError):
|
||||
"""A requested path resolves outside the sandbox working directory."""
|
||||
|
||||
|
||||
def image_for_language(language: str | None) -> str:
|
||||
"""Base image for an app profile language, with a safe fallback."""
|
||||
return LANGUAGE_IMAGES.get((language or "").strip().lower(), FALLBACK_IMAGE)
|
||||
|
||||
|
||||
def container_name(role: str, organization_id: str, project_id: str) -> str:
|
||||
"""Deterministic container name so a restart can find its own containers."""
|
||||
safe = f"smartorg-{role}-{organization_id}-{project_id}".lower()
|
||||
return re.sub(r"[^a-z0-9_.-]", "-", safe)[:200]
|
||||
|
||||
|
||||
def _check_id(value: str, label: str) -> str:
|
||||
clean = (value or "").strip()
|
||||
if not clean or not _SAFE_ID.match(clean):
|
||||
raise ValidationError(f"شناسهٔ {label} نامعتبر است.")
|
||||
return clean
|
||||
|
||||
|
||||
def safe_join(root: str, path: str) -> str:
|
||||
"""Resolve ``path`` under ``root``, refusing any escape.
|
||||
|
||||
``..`` segments, absolute paths and symlinks that point outside the root are
|
||||
all rejected after ``realpath``, so a symlink created inside the sandbox
|
||||
cannot be used to read ``/etc/shadow`` on the host.
|
||||
"""
|
||||
root_real = os.path.realpath(root)
|
||||
relative = (path or "").lstrip("/")
|
||||
candidate = os.path.realpath(os.path.join(root_real, relative))
|
||||
if candidate != root_real and not candidate.startswith(root_real + os.sep):
|
||||
raise PathEscapeError("مسیر درخواستی خارج از محیط اجرا است.")
|
||||
return candidate
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ManagedContainer:
|
||||
"""One container carrying our labels, as reported by the CLI."""
|
||||
|
||||
container_id: str
|
||||
name: str = ""
|
||||
organization_id: str = ""
|
||||
project_id: str = ""
|
||||
role: str = ""
|
||||
status: str = ""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _WorkspaceState:
|
||||
organization_id: str
|
||||
project_id: str
|
||||
host_dir: str
|
||||
image: str
|
||||
session_ids: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
class DockerSandboxAdapter:
|
||||
"""``SandboxProviderPort`` over the docker/podman CLI.
|
||||
|
||||
``runner`` is injectable so unit tests assert on the exact argv without a
|
||||
container runtime present.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
provider_bin: str | None = None,
|
||||
workspaces_root: str | None = None,
|
||||
default_cpus: float | None = None,
|
||||
default_memory_mb: int | None = None,
|
||||
max_output_bytes: int | None = None,
|
||||
runner: Callable[..., Awaitable[ProcessResult]] = run_process,
|
||||
) -> None:
|
||||
self.bin = (provider_bin or settings.sandbox_docker_bin or "docker").strip()
|
||||
self.workspaces_root = os.path.abspath(
|
||||
workspaces_root or settings.sandbox_workspaces_root
|
||||
)
|
||||
self.default_cpus = float(
|
||||
settings.sandbox_default_cpus if default_cpus is None else default_cpus
|
||||
)
|
||||
self.default_memory_mb = int(
|
||||
settings.sandbox_default_memory_mb if default_memory_mb is None else default_memory_mb
|
||||
)
|
||||
self.max_output_bytes = int(
|
||||
settings.sandbox_max_output_bytes if max_output_bytes is None else max_output_bytes
|
||||
)
|
||||
self._runner = runner
|
||||
self._workspaces: dict[str, _WorkspaceState] = {}
|
||||
#: container id → the workspace state it belongs to (filled by start_session).
|
||||
self._sessions: dict[str, _WorkspaceState] = {}
|
||||
|
||||
# -- helpers -------------------------------------------------------------
|
||||
|
||||
def _workspace_dir(self, organization_id: str, project_id: str) -> str:
|
||||
return os.path.join(self.workspaces_root, organization_id, project_id, "work")
|
||||
|
||||
def _external_id(self, organization_id: str, project_id: str) -> str:
|
||||
return f"ws-{organization_id}-{project_id}"
|
||||
|
||||
@staticmethod
|
||||
def _parse_external_id(external_id: str) -> tuple[str, str] | None:
|
||||
if not external_id.startswith("ws-"):
|
||||
return None
|
||||
parts = external_id[3:].split("-", 1)
|
||||
if len(parts) != 2:
|
||||
return None
|
||||
return parts[0], parts[1]
|
||||
|
||||
def _state_for(self, external_id: str, image: str | None = None) -> _WorkspaceState:
|
||||
"""Look up (or rebuild) the workspace state behind an external id."""
|
||||
state = self._workspaces.get(external_id)
|
||||
if state is not None:
|
||||
return state
|
||||
parsed = self._parse_external_id(external_id)
|
||||
if parsed is None:
|
||||
raise ValidationError("شناسهٔ محیط اجرا نامعتبر است.")
|
||||
organization_id, project_id = parsed
|
||||
state = _WorkspaceState(
|
||||
organization_id=organization_id,
|
||||
project_id=project_id,
|
||||
host_dir=self._workspace_dir(organization_id, project_id),
|
||||
image=image or FALLBACK_IMAGE,
|
||||
)
|
||||
self._workspaces[external_id] = state
|
||||
return state
|
||||
|
||||
def _state_for_session(self, session_external_id: str) -> _WorkspaceState | None:
|
||||
return self._sessions.get(session_external_id)
|
||||
|
||||
async def _docker(
|
||||
self,
|
||||
args: Sequence[str],
|
||||
*,
|
||||
timeout_seconds: float = 120.0,
|
||||
on_output: Callable[[str], Awaitable[None]] | None = None,
|
||||
) -> ProcessResult:
|
||||
return await self._runner(
|
||||
[self.bin, *args],
|
||||
timeout_seconds=timeout_seconds,
|
||||
max_output_bytes=self.max_output_bytes,
|
||||
on_output=on_output,
|
||||
env={},
|
||||
)
|
||||
|
||||
# -- workspaces ----------------------------------------------------------
|
||||
|
||||
async def create_workspace(
|
||||
self, *, organization_id: str, project_id: str, template_id: str, image: str
|
||||
) -> WorkspaceInfo:
|
||||
org = _check_id(organization_id, "سازمان")
|
||||
project = _check_id(project_id, "پروژه")
|
||||
|
||||
host_dir = self._workspace_dir(org, project)
|
||||
# 0700: the workspace holds the tenant's source code. The platform user
|
||||
# owns it; nothing else on the host may read it.
|
||||
os.makedirs(host_dir, exist_ok=True)
|
||||
with contextlib.suppress(OSError):
|
||||
os.chmod(self.workspaces_root, 0o700)
|
||||
os.chmod(os.path.dirname(host_dir), 0o700)
|
||||
os.chmod(host_dir, 0o700)
|
||||
|
||||
external_id = self._external_id(org, project)
|
||||
self._workspaces[external_id] = _WorkspaceState(
|
||||
organization_id=org,
|
||||
project_id=project,
|
||||
host_dir=host_dir,
|
||||
image=image or FALLBACK_IMAGE,
|
||||
)
|
||||
return WorkspaceInfo(
|
||||
external_id=external_id,
|
||||
status="ready",
|
||||
raw={"host_dir": host_dir, "image": image},
|
||||
)
|
||||
|
||||
async def start_session(self, *, workspace_external_id: str) -> SessionInfo:
|
||||
state = self._state_for(workspace_external_id)
|
||||
name = container_name(ROLE_WORKSPACE, state.organization_id, state.project_id)
|
||||
|
||||
# A restart leaves the old container behind; reuse it rather than
|
||||
# stacking duplicates, and start it if it was stopped.
|
||||
existing = await self._container_id_by_name(name)
|
||||
if existing:
|
||||
await self._docker(["start", existing], timeout_seconds=60)
|
||||
self._sessions[existing] = state
|
||||
state.session_ids.append(existing)
|
||||
return SessionInfo(external_id=existing, status="running")
|
||||
|
||||
result = await self._docker(
|
||||
[
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
name,
|
||||
# --- isolation -------------------------------------------------
|
||||
"--user",
|
||||
"1000:1000",
|
||||
"--cap-drop",
|
||||
"ALL",
|
||||
"--security-opt",
|
||||
"no-new-privileges",
|
||||
"--pids-limit",
|
||||
"512",
|
||||
"--cpus",
|
||||
str(self.default_cpus),
|
||||
"--memory",
|
||||
f"{self.default_memory_mb}m",
|
||||
# --- immutable root, scratch in tmpfs --------------------------
|
||||
"--read-only",
|
||||
"--tmpfs",
|
||||
"/tmp",
|
||||
"--tmpfs",
|
||||
"/home/app",
|
||||
# --- only the work tree, never the git dir ---------------------
|
||||
"-v",
|
||||
f"{state.host_dir}:{SANDBOX_WORKSPACE_PATH}",
|
||||
"-w",
|
||||
SANDBOX_WORKSPACE_PATH,
|
||||
"--label",
|
||||
f"{LABEL_ORG}={state.organization_id}",
|
||||
"--label",
|
||||
f"{LABEL_PROJECT}={state.project_id}",
|
||||
"--label",
|
||||
f"{LABEL_ROLE}={ROLE_WORKSPACE}",
|
||||
# Default bridge network: package installs need egress. No
|
||||
# published ports — the container is reached with ``exec``.
|
||||
state.image,
|
||||
"sh",
|
||||
"-lc",
|
||||
_KEEPALIVE,
|
||||
],
|
||||
timeout_seconds=180,
|
||||
)
|
||||
if not result.ok:
|
||||
raise ValidationError("راهاندازی محیط اجرا ناموفق بود.")
|
||||
|
||||
container_id = result.stdout.strip().splitlines()[-1].strip() if result.stdout else ""
|
||||
if not container_id:
|
||||
raise ValidationError("راهاندازی محیط اجرا ناموفق بود.")
|
||||
self._sessions[container_id] = state
|
||||
state.session_ids.append(container_id)
|
||||
return SessionInfo(
|
||||
external_id=container_id,
|
||||
status="running",
|
||||
raw={"name": name, "image": state.image},
|
||||
)
|
||||
|
||||
async def _container_id_by_name(self, name: str) -> str | None:
|
||||
result = await self._docker(
|
||||
["ps", "-a", "--filter", f"name=^{name}$", "--format", "{{.ID}}"],
|
||||
timeout_seconds=30,
|
||||
)
|
||||
if not result.ok or not result.stdout.strip():
|
||||
return None
|
||||
return result.stdout.strip().splitlines()[0].strip()
|
||||
|
||||
# -- commands ------------------------------------------------------------
|
||||
|
||||
async def run_command(
|
||||
self,
|
||||
*,
|
||||
session_external_id: str,
|
||||
command: str,
|
||||
timeout_seconds: int,
|
||||
max_output_bytes: int,
|
||||
env: dict[str, str] | None = None,
|
||||
cwd: str | None = None,
|
||||
on_output: Callable[[str], Awaitable[None]] | None = None,
|
||||
) -> CommandResult:
|
||||
"""``docker exec`` the command inside the container.
|
||||
|
||||
``cwd``/``on_output`` are adapter-level extras used by the engine; the
|
||||
port signature is unchanged.
|
||||
|
||||
``env`` is passed with ``-e``. Keys that *look* like secrets are refused
|
||||
outright: argv is visible in ``ps`` and in the audit log, and the whole
|
||||
point of the design is that credentials never travel that way.
|
||||
"""
|
||||
cleaned = (command or "").strip()
|
||||
if not cleaned:
|
||||
raise ValidationError("دستور نمیتواند خالی باشد.")
|
||||
|
||||
env_flags: list[str] = []
|
||||
for key, value in (env or {}).items():
|
||||
if _looks_like_secret(key):
|
||||
raise ValidationError(
|
||||
"متغیرهای محیطی حاوی اطلاعات محرمانه مجاز نیستند."
|
||||
)
|
||||
env_flags.extend(["-e", f"{key}={value}"])
|
||||
|
||||
workdir = cwd or SANDBOX_WORKSPACE_PATH
|
||||
args = [
|
||||
"exec",
|
||||
"-w",
|
||||
workdir,
|
||||
*env_flags,
|
||||
session_external_id,
|
||||
"sh",
|
||||
"-lc",
|
||||
cleaned,
|
||||
]
|
||||
result = await self._runner(
|
||||
[self.bin, *args],
|
||||
timeout_seconds=float(timeout_seconds),
|
||||
max_output_bytes=int(max_output_bytes),
|
||||
on_output=on_output,
|
||||
env={},
|
||||
)
|
||||
return CommandResult(
|
||||
exit_code=result.exit_code,
|
||||
stdout=result.stdout,
|
||||
stderr=result.stderr,
|
||||
duration_ms=result.duration_ms,
|
||||
truncated=result.truncated,
|
||||
)
|
||||
|
||||
# -- files ---------------------------------------------------------------
|
||||
|
||||
def _host_path(self, session_external_id: str, path: str) -> str:
|
||||
state = self._state_for_session(session_external_id)
|
||||
if state is None:
|
||||
raise ValidationError("نشست اجرا یافت نشد.")
|
||||
# Container paths under /workspace map 1:1 onto the bind-mounted host
|
||||
# dir; anything else is not reachable through this adapter.
|
||||
relative = (path or "").lstrip("/")
|
||||
if relative.startswith("workspace"):
|
||||
relative = relative[len("workspace") :].lstrip("/")
|
||||
return safe_join(state.host_dir, relative)
|
||||
|
||||
async def read_file(self, *, session_external_id: str, path: str) -> str:
|
||||
host_path = self._host_path(session_external_id, path)
|
||||
if not os.path.isfile(host_path):
|
||||
return ""
|
||||
with open(host_path, encoding="utf-8", errors="replace") as handle:
|
||||
return handle.read()
|
||||
|
||||
async def write_file(self, *, session_external_id: str, path: str, content: str) -> None:
|
||||
host_path = self._host_path(session_external_id, path)
|
||||
os.makedirs(os.path.dirname(host_path) or ".", exist_ok=True)
|
||||
with open(host_path, "w", encoding="utf-8") as handle:
|
||||
handle.write(content)
|
||||
|
||||
# -- lifecycle -----------------------------------------------------------
|
||||
|
||||
async def snapshot(self, *, session_external_id: str, name: str) -> SnapshotInfo:
|
||||
"""``docker commit`` the container.
|
||||
|
||||
Supported, but documented as a heavy operation: it captures the whole
|
||||
root filesystem, and the work tree lives on a bind mount that is *not*
|
||||
included. Snapshots are therefore useful for the base image state only.
|
||||
"""
|
||||
state = self._state_for_session(session_external_id)
|
||||
slug = _SNAPSHOT_NAME.sub("-", (name or "snapshot").lower()).strip("-") or "snapshot"
|
||||
org = state.organization_id if state else "unknown"
|
||||
tag = f"smartorg-snapshot-{org}-{slug}"[:120]
|
||||
result = await self._docker(
|
||||
["commit", session_external_id, tag], timeout_seconds=300
|
||||
)
|
||||
if not result.ok:
|
||||
raise ValidationError("تهیهٔ اسنپشات ناموفق بود.")
|
||||
return SnapshotInfo(external_id=tag, name=name)
|
||||
|
||||
async def pause(self, *, session_external_id: str) -> None:
|
||||
await self._docker(["stop", session_external_id], timeout_seconds=60)
|
||||
|
||||
async def resume(self, *, session_external_id: str) -> SessionInfo:
|
||||
result = await self._docker(["start", session_external_id], timeout_seconds=60)
|
||||
return SessionInfo(
|
||||
external_id=session_external_id,
|
||||
status="running" if result.ok else "error",
|
||||
)
|
||||
|
||||
async def destroy(self, *, session_external_id: str) -> None:
|
||||
# ``-f`` covers running and stopped; ``-v`` removes anonymous volumes so
|
||||
# a destroyed workspace does not leave data behind on the host.
|
||||
await self._docker(["rm", "-f", "-v", session_external_id], timeout_seconds=60)
|
||||
self._sessions.pop(session_external_id, None)
|
||||
|
||||
async def get_preview_url(self, *, session_external_id: str, port: int) -> str:
|
||||
# Loopback only, and never an internal platform port. The public hostname
|
||||
# is a domain-layer concern (Phase 4), not the sandbox's.
|
||||
assert_not_protected_port(int(port))
|
||||
return f"http://127.0.0.1:{int(port)}"
|
||||
|
||||
# -- demo / build containers (used by the delivery service) --------------
|
||||
|
||||
async def start_demo_container(
|
||||
self,
|
||||
*,
|
||||
organization_id: str,
|
||||
project_id: str,
|
||||
image: str,
|
||||
host_port: int,
|
||||
app_port: int,
|
||||
command: str,
|
||||
work_dir: str,
|
||||
cpus: float | None = None,
|
||||
memory_mb: int | None = None,
|
||||
name_suffix: str = "",
|
||||
) -> str:
|
||||
"""Start a demo container and return its id.
|
||||
|
||||
Blue/green is implemented by the caller: start on a temporary port, wait
|
||||
for the healthcheck, switch the route, then stop the previous container.
|
||||
Publishing on ``127.0.0.1`` means only Caddy on this host can reach it.
|
||||
"""
|
||||
org = _check_id(organization_id, "سازمان")
|
||||
project = _check_id(project_id, "پروژه")
|
||||
assert_not_protected_port(int(host_port))
|
||||
name = container_name(ROLE_DEMO, org, project)
|
||||
if name_suffix:
|
||||
name = f"{name}-{_SNAPSHOT_NAME.sub('-', name_suffix.lower()).strip('-')}"[:200]
|
||||
|
||||
result = await self._docker(
|
||||
[
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
name,
|
||||
"--user",
|
||||
"1000:1000",
|
||||
"--cap-drop",
|
||||
"ALL",
|
||||
"--security-opt",
|
||||
"no-new-privileges",
|
||||
"--pids-limit",
|
||||
"512",
|
||||
"--cpus",
|
||||
str(cpus if cpus is not None else self.default_cpus),
|
||||
"--memory",
|
||||
f"{int(memory_mb if memory_mb is not None else self.default_memory_mb)}m",
|
||||
"--read-only",
|
||||
"--tmpfs",
|
||||
"/tmp",
|
||||
"--tmpfs",
|
||||
"/home/app",
|
||||
"-v",
|
||||
f"{work_dir}:{SANDBOX_WORKSPACE_PATH}",
|
||||
"-w",
|
||||
SANDBOX_WORKSPACE_PATH,
|
||||
# Loopback publish only — never 0.0.0.0.
|
||||
"-p",
|
||||
f"127.0.0.1:{int(host_port)}:{int(app_port)}",
|
||||
"--label",
|
||||
f"{LABEL_ORG}={org}",
|
||||
"--label",
|
||||
f"{LABEL_PROJECT}={project}",
|
||||
"--label",
|
||||
f"{LABEL_ROLE}={ROLE_DEMO}",
|
||||
image,
|
||||
"sh",
|
||||
"-lc",
|
||||
command or _KEEPALIVE,
|
||||
],
|
||||
timeout_seconds=180,
|
||||
)
|
||||
if not result.ok or not result.stdout.strip():
|
||||
raise ValidationError("راهاندازی نسخهٔ نمایشی ناموفق بود.")
|
||||
return result.stdout.strip().splitlines()[-1].strip()
|
||||
|
||||
# -- maintenance ---------------------------------------------------------
|
||||
|
||||
async def list_managed_containers(self) -> list[ManagedContainer]:
|
||||
"""Every container carrying our labels, running or not.
|
||||
|
||||
Used by the sandbox-gateway worker to find orphans: a container whose
|
||||
session row is gone is still holding CPU and disk.
|
||||
"""
|
||||
result = await self._docker(
|
||||
[
|
||||
"ps",
|
||||
"-a",
|
||||
"--filter",
|
||||
f"label={LABEL_ROLE}",
|
||||
"--format",
|
||||
"{{.ID}}\t{{.Names}}\t{{.Status}}\t{{.Labels}}",
|
||||
],
|
||||
timeout_seconds=30,
|
||||
)
|
||||
if not result.ok:
|
||||
return []
|
||||
|
||||
containers: list[ManagedContainer] = []
|
||||
for line in result.stdout.splitlines():
|
||||
parts = line.split("\t")
|
||||
if len(parts) < 4:
|
||||
continue
|
||||
labels = _parse_labels(parts[3])
|
||||
containers.append(
|
||||
ManagedContainer(
|
||||
container_id=parts[0].strip(),
|
||||
name=parts[1].strip(),
|
||||
status=parts[2].strip(),
|
||||
organization_id=labels.get(LABEL_ORG, ""),
|
||||
project_id=labels.get(LABEL_PROJECT, ""),
|
||||
role=labels.get(LABEL_ROLE, ""),
|
||||
)
|
||||
)
|
||||
return containers
|
||||
|
||||
async def remove_container(self, container_id: str) -> bool:
|
||||
result = await self._docker(["rm", "-f", "-v", container_id], timeout_seconds=60)
|
||||
self._sessions.pop(container_id, None)
|
||||
return result.ok
|
||||
|
||||
def host_dir_for(self, organization_id: str, project_id: str) -> str:
|
||||
"""Absolute host path of a project's working tree."""
|
||||
return self._workspace_dir(
|
||||
_check_id(organization_id, "سازمان"), _check_id(project_id, "پروژه")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def binary_available(binary: str | None = None) -> bool:
|
||||
"""True when the container runtime CLI is on PATH."""
|
||||
return shutil.which((binary or settings.sandbox_docker_bin or "docker").strip()) is not None
|
||||
|
||||
|
||||
def _parse_labels(raw: str) -> dict[str, str]:
|
||||
labels: dict[str, str] = {}
|
||||
for item in (raw or "").split(","):
|
||||
if "=" not in item:
|
||||
continue
|
||||
key, _, value = item.partition("=")
|
||||
labels[key.strip()] = value.strip()
|
||||
return labels
|
||||
|
||||
|
||||
def _looks_like_secret(key: str) -> bool:
|
||||
from app.core.redact import is_sensitive_key
|
||||
|
||||
return is_sensitive_key(key)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FALLBACK_IMAGE",
|
||||
"LABEL_ORG",
|
||||
"LABEL_PROJECT",
|
||||
"LABEL_ROLE",
|
||||
"LANGUAGE_IMAGES",
|
||||
"ROLE_BUILD",
|
||||
"ROLE_DEMO",
|
||||
"ROLE_WORKSPACE",
|
||||
"SANDBOX_WORKSPACE_PATH",
|
||||
"DockerSandboxAdapter",
|
||||
"ManagedContainer",
|
||||
"PathEscapeError",
|
||||
"container_name",
|
||||
"image_for_language",
|
||||
"safe_join",
|
||||
]
|
||||
@@ -0,0 +1,274 @@
|
||||
"""The one place the platform shells out to a host binary.
|
||||
|
||||
Nothing else in the codebase may call ``subprocess`` or ``os.system``. Every
|
||||
adapter that needs an external binary (docker/podman, git, caddy reload) goes
|
||||
through :func:`run_process`, which exists to make four failure modes impossible
|
||||
rather than merely unlikely:
|
||||
|
||||
**Shell injection.** The command is an ``argv`` sequence handed straight to
|
||||
``asyncio.create_subprocess_exec``. There is no ``shell=True`` path and no
|
||||
string interpolation into a shell, so a task id or a branch name can never
|
||||
become a second command.
|
||||
|
||||
**Orphaned children.** The child starts in its own session
|
||||
(``start_new_session=True``), so a timeout kills the whole *process group*. A
|
||||
``docker build`` that spawns helpers, or a ``git`` that runs a credential
|
||||
helper, dies with its parent instead of lingering as a stray process holding a
|
||||
lock.
|
||||
|
||||
**Pipe deadlock.** stdout and stderr are drained concurrently by two tasks.
|
||||
Reading one to completion before touching the other is the classic 64 KB
|
||||
deadlock: the child blocks writing stderr, we block reading stdout, and both
|
||||
wait forever. ``communicate()`` is not used because we want a byte cap and a
|
||||
streaming callback.
|
||||
|
||||
**Secret inheritance.** The child gets an explicit, minimal environment built
|
||||
from :func:`base_env`, never ``os.environ`` wholesale. The platform process
|
||||
holds the JWT secret, the database DSN and every provider key; a subprocess that
|
||||
inherits them can leak them through a stack trace, a build log or a core dump.
|
||||
|
||||
Output is capped in bytes while streaming, so a runaway build cannot exhaust
|
||||
memory. The cap is applied *after* the callback sees the chunk, so live logs
|
||||
still show everything that arrived.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
#: Environment variables a child may inherit from the platform process. Anything
|
||||
#: not listed is dropped. Deliberately tiny.
|
||||
_BASE_ENV_KEYS: tuple[str, ...] = ("PATH", "LANG", "LC_ALL", "LC_CTYPE", "TZ", "TMPDIR")
|
||||
|
||||
#: Fallback PATH when the parent has none (systemd units often do not).
|
||||
_FALLBACK_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
|
||||
#: Read size per pipe chunk. 64 KiB is the pipe buffer size, so this drains
|
||||
#: promptly without spinning.
|
||||
_CHUNK_BYTES = 65_536
|
||||
|
||||
#: How long a child gets to die politely after SIGTERM before SIGKILL.
|
||||
_TERM_GRACE_SECONDS = 5.0
|
||||
|
||||
#: An argv element longer than this is almost certainly a mistake (a base64 blob
|
||||
#: pasted into a command line). Refused rather than silently truncated.
|
||||
MAX_ARGV_ELEMENT_CHARS = 100_000
|
||||
|
||||
|
||||
class ProcessError(RuntimeError):
|
||||
"""A host command could not be started at all (binary missing, not executable)."""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ProcessResult:
|
||||
"""Outcome of one host command.
|
||||
|
||||
``argv`` is retained so a test can assert on the *exact* argument vector —
|
||||
which is how the hardening flags and the "no credential in argv" rule are
|
||||
verified.
|
||||
"""
|
||||
|
||||
argv: tuple[str, ...]
|
||||
exit_code: int
|
||||
stdout: str = ""
|
||||
stderr: str = ""
|
||||
duration_ms: int = 0
|
||||
timed_out: bool = False
|
||||
truncated: bool = False
|
||||
#: True when the binary itself could not be executed (ENOENT/EACCES).
|
||||
spawn_failed: bool = False
|
||||
spawn_error: str = ""
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.exit_code == 0 and not self.timed_out and not self.spawn_failed
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _DrainResult:
|
||||
data: bytes = b""
|
||||
truncated: bool = False
|
||||
lines: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def base_env(extra: Mapping[str, str] | None = None) -> dict[str, str]:
|
||||
"""Minimal child environment: a few safe keys plus explicit overrides.
|
||||
|
||||
Callers add what they need (``HOME`` for git, ``GIT_CONFIG_NOSYSTEM=1``) and
|
||||
nothing else is inherited.
|
||||
"""
|
||||
env = {key: os.environ[key] for key in _BASE_ENV_KEYS if key in os.environ}
|
||||
env.setdefault("PATH", _FALLBACK_PATH)
|
||||
if extra:
|
||||
env.update({str(k): str(v) for k, v in extra.items()})
|
||||
return env
|
||||
|
||||
|
||||
def _validate_argv(argv: Sequence[str]) -> tuple[str, ...]:
|
||||
if not argv:
|
||||
raise ValueError("argv must not be empty")
|
||||
cleaned: list[str] = []
|
||||
for index, element in enumerate(argv):
|
||||
if not isinstance(element, str):
|
||||
raise TypeError(f"argv[{index}] must be str, got {type(element).__name__}")
|
||||
if "\x00" in element:
|
||||
raise ValueError(f"argv[{index}] contains a NUL byte")
|
||||
if len(element) > MAX_ARGV_ELEMENT_CHARS:
|
||||
raise ValueError(
|
||||
f"argv[{index}] is {len(element)} chars; "
|
||||
f"the limit is {MAX_ARGV_ELEMENT_CHARS}"
|
||||
)
|
||||
cleaned.append(element)
|
||||
return tuple(cleaned)
|
||||
|
||||
|
||||
async def _drain(
|
||||
stream: asyncio.StreamReader | None,
|
||||
*,
|
||||
cap_bytes: int,
|
||||
on_output: Callable[[str], Awaitable[None]] | None,
|
||||
) -> _DrainResult:
|
||||
"""Read a pipe to EOF, keeping at most ``cap_bytes`` and streaming every chunk."""
|
||||
result = _DrainResult()
|
||||
if stream is None:
|
||||
return result
|
||||
|
||||
buffer = bytearray()
|
||||
truncated = False
|
||||
while True:
|
||||
chunk = await stream.read(_CHUNK_BYTES)
|
||||
if not chunk:
|
||||
break
|
||||
if len(buffer) < cap_bytes:
|
||||
remaining = cap_bytes - len(buffer)
|
||||
buffer.extend(chunk[:remaining])
|
||||
if len(chunk) > remaining:
|
||||
truncated = True
|
||||
else:
|
||||
truncated = True
|
||||
if on_output is not None:
|
||||
# The callback sees the *raw* chunk even past the cap: a live log
|
||||
# should not stop scrolling just because we stopped storing.
|
||||
with contextlib.suppress(Exception):
|
||||
await on_output(chunk.decode("utf-8", "replace"))
|
||||
|
||||
result.data = bytes(buffer)
|
||||
result.truncated = truncated
|
||||
return result
|
||||
|
||||
|
||||
async def _terminate(proc: asyncio.subprocess.Process) -> None:
|
||||
"""SIGTERM the child's process group, then SIGKILL if it will not go."""
|
||||
if proc.returncode is not None:
|
||||
return
|
||||
|
||||
def _signal_group(sig: int) -> None:
|
||||
# The child is a session leader (``start_new_session=True``), so its pid
|
||||
# *is* its process-group id. Fall back to the pid on platforms without
|
||||
# process groups.
|
||||
with contextlib.suppress(ProcessLookupError, PermissionError, OSError):
|
||||
if hasattr(os, "killpg"):
|
||||
os.killpg(proc.pid, sig)
|
||||
else: # pragma: no cover - POSIX is the only supported target
|
||||
proc.send_signal(sig)
|
||||
|
||||
_signal_group(signal.SIGTERM)
|
||||
try:
|
||||
await asyncio.wait_for(proc.wait(), timeout=_TERM_GRACE_SECONDS)
|
||||
return
|
||||
except TimeoutError:
|
||||
pass
|
||||
_signal_group(signal.SIGKILL)
|
||||
with contextlib.suppress(Exception):
|
||||
await proc.wait()
|
||||
|
||||
|
||||
async def run_process(
|
||||
argv: Sequence[str],
|
||||
*,
|
||||
cwd: str | None = None,
|
||||
env: Mapping[str, str] | None = None,
|
||||
timeout_seconds: float = 120.0,
|
||||
max_output_bytes: int = 200_000,
|
||||
on_output: Callable[[str], Awaitable[None]] | None = None,
|
||||
stdin_data: bytes | None = None,
|
||||
) -> ProcessResult:
|
||||
"""Run ``argv`` and return its bounded, timed result.
|
||||
|
||||
Never raises for a non-zero exit or a timeout — those are ordinary outcomes
|
||||
the caller inspects. It *does* raise :class:`ProcessError` when the binary
|
||||
cannot be started, because that is a deployment fault, not a command result.
|
||||
|
||||
``stdin_data`` is written and closed immediately; the platform never hands a
|
||||
child an interactive terminal.
|
||||
"""
|
||||
cleaned = _validate_argv(argv)
|
||||
started = time.perf_counter()
|
||||
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cleaned,
|
||||
cwd=cwd,
|
||||
env=base_env(env),
|
||||
stdin=asyncio.subprocess.PIPE if stdin_data is not None else asyncio.subprocess.DEVNULL,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
start_new_session=True,
|
||||
)
|
||||
except (FileNotFoundError, NotADirectoryError, PermissionError, OSError) as exc:
|
||||
return ProcessResult(
|
||||
argv=cleaned,
|
||||
exit_code=127,
|
||||
stderr=str(exc),
|
||||
duration_ms=int((time.perf_counter() - started) * 1000),
|
||||
spawn_failed=True,
|
||||
spawn_error=type(exc).__name__,
|
||||
)
|
||||
|
||||
if stdin_data is not None and proc.stdin is not None:
|
||||
with contextlib.suppress(BrokenPipeError, ConnectionResetError):
|
||||
proc.stdin.write(stdin_data)
|
||||
await proc.stdin.drain()
|
||||
with contextlib.suppress(Exception):
|
||||
proc.stdin.close()
|
||||
|
||||
stdout_task = asyncio.create_task(
|
||||
_drain(proc.stdout, cap_bytes=max_output_bytes, on_output=on_output)
|
||||
)
|
||||
stderr_task = asyncio.create_task(
|
||||
_drain(proc.stderr, cap_bytes=max_output_bytes, on_output=None)
|
||||
)
|
||||
|
||||
timed_out = False
|
||||
try:
|
||||
await asyncio.wait_for(proc.wait(), timeout=max(1.0, float(timeout_seconds)))
|
||||
except TimeoutError:
|
||||
timed_out = True
|
||||
await _terminate(proc)
|
||||
|
||||
stdout_result, stderr_result = await asyncio.gather(stdout_task, stderr_task)
|
||||
|
||||
return ProcessResult(
|
||||
argv=cleaned,
|
||||
exit_code=proc.returncode if proc.returncode is not None else -1,
|
||||
stdout=stdout_result.data.decode("utf-8", "replace"),
|
||||
stderr=stderr_result.data.decode("utf-8", "replace"),
|
||||
duration_ms=int((time.perf_counter() - started) * 1000),
|
||||
timed_out=timed_out,
|
||||
truncated=stdout_result.truncated or stderr_result.truncated,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MAX_ARGV_ELEMENT_CHARS",
|
||||
"ProcessError",
|
||||
"ProcessResult",
|
||||
"base_env",
|
||||
"run_process",
|
||||
]
|
||||
@@ -89,6 +89,13 @@ def get_sandbox_provider():
|
||||
provider = settings.sandbox_provider.lower()
|
||||
if provider == "mock":
|
||||
return mock.LocalMockSandboxAdapter(max_output_bytes=settings.sandbox_max_output_bytes)
|
||||
if provider == "docker":
|
||||
# Local docker/podman. One instance per process so the workspace
|
||||
# registry (external id → host dir) stays consistent between
|
||||
# ``create_workspace`` and the ``start_session`` that follows it.
|
||||
from app.infrastructure.adapters.docker_sandbox import DockerSandboxAdapter
|
||||
|
||||
return DockerSandboxAdapter()
|
||||
if provider in {"daytona", "e2b"}:
|
||||
from app.infrastructure.adapters.sandbox_remote import build_remote_sandbox_adapter
|
||||
|
||||
@@ -99,7 +106,8 @@ def get_sandbox_provider():
|
||||
e2b_api_key=settings.e2b_api_key,
|
||||
)
|
||||
raise ProviderUnavailableError(
|
||||
f"SANDBOX_PROVIDER='{provider}' پشتیبانی نمیشود. مقادیر مجاز: mock | daytona | e2b"
|
||||
f"SANDBOX_PROVIDER='{provider}' پشتیبانی نمیشود. "
|
||||
"مقادیر مجاز: mock | docker | daytona | e2b"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -198,6 +198,29 @@ class SandboxSessionRepository:
|
||||
)
|
||||
return [_session(r) for r in (await self.session.execute(stmt)).scalars()]
|
||||
|
||||
async def list_active_external_ids(self, *, limit: int = 5_000) -> set[str]:
|
||||
"""External ids of every session that is not stopped.
|
||||
|
||||
The orphan reaper's query: a container whose id is not in this set has
|
||||
no live row pointing at it, so it is holding resources for nobody.
|
||||
"""
|
||||
stmt = (
|
||||
select(SandboxSessionRow.external_session_id)
|
||||
.where(
|
||||
SandboxSessionRow.status.in_(
|
||||
[
|
||||
str(SessionStatus.STARTING),
|
||||
str(SessionStatus.RUNNING),
|
||||
str(SessionStatus.PAUSED),
|
||||
]
|
||||
),
|
||||
SandboxSessionRow.external_session_id.is_not(None),
|
||||
)
|
||||
.limit(min(limit, 20_000))
|
||||
)
|
||||
rows = (await self.session.execute(stmt)).scalars().all()
|
||||
return {str(value) for value in rows if value}
|
||||
|
||||
async def create(self, sandbox_session: SandboxSession) -> SandboxSession:
|
||||
row = SandboxSessionRow(
|
||||
id=sandbox_session.id, workspace_id=sandbox_session.workspace_id,
|
||||
|
||||
@@ -4,8 +4,17 @@ Note what is **absent**: there is no route that accepts a shell command and runs
|
||||
it on the host. Every execution goes to ``SandboxProviderPort``, is bounded by a
|
||||
policy timeout and output cap, and is metered into the usage ledger.
|
||||
|
||||
Running a command requires ``sandbox.use``; tearing a workspace down requires
|
||||
``projects.manage`` because it destroys work.
|
||||
**Who may call what (v2).** Containers, ports and command output are
|
||||
*infrastructure*, and v2 gives infrastructure to the platform owner alone
|
||||
(PLAN-v2 §7.7): a customer only ever gives orders to the AI. So every mutating
|
||||
route here — create/start/stop a workspace or session, run a command, read or
|
||||
write a file, snapshot, mint a preview URL, set limits — requires the platform
|
||||
owner. The read-only status routes (templates, effective limits, workspace list,
|
||||
the command log) stay open to org members, because the project page shows them.
|
||||
|
||||
The org-scoped prefix is kept even for owner-only routes: it is what scopes the
|
||||
audit trail and the tenancy check, and it means the platform owner must
|
||||
*deliberately* be a member of the org they are operating on.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -19,8 +28,8 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.application.services.audit_service import record_audit
|
||||
from app.application.services.sandbox_service import SandboxService
|
||||
from app.core.database import get_session
|
||||
from app.core.errors import PermissionDeniedError
|
||||
from app.domain.audit.models import AuditAction
|
||||
from app.domain.identity.models import CAN_MANAGE_PROJECTS, CAN_USE_SANDBOX
|
||||
from app.domain.sandbox.models import NetworkPolicy
|
||||
from app.interfaces.http.deps import (
|
||||
OrgContext,
|
||||
@@ -32,6 +41,19 @@ from app.interfaces.http.deps import (
|
||||
router = APIRouter(prefix="/organizations/{org_id}/sandbox", tags=["sandbox"])
|
||||
|
||||
|
||||
def _require_platform_owner(context: OrgContext) -> None:
|
||||
"""Infrastructure belongs to the platform owner (PLAN-v2 §7.7).
|
||||
|
||||
Deliberately *not* an org-capability check: an org admin running a command
|
||||
in a container is exactly the "users touch infrastructure" behaviour v2
|
||||
removes.
|
||||
"""
|
||||
if not context.user.is_platform_owner:
|
||||
raise PermissionDeniedError(
|
||||
"دسترسی به زیرساخت فقط برای مالک سامانه امکانپذیر است."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -181,7 +203,7 @@ async def set_limits(
|
||||
context: OrgContext = Depends(get_org_context),
|
||||
service: SandboxService = Depends(get_sandbox_service),
|
||||
) -> ResourceLimitOut:
|
||||
context.require(CAN_MANAGE_PROJECTS)
|
||||
_require_platform_owner(context)
|
||||
limit = await service.set_limits(
|
||||
scope=payload.scope, scope_id=payload.scope_id or context.org_id,
|
||||
cpu=payload.cpu, memory_mb=payload.memory_mb, disk_mb=payload.disk_mb,
|
||||
@@ -211,7 +233,7 @@ async def create_workspace(
|
||||
service: SandboxService = Depends(get_sandbox_service),
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> WorkspaceOut:
|
||||
context.require(CAN_USE_SANDBOX)
|
||||
_require_platform_owner(context)
|
||||
|
||||
workspace = await service.create_workspace(
|
||||
organization_id=context.org_id,
|
||||
@@ -264,7 +286,7 @@ async def destroy_workspace(
|
||||
service: SandboxService = Depends(get_sandbox_service),
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> WorkspaceOut:
|
||||
context.require(CAN_MANAGE_PROJECTS)
|
||||
_require_platform_owner(context)
|
||||
workspace = await service.destroy_workspace(
|
||||
organization_id=context.org_id, workspace_id=workspace_id
|
||||
)
|
||||
@@ -295,7 +317,7 @@ async def start_session(
|
||||
context: OrgContext = Depends(get_org_context),
|
||||
service: SandboxService = Depends(get_sandbox_service),
|
||||
) -> SessionOut:
|
||||
context.require(CAN_USE_SANDBOX)
|
||||
_require_platform_owner(context)
|
||||
sandbox_session = await service.start_session(
|
||||
organization_id=context.org_id, workspace_id=payload.workspace_id
|
||||
)
|
||||
@@ -312,7 +334,7 @@ async def stop_session(
|
||||
context: OrgContext = Depends(get_org_context),
|
||||
service: SandboxService = Depends(get_sandbox_service),
|
||||
) -> SessionOut:
|
||||
context.require(CAN_USE_SANDBOX)
|
||||
_require_platform_owner(context)
|
||||
sandbox_session = await service.stop_session(
|
||||
organization_id=context.org_id, session_id=session_id
|
||||
)
|
||||
@@ -337,7 +359,7 @@ async def run_command(
|
||||
service: SandboxService = Depends(get_sandbox_service),
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> CommandRunOut:
|
||||
context.require(CAN_USE_SANDBOX)
|
||||
_require_platform_owner(context)
|
||||
|
||||
run, _result = await service.run_command(
|
||||
organization_id=context.org_id,
|
||||
@@ -389,7 +411,7 @@ async def read_file(
|
||||
context: OrgContext = Depends(get_org_context),
|
||||
service: SandboxService = Depends(get_sandbox_service),
|
||||
) -> FileReadOut:
|
||||
context.require(CAN_USE_SANDBOX)
|
||||
_require_platform_owner(context)
|
||||
content = await service.read_file(
|
||||
organization_id=context.org_id, session_id=session_id, path=payload.path
|
||||
)
|
||||
@@ -403,7 +425,7 @@ async def write_file(
|
||||
context: OrgContext = Depends(get_org_context),
|
||||
service: SandboxService = Depends(get_sandbox_service),
|
||||
) -> None:
|
||||
context.require(CAN_USE_SANDBOX)
|
||||
_require_platform_owner(context)
|
||||
await service.write_file(
|
||||
organization_id=context.org_id, session_id=session_id,
|
||||
path=payload.path, content=payload.content,
|
||||
@@ -426,7 +448,7 @@ async def create_snapshot(
|
||||
context: OrgContext = Depends(get_org_context),
|
||||
service: SandboxService = Depends(get_sandbox_service),
|
||||
) -> SnapshotOut:
|
||||
context.require(CAN_USE_SANDBOX)
|
||||
_require_platform_owner(context)
|
||||
snapshot = await service.snapshot(
|
||||
organization_id=context.org_id, session_id=session_id, name=payload.name
|
||||
)
|
||||
@@ -442,7 +464,7 @@ async def get_preview_url(
|
||||
context: OrgContext = Depends(get_org_context),
|
||||
service: SandboxService = Depends(get_sandbox_service),
|
||||
) -> PreviewUrlOut:
|
||||
context.require(CAN_USE_SANDBOX)
|
||||
_require_platform_owner(context)
|
||||
preview = await service.get_preview_url(
|
||||
organization_id=context.org_id, session_id=session_id, port=port
|
||||
)
|
||||
|
||||
@@ -86,7 +86,7 @@ def _sandbox_gateway_pass(args: argparse.Namespace):
|
||||
gateway = SandboxGatewayWorker()
|
||||
|
||||
async def one_pass() -> WorkerOutcome:
|
||||
return await gateway.reap(
|
||||
return await gateway.run_pass(
|
||||
idle_minutes=args.idle_minutes, limit=max(args.limit, 200)
|
||||
)
|
||||
|
||||
|
||||
@@ -23,10 +23,19 @@ from datetime import timedelta
|
||||
from app.application.container import build_sandbox_service
|
||||
from app.application.ports.realtime import emit_sandbox_reaped
|
||||
from app.core.config import settings
|
||||
from app.domain.audit.models import AuditAction, AuditEvent
|
||||
from app.domain.audit.models import (
|
||||
AuditAction,
|
||||
AuditEvent,
|
||||
ErrorSeverity,
|
||||
ErrorSource,
|
||||
)
|
||||
from app.domain.base import utcnow
|
||||
from app.infrastructure.error_log import record_error
|
||||
from app.infrastructure.repositories.audit import AuditRepository
|
||||
from app.infrastructure.repositories.sandbox import WorkspaceRepository
|
||||
from app.infrastructure.repositories.sandbox import (
|
||||
SandboxSessionRepository,
|
||||
WorkspaceRepository,
|
||||
)
|
||||
from app.workers.base import WorkerOutcome, worker_session
|
||||
|
||||
#: Sessions are considered abandoned after this long with no activity.
|
||||
@@ -38,6 +47,22 @@ class SandboxGatewayWorker:
|
||||
|
||||
name = "sandbox-gateway"
|
||||
|
||||
async def run_pass(
|
||||
self, *, idle_minutes: int = DEFAULT_IDLE_MINUTES, limit: int = 200
|
||||
) -> WorkerOutcome:
|
||||
"""One housekeeping pass: idle sessions first, then orphan containers.
|
||||
|
||||
Combined into one pass so a cron entry needs a single invocation and an
|
||||
operator sees one summary line per run.
|
||||
"""
|
||||
outcome = await self.reap(idle_minutes=idle_minutes, limit=limit)
|
||||
orphans = await self.reap_orphans(limit=limit)
|
||||
outcome.processed += orphans.processed
|
||||
outcome.failed += orphans.failed
|
||||
for note in orphans.notes:
|
||||
outcome.note(note)
|
||||
return outcome
|
||||
|
||||
async def reap(
|
||||
self, *, idle_minutes: int = DEFAULT_IDLE_MINUTES, limit: int = 200
|
||||
) -> WorkerOutcome:
|
||||
@@ -98,5 +123,86 @@ class SandboxGatewayWorker:
|
||||
outcome.note("egress policy default = deny_all (set per resource limit)")
|
||||
return outcome
|
||||
|
||||
async def reap_orphans(self, *, limit: int = 200) -> WorkerOutcome:
|
||||
"""Remove labelled containers that no live session row points at.
|
||||
|
||||
``reap`` above stops *sessions*; this stops *containers*. The two are
|
||||
not the same: a session row can be stopped (or the database restored
|
||||
from a backup) while the container keeps running, and a container can be
|
||||
started by a process that then died before writing its row. Without this
|
||||
pass those containers hold CPU, memory and disk forever, because nothing
|
||||
in the application knows they exist.
|
||||
|
||||
Only the container runtime is inspected. A provider without
|
||||
``list_managed_containers`` (the mock, the remote adapters) has nothing
|
||||
to reap and is skipped silently — this is not an error condition.
|
||||
"""
|
||||
outcome = WorkerOutcome(name=self.name)
|
||||
|
||||
from app.infrastructure.adapters.registry import get_sandbox_provider
|
||||
|
||||
provider = get_sandbox_provider()
|
||||
lister = getattr(provider, "list_managed_containers", None)
|
||||
if lister is None:
|
||||
outcome.skipped += 1
|
||||
outcome.note("provider does not expose managed containers; nothing to reap")
|
||||
return outcome
|
||||
|
||||
try:
|
||||
containers = await lister()
|
||||
except Exception as exc: # noqa: BLE001 - a CLI outage must not kill the worker
|
||||
await record_error(
|
||||
kind="sandbox_orphan_list_failed",
|
||||
message=str(exc),
|
||||
exc=exc,
|
||||
source=ErrorSource.WORKER,
|
||||
severity=ErrorSeverity.WARNING,
|
||||
metadata={"worker": self.name},
|
||||
)
|
||||
outcome.failed += 1
|
||||
return outcome
|
||||
|
||||
async with worker_session() as session:
|
||||
live = await SandboxSessionRepository(session).list_active_external_ids()
|
||||
audit = AuditRepository(session)
|
||||
|
||||
reaped = 0
|
||||
for container in containers:
|
||||
if reaped >= limit:
|
||||
break
|
||||
if container.container_id in live:
|
||||
continue
|
||||
# A demo/build container is owned by the delivery pipeline, which
|
||||
# tracks it in its own table. Until that table exists it is left
|
||||
# alone rather than guessed at.
|
||||
if container.role and container.role != "workspace":
|
||||
outcome.skipped += 1
|
||||
continue
|
||||
removed = await provider.remove_container(container.container_id)
|
||||
if not removed:
|
||||
outcome.failed += 1
|
||||
continue
|
||||
reaped += 1
|
||||
outcome.processed += 1
|
||||
outcome.note(f"removed orphan container {container.container_id}")
|
||||
await audit.record(
|
||||
event=AuditEvent(
|
||||
action=AuditAction.SANDBOX_REAP,
|
||||
organization_id=container.organization_id or None,
|
||||
target_type="sandbox_container",
|
||||
target_id=container.container_id,
|
||||
metadata_json={
|
||||
"reason": "orphan_container",
|
||||
"project_id": container.project_id,
|
||||
"role": container.role,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
if not reaped:
|
||||
outcome.skipped += 1
|
||||
|
||||
return outcome
|
||||
|
||||
|
||||
__all__ = ["DEFAULT_IDLE_MINUTES", "SandboxGatewayWorker"]
|
||||
|
||||
Reference in New Issue
Block a user