feat(delivery): live CI/CD pipeline + demo runtime (v2 phase A3-3)
A push to `dev` now becomes a live demo, and a failed build never takes the demo down. - Four tables (pipeline_runs, pipeline_steps, releases, app_runtimes) and their migration; alembic upgrade/downgrade/check all clean. - DeliveryService implements DeliveryHooks: on_app_created provisions the demo port + host + first build (idempotently); on_branch_pushed queues a run for `dev` and supersedes older *queued* runs, leaving a running one alone. - The 8-step pipeline. Blue/green is real: the replacement starts on a staging port, is healthchecked, and only then does the route move — then the previous container is stopped. A failure removes the replacement, releases its port and leaves the old container serving. - The demo port ledger is the DATABASE, not the router adapters' in-memory maps: the API and the delivery worker are separate processes. - Runs are claimed with a conditional UPDATE, so two workers never build the same run. - Live logs are batched (1 event / 500 ms, 200 lines) and redacted as a whole tail — per-chunk redaction misses a secret split across pipe reads, and the *stored* tail leaked until this was fixed. - delivery worker: drain the queue, then poll remotes for pushes made outside the engine. - 7 routes; reads need membership, actions need projects.manage and are audited. Tests: 415 API (+21, 1 docker-marked skipped); ruff, mypy, alembic check clean. Report: docs/plan/reports/agent-3-phase-3.md
This commit is contained in:
mjalmoudy94
@@ -17,6 +17,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.application.container_delivery import (
|
||||
build_delivery_hooks,
|
||||
build_delivery_service,
|
||||
build_deploy_target_tester,
|
||||
build_repo_workspace,
|
||||
)
|
||||
@@ -166,6 +167,7 @@ def build_sandbox_service(session: AsyncSession) -> SandboxService:
|
||||
__all__ = [
|
||||
"build_billing_service",
|
||||
"build_delivery_hooks",
|
||||
"build_delivery_service",
|
||||
"build_deploy_target_tester",
|
||||
"build_memory",
|
||||
"build_model_grants",
|
||||
|
||||
@@ -18,17 +18,28 @@ import logging
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.application.ports.delivery_hooks import DeliveryHooks, NullDeliveryHooks
|
||||
from app.application.ports.delivery_hooks import DeliveryHooks
|
||||
from app.application.ports.deploy_target import DeployTargetTester, FakeDeployTargetTester
|
||||
from app.application.ports.repo_workspace import FakeRepoWorkspace, RepoWorkspacePort
|
||||
from app.application.services.delivery_service import DeliveryService
|
||||
from app.core.config import settings
|
||||
from app.core.crypto import decrypt
|
||||
from app.infrastructure.adapters.git_workspace import (
|
||||
LocalGitWorkspaceAdapter,
|
||||
ProjectGitSource,
|
||||
)
|
||||
from app.infrastructure.adapters.registry import get_domain_router, get_sandbox_provider
|
||||
from app.infrastructure.db.models import ProjectRow
|
||||
from app.infrastructure.repositories.app_profile import AppProfileRepository
|
||||
from app.infrastructure.repositories.delivery import (
|
||||
AppRuntimeRepository,
|
||||
PipelineRunRepository,
|
||||
PipelineStepRepository,
|
||||
ReleaseRepository,
|
||||
)
|
||||
from app.infrastructure.repositories.identity import OrganizationRepository
|
||||
from app.infrastructure.repositories.projects import (
|
||||
PreviewDomainRepository,
|
||||
ProjectExternalGitRepository,
|
||||
ProjectGitCredentialRepository,
|
||||
ProjectGitRepoRepository,
|
||||
@@ -38,7 +49,6 @@ from app.infrastructure.repositories.projects import (
|
||||
logger = logging.getLogger("agy.container.delivery")
|
||||
|
||||
_fake_workspace = FakeRepoWorkspace()
|
||||
_null_hooks = NullDeliveryHooks()
|
||||
_fake_deploy_tester = FakeDeployTargetTester()
|
||||
|
||||
|
||||
@@ -119,7 +129,60 @@ def _decrypt_or_none(ciphertext: str | None, project_id: str) -> str | None:
|
||||
|
||||
|
||||
def build_delivery_hooks(session: AsyncSession) -> DeliveryHooks:
|
||||
return _null_hooks
|
||||
"""The real CI/CD service — it *is* the hook implementation.
|
||||
|
||||
The org wizard calls `on_app_created` after its own transaction commits, and
|
||||
the engine calls `on_branch_pushed` after a successful merge; both get the
|
||||
same object the delivery router and worker use, so there is exactly one
|
||||
implementation of "a push becomes a build".
|
||||
"""
|
||||
return build_delivery_service(session)
|
||||
|
||||
|
||||
def build_delivery_service(session: AsyncSession) -> DeliveryService:
|
||||
"""CI/CD: pipeline runs, the demo runtime and the deploy hooks."""
|
||||
return DeliveryService(
|
||||
runs=PipelineRunRepository(session),
|
||||
steps=PipelineStepRepository(session),
|
||||
releases=ReleaseRepository(session),
|
||||
runtimes=AppRuntimeRepository(session),
|
||||
projects=ProjectRepository(session),
|
||||
profiles=AppProfileRepository(session),
|
||||
organizations=OrganizationRepository(session),
|
||||
ports=PreviewDomainRepository(session),
|
||||
workspace=build_repo_workspace(session),
|
||||
domains=get_domain_router(),
|
||||
containers=_container_runtime(),
|
||||
)
|
||||
|
||||
|
||||
def _container_runtime():
|
||||
"""The adapter that can start build and demo containers.
|
||||
|
||||
Falls back to a runtime that refuses everything when the configured sandbox
|
||||
provider cannot run containers (the mock, or a remote provider): a pipeline
|
||||
run then fails cleanly with a plain message instead of pretending to deploy.
|
||||
"""
|
||||
provider = get_sandbox_provider()
|
||||
if hasattr(provider, "run_step") and hasattr(provider, "remove_container"):
|
||||
return provider
|
||||
return _UnavailableContainerRuntime()
|
||||
|
||||
|
||||
class _UnavailableContainerRuntime:
|
||||
"""A container runtime that cannot run anything, and says so."""
|
||||
|
||||
async def run_step(self, **_kwargs): # noqa: ANN003
|
||||
from app.application.ports import CommandResult
|
||||
|
||||
return CommandResult(
|
||||
exit_code=127,
|
||||
stdout="",
|
||||
stderr="SANDBOX_PROVIDER cannot run containers; set SANDBOX_PROVIDER=docker.",
|
||||
)
|
||||
|
||||
async def remove_container(self, container_id: str) -> bool: # noqa: ARG002
|
||||
return False
|
||||
|
||||
|
||||
def build_deploy_target_tester(session: AsyncSession) -> DeployTargetTester:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,159 @@
|
||||
"""Delivery domain: CI/CD runs, releases and the demo runtime.
|
||||
|
||||
Framework-free, like every other domain package. The interesting rules live in
|
||||
the *state* enums rather than in behaviour, because the behaviour is all about
|
||||
ordering: a run is `queued → running → succeeded|failed|cancelled`, a step is
|
||||
`pending → running → succeeded|failed|skipped`, and a release only becomes the
|
||||
live demo one when the healthcheck has passed and the route has been switched.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
|
||||
from app.domain.base import new_id, utcnow
|
||||
|
||||
|
||||
class RunTrigger(StrEnum):
|
||||
"""Why a pipeline run exists.
|
||||
|
||||
``engine`` and ``poll`` are the two automatic sources; ``manual`` is an
|
||||
org admin pressing re-run. The distinction matters in the audit trail: "who
|
||||
started this build?" is the first question asked when a deploy goes wrong.
|
||||
"""
|
||||
|
||||
ENGINE = "engine"
|
||||
POLL = "poll"
|
||||
MANUAL = "manual"
|
||||
|
||||
|
||||
class RunStatus(StrEnum):
|
||||
QUEUED = "queued"
|
||||
RUNNING = "running"
|
||||
SUCCEEDED = "succeeded"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
#: A newer run for the same project replaced this one before it started.
|
||||
SUPERSEDED = "superseded"
|
||||
|
||||
|
||||
class StepStatus(StrEnum):
|
||||
PENDING = "pending"
|
||||
RUNNING = "running"
|
||||
SUCCEEDED = "succeeded"
|
||||
FAILED = "failed"
|
||||
#: The step had nothing to do (an empty build_command, say). Not a failure.
|
||||
SKIPPED = "skipped"
|
||||
|
||||
|
||||
class RuntimeStatus(StrEnum):
|
||||
PROVISIONING = "provisioning"
|
||||
LIVE = "live"
|
||||
DEGRADED = "degraded"
|
||||
STOPPED = "stopped"
|
||||
|
||||
|
||||
#: The eight steps of a run, in order. The names are the product contract: the
|
||||
#: UI renders them verbatim and the tests assert on the exact sequence.
|
||||
PIPELINE_STEPS: tuple[str, ...] = (
|
||||
"checkout",
|
||||
"setup",
|
||||
"build",
|
||||
"test",
|
||||
"package",
|
||||
"deploy_demo",
|
||||
"healthcheck",
|
||||
"route",
|
||||
)
|
||||
|
||||
#: Steps that are skipped (not failed) when the app profile leaves the command
|
||||
#: empty. `checkout`, `deploy_demo`, `healthcheck` and `route` always run.
|
||||
OPTIONAL_STEPS: frozenset[str] = frozenset({"setup", "build", "test"})
|
||||
|
||||
#: Statuses that mean "this run is no longer interesting".
|
||||
TERMINAL_RUN_STATUSES: frozenset[str] = frozenset(
|
||||
{
|
||||
RunStatus.SUCCEEDED.value,
|
||||
RunStatus.FAILED.value,
|
||||
RunStatus.CANCELLED.value,
|
||||
RunStatus.SUPERSEDED.value,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True, kw_only=True)
|
||||
class PipelineRun:
|
||||
organization_id: str
|
||||
project_id: str
|
||||
branch: str = "dev"
|
||||
sha: str = ""
|
||||
trigger: str = RunTrigger.ENGINE.value
|
||||
status: str = RunStatus.QUEUED.value
|
||||
error_summary: str = ""
|
||||
id: str = field(default_factory=new_id)
|
||||
started_at: datetime | None = None
|
||||
finished_at: datetime | None = None
|
||||
created_at: datetime = field(default_factory=utcnow)
|
||||
|
||||
@property
|
||||
def is_terminal(self) -> bool:
|
||||
return self.status in TERMINAL_RUN_STATUSES
|
||||
|
||||
@property
|
||||
def is_active(self) -> bool:
|
||||
return self.status in {RunStatus.QUEUED.value, RunStatus.RUNNING.value}
|
||||
|
||||
|
||||
@dataclass(slots=True, kw_only=True)
|
||||
class PipelineStep:
|
||||
run_id: str
|
||||
index: int
|
||||
name: str
|
||||
status: str = StepStatus.PENDING.value
|
||||
exit_code: int | None = None
|
||||
log_tail: str = ""
|
||||
id: str = field(default_factory=new_id)
|
||||
started_at: datetime | None = None
|
||||
finished_at: datetime | None = None
|
||||
|
||||
@property
|
||||
def duration_ms(self) -> int:
|
||||
if not self.started_at or not self.finished_at:
|
||||
return 0
|
||||
return max(0, int((self.finished_at - self.started_at).total_seconds() * 1000))
|
||||
|
||||
|
||||
@dataclass(slots=True, kw_only=True)
|
||||
class Release:
|
||||
organization_id: str
|
||||
project_id: str
|
||||
run_id: str
|
||||
sha: str
|
||||
artifact_path: str = ""
|
||||
is_demo_live: bool = False
|
||||
id: str = field(default_factory=new_id)
|
||||
created_at: datetime = field(default_factory=utcnow)
|
||||
|
||||
|
||||
@dataclass(slots=True, kw_only=True)
|
||||
class AppRuntime:
|
||||
"""The one demo container a project has.
|
||||
|
||||
``host_port`` is the *live* port; a blue/green switch starts the replacement
|
||||
on a second port and only rewrites this row once the healthcheck passes.
|
||||
"""
|
||||
|
||||
project_id: str
|
||||
organization_id: str
|
||||
demo_host: str = ""
|
||||
host_port: int | None = None
|
||||
demo_container_id: str | None = None
|
||||
status: str = RuntimeStatus.PROVISIONING.value
|
||||
current_release_id: str | None = None
|
||||
updated_at: datetime = field(default_factory=utcnow)
|
||||
|
||||
@property
|
||||
def demo_url(self) -> str:
|
||||
return f"https://{self.demo_host}" if self.demo_host else ""
|
||||
@@ -475,6 +475,107 @@ class DockerSandboxAdapter:
|
||||
|
||||
# -- demo / build containers (used by the delivery service) --------------
|
||||
|
||||
async def run_step(
|
||||
self,
|
||||
*,
|
||||
organization_id: str,
|
||||
project_id: str,
|
||||
run_id: str,
|
||||
image: str,
|
||||
work_dir: str,
|
||||
command: str,
|
||||
timeout_seconds: float,
|
||||
on_output: Callable[[str], Awaitable[None]] | None = None,
|
||||
role: str = ROLE_BUILD,
|
||||
publish_port: int | None = None,
|
||||
app_port: int | None = None,
|
||||
keep_alive: bool = False,
|
||||
name_suffix: str = "",
|
||||
) -> ProcessResult:
|
||||
"""Run one pipeline step in a **fresh** container over ``work_dir``.
|
||||
|
||||
A fresh container per step is the point: a build must not be able to
|
||||
inherit a process, a cache or a stray file from the previous step or from
|
||||
an employee's interactive workspace. ``--rm`` guarantees nothing is left
|
||||
behind on the host.
|
||||
|
||||
``publish_port`` publishes on ``127.0.0.1`` only. ``keep_alive`` starts
|
||||
the container detached so it outlives this call — that is how the demo
|
||||
container is created; a build step is run in the foreground so its output
|
||||
streams straight back.
|
||||
"""
|
||||
org = _check_id(organization_id, "سازمان")
|
||||
project = _check_id(project_id, "پروژه")
|
||||
safe_run = re.sub(r"[^a-z0-9]", "", (run_id or "").lower())[:12] or "run"
|
||||
|
||||
name = f"smartorg-{role}-{org}-{project}-{safe_run}"
|
||||
if name_suffix:
|
||||
name = f"{name}-{_SNAPSHOT_NAME.sub('-', name_suffix.lower()).strip('-')}"
|
||||
name = name[:200]
|
||||
|
||||
args: list[str] = ["run"]
|
||||
if keep_alive:
|
||||
args.append("-d")
|
||||
else:
|
||||
args.extend(["--rm"])
|
||||
args.extend(
|
||||
[
|
||||
"--name",
|
||||
name,
|
||||
"--user",
|
||||
"1000:1000",
|
||||
"--cap-drop",
|
||||
"ALL",
|
||||
"--security-opt",
|
||||
"no-new-privileges",
|
||||
"--pids-limit",
|
||||
"512",
|
||||
"--cpus",
|
||||
str(self.default_cpus),
|
||||
"--memory",
|
||||
f"{self.default_memory_mb}m",
|
||||
"--read-only",
|
||||
"--tmpfs",
|
||||
"/tmp",
|
||||
"--tmpfs",
|
||||
"/home/app",
|
||||
"-v",
|
||||
f"{work_dir}:{SANDBOX_WORKSPACE_PATH}",
|
||||
"-w",
|
||||
SANDBOX_WORKSPACE_PATH,
|
||||
]
|
||||
)
|
||||
if publish_port is not None:
|
||||
assert_not_protected_port(int(publish_port))
|
||||
# Loopback publish only — never 0.0.0.0.
|
||||
args.extend(
|
||||
[
|
||||
"-p",
|
||||
f"127.0.0.1:{int(publish_port)}:{int(app_port or publish_port)}",
|
||||
]
|
||||
)
|
||||
args.extend(
|
||||
[
|
||||
"--label",
|
||||
f"{LABEL_ORG}={org}",
|
||||
"--label",
|
||||
f"{LABEL_PROJECT}={project}",
|
||||
"--label",
|
||||
f"{LABEL_ROLE}={role}",
|
||||
image or FALLBACK_IMAGE,
|
||||
"sh",
|
||||
"-lc",
|
||||
command or _KEEPALIVE,
|
||||
]
|
||||
)
|
||||
return await self._runner(
|
||||
[self.bin, *args],
|
||||
timeout_seconds=float(timeout_seconds),
|
||||
max_output_bytes=self.max_output_bytes,
|
||||
on_output=on_output,
|
||||
env={},
|
||||
)
|
||||
|
||||
async def start_demo_container(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -5,6 +5,126 @@ Use ``from app.core.database import Base`` and the helpers in
|
||||
``app.infrastructure.db.models`` here: that module imports this one (import cycle).
|
||||
Foreign keys to existing tables are written as plain strings, e.g.
|
||||
``ForeignKey("organizations.id", ondelete="CASCADE")``. See docs/plan/PLAN-v2.md §6.
|
||||
|
||||
Four tables carry the whole CI/CD story:
|
||||
|
||||
* ``pipeline_runs`` — one row per build. ``(project_id, status)`` is indexed
|
||||
because the worker's hot query is "the queued runs, oldest first".
|
||||
* ``pipeline_steps`` — the eight steps of a run, with a bounded log tail. The
|
||||
full output is streamed live and never stored; a build log is unbounded and a
|
||||
database is the wrong place for it.
|
||||
* ``releases`` — every sha that passed, so a demo rollback is a pointer move
|
||||
rather than a rebuild.
|
||||
* ``app_runtimes`` — the demo container a project currently has. One row per
|
||||
project (it is the primary key), because D5 gives each app exactly one demo
|
||||
container; blue/green is a temporary second container, not a second row.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.core.database import Base
|
||||
from app.domain.base import utcnow
|
||||
from app.infrastructure.db.columns import created, pk
|
||||
|
||||
#: 64 KB, matching the plan's cap on a stored step log tail.
|
||||
MAX_LOG_TAIL_CHARS = 65_536
|
||||
|
||||
|
||||
class PipelineRunRow(Base):
|
||||
__tablename__ = "pipeline_runs"
|
||||
__table_args__ = (
|
||||
Index("ix_pipeline_run_project_status", "project_id", "status"),
|
||||
Index("ix_pipeline_run_org_created", "organization_id", "created_at"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = pk()
|
||||
organization_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
project_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("projects.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
trigger: Mapped[str] = mapped_column(String(16), default="engine", nullable=False)
|
||||
branch: Mapped[str] = mapped_column(String(120), default="dev", nullable=False)
|
||||
sha: Mapped[str] = mapped_column(String(64), default="", nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(16), default="queued", nullable=False)
|
||||
error_summary: Mapped[str] = mapped_column(Text, default="", nullable=False)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
created_at: Mapped[datetime] = created()
|
||||
|
||||
|
||||
class PipelineStepRow(Base):
|
||||
__tablename__ = "pipeline_steps"
|
||||
__table_args__ = (Index("ix_pipeline_step_run_index", "run_id", "index"),)
|
||||
|
||||
id: Mapped[str] = pk()
|
||||
run_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("pipeline_runs.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
index: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(48), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(16), default="pending", nullable=False)
|
||||
exit_code: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
log_tail: Mapped[str] = mapped_column(Text, default="", nullable=False)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class ReleaseRow(Base):
|
||||
__tablename__ = "releases"
|
||||
__table_args__ = (Index("ix_release_project_created", "project_id", "created_at"),)
|
||||
|
||||
id: Mapped[str] = pk()
|
||||
organization_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
project_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("projects.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
run_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("pipeline_runs.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
sha: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
artifact_path: Mapped[str] = mapped_column(String(512), default="", nullable=False)
|
||||
#: Exactly one release per project is live; the switch is a transaction.
|
||||
is_demo_live: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
|
||||
created_at: Mapped[datetime] = created()
|
||||
|
||||
|
||||
class AppRuntimeRow(Base):
|
||||
"""The demo container for one project (one row per project, by design)."""
|
||||
|
||||
__tablename__ = "app_runtimes"
|
||||
__table_args__ = (Index("ix_app_runtime_org", "organization_id"),)
|
||||
|
||||
project_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("projects.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
organization_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
demo_container_id: Mapped[str | None] = mapped_column(String(160), nullable=True)
|
||||
host_port: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
demo_host: Mapped[str] = mapped_column(String(255), default="", nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(16), default="provisioning", nullable=False)
|
||||
current_release_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("releases.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AppRuntimeRow",
|
||||
"MAX_LOG_TAIL_CHARS",
|
||||
"PipelineRunRow",
|
||||
"PipelineStepRow",
|
||||
"ReleaseRow",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,450 @@
|
||||
"""Repositories for CI/CD runs, their steps, releases and the demo runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.domain.base import utcnow
|
||||
from app.domain.delivery.models import (
|
||||
AppRuntime,
|
||||
PipelineRun,
|
||||
PipelineStep,
|
||||
Release,
|
||||
RunStatus,
|
||||
StepStatus,
|
||||
)
|
||||
from app.infrastructure.db.models_delivery import (
|
||||
AppRuntimeRow,
|
||||
PipelineRunRow,
|
||||
PipelineStepRow,
|
||||
ReleaseRow,
|
||||
)
|
||||
|
||||
#: Statuses that count as "in flight" for the one-active-run-per-project rule.
|
||||
ACTIVE_STATUSES = (RunStatus.QUEUED.value, RunStatus.RUNNING.value)
|
||||
|
||||
|
||||
def _run(row: PipelineRunRow) -> PipelineRun:
|
||||
return PipelineRun(
|
||||
id=row.id,
|
||||
organization_id=row.organization_id,
|
||||
project_id=row.project_id,
|
||||
branch=row.branch,
|
||||
sha=row.sha,
|
||||
trigger=row.trigger,
|
||||
status=row.status,
|
||||
error_summary=row.error_summary,
|
||||
started_at=row.started_at,
|
||||
finished_at=row.finished_at,
|
||||
created_at=row.created_at,
|
||||
)
|
||||
|
||||
|
||||
def _step(row: PipelineStepRow) -> PipelineStep:
|
||||
return PipelineStep(
|
||||
id=row.id,
|
||||
run_id=row.run_id,
|
||||
index=row.index,
|
||||
name=row.name,
|
||||
status=row.status,
|
||||
exit_code=row.exit_code,
|
||||
log_tail=row.log_tail,
|
||||
started_at=row.started_at,
|
||||
finished_at=row.finished_at,
|
||||
)
|
||||
|
||||
|
||||
def _release(row: ReleaseRow) -> Release:
|
||||
return Release(
|
||||
id=row.id,
|
||||
organization_id=row.organization_id,
|
||||
project_id=row.project_id,
|
||||
run_id=row.run_id,
|
||||
sha=row.sha,
|
||||
artifact_path=row.artifact_path,
|
||||
is_demo_live=row.is_demo_live,
|
||||
created_at=row.created_at,
|
||||
)
|
||||
|
||||
|
||||
def _runtime(row: AppRuntimeRow) -> AppRuntime:
|
||||
return AppRuntime(
|
||||
project_id=row.project_id,
|
||||
organization_id=row.organization_id,
|
||||
demo_container_id=row.demo_container_id,
|
||||
host_port=row.host_port,
|
||||
demo_host=row.demo_host,
|
||||
status=row.status,
|
||||
current_release_id=row.current_release_id,
|
||||
updated_at=row.updated_at,
|
||||
)
|
||||
|
||||
|
||||
class PipelineRunRepository:
|
||||
def __init__(self, session: AsyncSession) -> None:
|
||||
self.session = session
|
||||
|
||||
async def get(self, run_id: str) -> PipelineRun | None:
|
||||
row = await self.session.get(PipelineRunRow, run_id)
|
||||
return _run(row) if row else None
|
||||
|
||||
async def get_for_org(self, organization_id: str, run_id: str) -> PipelineRun | None:
|
||||
"""Tenant-scoped lookup: a foreign run id is indistinguishable from a missing one."""
|
||||
row = await self.session.scalar(
|
||||
select(PipelineRunRow).where(
|
||||
PipelineRunRow.id == run_id,
|
||||
PipelineRunRow.organization_id == organization_id,
|
||||
)
|
||||
)
|
||||
return _run(row) if row else None
|
||||
|
||||
async def list_for_project(
|
||||
self, *, organization_id: str, project_id: str, limit: int = 50, offset: int = 0
|
||||
) -> list[PipelineRun]:
|
||||
stmt = (
|
||||
select(PipelineRunRow)
|
||||
.where(
|
||||
PipelineRunRow.organization_id == organization_id,
|
||||
PipelineRunRow.project_id == project_id,
|
||||
)
|
||||
.order_by(PipelineRunRow.created_at.desc())
|
||||
.limit(min(max(limit, 1), 200))
|
||||
.offset(max(offset, 0))
|
||||
)
|
||||
return [_run(r) for r in (await self.session.execute(stmt)).scalars()]
|
||||
|
||||
async def active_for_project(self, project_id: str) -> PipelineRun | None:
|
||||
row = await self.session.scalar(
|
||||
select(PipelineRunRow)
|
||||
.where(
|
||||
PipelineRunRow.project_id == project_id,
|
||||
PipelineRunRow.status.in_(ACTIVE_STATUSES),
|
||||
)
|
||||
.order_by(PipelineRunRow.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return _run(row) if row else None
|
||||
|
||||
async def last_succeeded_sha(self, project_id: str) -> str | None:
|
||||
return await self.session.scalar(
|
||||
select(PipelineRunRow.sha)
|
||||
.where(
|
||||
PipelineRunRow.project_id == project_id,
|
||||
PipelineRunRow.status == RunStatus.SUCCEEDED.value,
|
||||
)
|
||||
.order_by(PipelineRunRow.finished_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
async def queue(self, run: PipelineRun) -> PipelineRun:
|
||||
row = PipelineRunRow(
|
||||
id=run.id,
|
||||
organization_id=run.organization_id,
|
||||
project_id=run.project_id,
|
||||
trigger=run.trigger,
|
||||
branch=run.branch,
|
||||
sha=run.sha,
|
||||
status=run.status,
|
||||
error_summary=run.error_summary,
|
||||
started_at=run.started_at,
|
||||
finished_at=run.finished_at,
|
||||
created_at=run.created_at,
|
||||
)
|
||||
self.session.add(row)
|
||||
await self.session.flush()
|
||||
return _run(row)
|
||||
|
||||
async def list_queued(self, *, limit: int = 5) -> list[PipelineRun]:
|
||||
"""Oldest queued run first — the worker's hot query."""
|
||||
stmt = (
|
||||
select(PipelineRunRow)
|
||||
.where(PipelineRunRow.status == RunStatus.QUEUED.value)
|
||||
.order_by(PipelineRunRow.created_at.asc())
|
||||
.limit(min(max(limit, 1), 50))
|
||||
)
|
||||
return [_run(r) for r in (await self.session.execute(stmt)).scalars()]
|
||||
|
||||
async def list_running(self, *, limit: int = 50) -> list[PipelineRun]:
|
||||
stmt = (
|
||||
select(PipelineRunRow)
|
||||
.where(PipelineRunRow.status == RunStatus.RUNNING.value)
|
||||
.order_by(PipelineRunRow.started_at.asc())
|
||||
.limit(min(max(limit, 1), 200))
|
||||
)
|
||||
return [_run(r) for r in (await self.session.execute(stmt)).scalars()]
|
||||
|
||||
async def supersede_queued(self, *, project_id: str, keep_id: str) -> int:
|
||||
"""Mark every *queued* run of a project except ``keep_id`` as superseded.
|
||||
|
||||
Only queued ones: a run that has already started is doing real work and
|
||||
cancelling it mid-flight would leave a half-built container behind.
|
||||
"""
|
||||
result = await self.session.execute(
|
||||
update(PipelineRunRow)
|
||||
.where(
|
||||
PipelineRunRow.project_id == project_id,
|
||||
PipelineRunRow.status == RunStatus.QUEUED.value,
|
||||
PipelineRunRow.id != keep_id,
|
||||
)
|
||||
.values(status=RunStatus.SUPERSEDED.value, finished_at=utcnow())
|
||||
)
|
||||
await self.session.flush()
|
||||
# `session.execute(update(...))` returns a CursorResult at runtime; the
|
||||
# typed signature is the generic Result, which has no `rowcount`.
|
||||
return int(getattr(result, "rowcount", 0) or 0)
|
||||
|
||||
async def claim(self, run_id: str) -> bool:
|
||||
"""Atomically move a *queued* run to running. True when we won the race.
|
||||
|
||||
A conditional UPDATE rather than a read-then-write: two worker processes
|
||||
polling the same queue would otherwise both see `queued` and both start a
|
||||
build, publishing two containers to the same demo port.
|
||||
"""
|
||||
result = await self.session.execute(
|
||||
update(PipelineRunRow)
|
||||
.where(
|
||||
PipelineRunRow.id == run_id,
|
||||
PipelineRunRow.status == RunStatus.QUEUED.value,
|
||||
)
|
||||
.values(status=RunStatus.RUNNING.value, started_at=utcnow())
|
||||
)
|
||||
await self.session.flush()
|
||||
return int(getattr(result, "rowcount", 0) or 0) == 1
|
||||
|
||||
async def set_status(
|
||||
self,
|
||||
run_id: str,
|
||||
status: str,
|
||||
*,
|
||||
error_summary: str | None = None,
|
||||
started: bool = False,
|
||||
finished: bool = False,
|
||||
) -> None:
|
||||
row = await self.session.get(PipelineRunRow, run_id)
|
||||
if row is None:
|
||||
return
|
||||
row.status = status
|
||||
if error_summary is not None:
|
||||
row.error_summary = error_summary
|
||||
if started and row.started_at is None:
|
||||
row.started_at = utcnow()
|
||||
if finished:
|
||||
row.finished_at = utcnow()
|
||||
await self.session.flush()
|
||||
|
||||
async def count_by_status(self, *, organization_id: str) -> dict[str, int]:
|
||||
stmt = (
|
||||
select(PipelineRunRow.status, func.count())
|
||||
.where(PipelineRunRow.organization_id == organization_id)
|
||||
.group_by(PipelineRunRow.status)
|
||||
)
|
||||
return dict((await self.session.execute(stmt)).all())
|
||||
|
||||
|
||||
class PipelineStepRepository:
|
||||
def __init__(self, session: AsyncSession) -> None:
|
||||
self.session = session
|
||||
|
||||
async def list_for_run(self, run_id: str) -> list[PipelineStep]:
|
||||
stmt = (
|
||||
select(PipelineStepRow)
|
||||
.where(PipelineStepRow.run_id == run_id)
|
||||
.order_by(PipelineStepRow.index.asc())
|
||||
)
|
||||
return [_step(r) for r in (await self.session.execute(stmt)).scalars()]
|
||||
|
||||
async def create_many(self, steps: list[PipelineStep]) -> list[PipelineStep]:
|
||||
rows = [
|
||||
PipelineStepRow(
|
||||
id=step.id,
|
||||
run_id=step.run_id,
|
||||
index=step.index,
|
||||
name=step.name,
|
||||
status=step.status,
|
||||
exit_code=step.exit_code,
|
||||
log_tail=step.log_tail,
|
||||
started_at=step.started_at,
|
||||
finished_at=step.finished_at,
|
||||
)
|
||||
for step in steps
|
||||
]
|
||||
self.session.add_all(rows)
|
||||
await self.session.flush()
|
||||
return [_step(r) for r in rows]
|
||||
|
||||
async def mark(
|
||||
self,
|
||||
step_id: str,
|
||||
status: str,
|
||||
*,
|
||||
exit_code: int | None = None,
|
||||
log_tail: str | None = None,
|
||||
started: bool = False,
|
||||
finished: bool = False,
|
||||
) -> None:
|
||||
row = await self.session.get(PipelineStepRow, step_id)
|
||||
if row is None:
|
||||
return
|
||||
row.status = status
|
||||
if exit_code is not None:
|
||||
row.exit_code = exit_code
|
||||
if log_tail is not None:
|
||||
row.log_tail = log_tail
|
||||
if started and row.started_at is None:
|
||||
row.started_at = utcnow()
|
||||
if finished:
|
||||
row.finished_at = utcnow()
|
||||
await self.session.flush()
|
||||
|
||||
|
||||
class ReleaseRepository:
|
||||
def __init__(self, session: AsyncSession) -> None:
|
||||
self.session = session
|
||||
|
||||
async def get(self, release_id: str) -> Release | None:
|
||||
row = await self.session.get(ReleaseRow, release_id)
|
||||
return _release(row) if row else None
|
||||
|
||||
async def get_for_org(self, organization_id: str, release_id: str) -> Release | None:
|
||||
row = await self.session.scalar(
|
||||
select(ReleaseRow).where(
|
||||
ReleaseRow.id == release_id,
|
||||
ReleaseRow.organization_id == organization_id,
|
||||
)
|
||||
)
|
||||
return _release(row) if row else None
|
||||
|
||||
async def live_for_project(self, project_id: str) -> Release | None:
|
||||
row = await self.session.scalar(
|
||||
select(ReleaseRow).where(
|
||||
ReleaseRow.project_id == project_id, ReleaseRow.is_demo_live.is_(True)
|
||||
)
|
||||
)
|
||||
return _release(row) if row else None
|
||||
|
||||
async def list_for_project(
|
||||
self, *, project_id: str, limit: int = 20
|
||||
) -> list[Release]:
|
||||
stmt = (
|
||||
select(ReleaseRow)
|
||||
.where(ReleaseRow.project_id == project_id)
|
||||
.order_by(ReleaseRow.created_at.desc())
|
||||
.limit(min(max(limit, 1), 100))
|
||||
)
|
||||
return [_release(r) for r in (await self.session.execute(stmt)).scalars()]
|
||||
|
||||
async def create(self, release: Release) -> Release:
|
||||
row = ReleaseRow(
|
||||
id=release.id,
|
||||
organization_id=release.organization_id,
|
||||
project_id=release.project_id,
|
||||
run_id=release.run_id,
|
||||
sha=release.sha,
|
||||
artifact_path=release.artifact_path,
|
||||
is_demo_live=release.is_demo_live,
|
||||
created_at=release.created_at,
|
||||
)
|
||||
self.session.add(row)
|
||||
await self.session.flush()
|
||||
return _release(row)
|
||||
|
||||
async def mark_live(self, *, project_id: str, release_id: str) -> None:
|
||||
"""Exactly one live release per project, switched in one transaction."""
|
||||
await self.session.execute(
|
||||
update(ReleaseRow)
|
||||
.where(ReleaseRow.project_id == project_id, ReleaseRow.id != release_id)
|
||||
.values(is_demo_live=False)
|
||||
)
|
||||
await self.session.execute(
|
||||
update(ReleaseRow)
|
||||
.where(ReleaseRow.id == release_id)
|
||||
.values(is_demo_live=True)
|
||||
)
|
||||
await self.session.flush()
|
||||
|
||||
|
||||
class AppRuntimeRepository:
|
||||
def __init__(self, session: AsyncSession) -> None:
|
||||
self.session = session
|
||||
|
||||
async def get(self, project_id: str) -> AppRuntime | None:
|
||||
row = await self.session.get(AppRuntimeRow, project_id)
|
||||
return _runtime(row) if row else None
|
||||
|
||||
async def get_for_org(self, organization_id: str, project_id: str) -> AppRuntime | None:
|
||||
row = await self.session.scalar(
|
||||
select(AppRuntimeRow).where(
|
||||
AppRuntimeRow.project_id == project_id,
|
||||
AppRuntimeRow.organization_id == organization_id,
|
||||
)
|
||||
)
|
||||
return _runtime(row) if row else None
|
||||
|
||||
async def list_for_org(self, organization_id: str) -> list[AppRuntime]:
|
||||
stmt = select(AppRuntimeRow).where(
|
||||
AppRuntimeRow.organization_id == organization_id
|
||||
)
|
||||
return [_runtime(r) for r in (await self.session.execute(stmt)).scalars()]
|
||||
|
||||
async def list_all(self, *, limit: int = 500) -> list[AppRuntime]:
|
||||
"""Every app runtime, for the delivery worker's remote poll.
|
||||
|
||||
Deliberately not tenant-scoped: the poll is platform housekeeping and is
|
||||
never reachable from a tenant-scoped route.
|
||||
"""
|
||||
stmt = select(AppRuntimeRow).limit(min(max(limit, 1), 2_000))
|
||||
return [_runtime(r) for r in (await self.session.execute(stmt)).scalars()]
|
||||
|
||||
async def upsert(self, runtime: AppRuntime) -> AppRuntime:
|
||||
row = await self.session.get(AppRuntimeRow, runtime.project_id)
|
||||
if row is None:
|
||||
row = AppRuntimeRow(
|
||||
project_id=runtime.project_id, organization_id=runtime.organization_id
|
||||
)
|
||||
self.session.add(row)
|
||||
row.demo_container_id = runtime.demo_container_id
|
||||
row.host_port = runtime.host_port
|
||||
row.demo_host = runtime.demo_host
|
||||
row.status = runtime.status
|
||||
row.current_release_id = runtime.current_release_id
|
||||
row.updated_at = utcnow()
|
||||
await self.session.flush()
|
||||
return _runtime(row)
|
||||
|
||||
async def set_status(self, project_id: str, status: str) -> None:
|
||||
row = await self.session.get(AppRuntimeRow, project_id)
|
||||
if row is not None:
|
||||
row.status = status
|
||||
row.updated_at = utcnow()
|
||||
await self.session.flush()
|
||||
|
||||
async def set_live(
|
||||
self,
|
||||
project_id: str,
|
||||
*,
|
||||
container_id: str | None,
|
||||
host_port: int | None,
|
||||
release_id: str | None,
|
||||
status: str = "live",
|
||||
) -> None:
|
||||
"""Commit the blue/green switch: container, port and release together."""
|
||||
row = await self.session.get(AppRuntimeRow, project_id)
|
||||
if row is None:
|
||||
return
|
||||
row.demo_container_id = container_id
|
||||
row.host_port = host_port
|
||||
row.current_release_id = release_id
|
||||
row.status = status
|
||||
row.updated_at = utcnow()
|
||||
await self.session.flush()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ACTIVE_STATUSES",
|
||||
"AppRuntimeRepository",
|
||||
"PipelineRunRepository",
|
||||
"PipelineStepRepository",
|
||||
"ReleaseRepository",
|
||||
"StepStatus",
|
||||
]
|
||||
@@ -236,6 +236,42 @@ class PreviewDomainRepository:
|
||||
)
|
||||
await self.session.flush()
|
||||
|
||||
async def set_port(
|
||||
self, *, project_id: str, service_name: str, port: int
|
||||
) -> None:
|
||||
"""Re-point an existing service to a new port.
|
||||
|
||||
The blue/green switch uses this: the *service name* (`demo`) always means
|
||||
"the live port", so a deploy rewrites it rather than accumulating one
|
||||
ledger row per release.
|
||||
"""
|
||||
row = (
|
||||
await self.session.execute(
|
||||
select(PreviewPortRow).where(
|
||||
PreviewPortRow.project_id == project_id,
|
||||
PreviewPortRow.service_name == service_name,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
return
|
||||
row.port = port
|
||||
await self.session.flush()
|
||||
|
||||
async def delete_port(self, *, project_id: str, service_name: str) -> None:
|
||||
"""Release a transient port reservation (a staging port, say)."""
|
||||
row = (
|
||||
await self.session.execute(
|
||||
select(PreviewPortRow).where(
|
||||
PreviewPortRow.project_id == project_id,
|
||||
PreviewPortRow.service_name == service_name,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is not None:
|
||||
await self.session.delete(row)
|
||||
await self.session.flush()
|
||||
|
||||
# -- domains -------------------------------------------------------------
|
||||
|
||||
async def get_by_host(self, host: str) -> dict | None:
|
||||
|
||||
@@ -195,3 +195,12 @@ async def get_sandbox_service(
|
||||
from app.application.container import build_sandbox_service
|
||||
|
||||
return build_sandbox_service(session)
|
||||
|
||||
|
||||
async def get_delivery_service(
|
||||
session: AsyncSession = Depends(get_session),
|
||||
):
|
||||
"""Build the CI/CD service with the configured workspace, router and runtime."""
|
||||
from app.application.container import build_delivery_service
|
||||
|
||||
return build_delivery_service(session)
|
||||
|
||||
@@ -2,10 +2,249 @@
|
||||
|
||||
Owned by Agent 3 (see docs/plan/PLAN-v2.md §6). Registered in ``rpc_app.py`` already; give each
|
||||
route its full path (no prefix is applied here besides ``/api``).
|
||||
|
||||
**Who may call what.** Reading a build — its steps, its logs, the demo URL — is
|
||||
an ordinary thing for a member of the organization to do, so the GET routes need
|
||||
only membership. Everything that *causes* work (re-run, cancel, rollback) or
|
||||
touches production requires `projects.manage`, which is org admin and above. The
|
||||
production deploy endpoint (Phase 5) additionally demands the org slug typed back
|
||||
as a confirmation, because it is the one irreversible action in the product.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter
|
||||
from datetime import datetime
|
||||
|
||||
router = APIRouter(tags=["delivery"])
|
||||
from fastapi import APIRouter, Depends, Query, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.application.services.audit_service import record_audit
|
||||
from app.application.services.delivery_service import DeliveryService
|
||||
from app.core.database import get_session
|
||||
from app.domain.identity.models import CAN_MANAGE_PROJECTS
|
||||
from app.interfaces.http.deps import (
|
||||
OrgContext,
|
||||
client_meta,
|
||||
get_delivery_service,
|
||||
get_org_context,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/organizations/{org_id}/delivery", tags=["delivery"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PipelineStepOut(BaseModel):
|
||||
id: str
|
||||
index: int
|
||||
name: str
|
||||
status: str
|
||||
exit_code: int | None
|
||||
log_tail: str
|
||||
duration_ms: int
|
||||
|
||||
|
||||
class PipelineRunOut(BaseModel):
|
||||
id: str
|
||||
project_id: str
|
||||
branch: str
|
||||
sha: str
|
||||
trigger: str
|
||||
status: str
|
||||
error_summary: str
|
||||
started_at: datetime | None
|
||||
finished_at: datetime | None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class PipelineRunDetailOut(PipelineRunOut):
|
||||
steps: list[PipelineStepOut]
|
||||
|
||||
|
||||
class ReleaseOut(BaseModel):
|
||||
id: str
|
||||
project_id: str
|
||||
run_id: str
|
||||
sha: str
|
||||
is_demo_live: bool
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class RuntimeOut(BaseModel):
|
||||
project_id: str
|
||||
demo_host: str
|
||||
demo_url: str
|
||||
host_port: int | None
|
||||
status: str
|
||||
current_release_id: str | None
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class ManualRunRequest(BaseModel):
|
||||
project_id: str = Field(min_length=1, max_length=32)
|
||||
|
||||
|
||||
class RollbackRequest(BaseModel):
|
||||
project_id: str = Field(min_length=1, max_length=32)
|
||||
release_id: str | None = Field(default=None, max_length=32)
|
||||
|
||||
|
||||
def _run_out(run) -> PipelineRunOut: # noqa: ANN001 - a domain dataclass
|
||||
return PipelineRunOut(
|
||||
id=run.id, project_id=run.project_id, branch=run.branch, sha=run.sha,
|
||||
trigger=run.trigger, status=run.status, error_summary=run.error_summary,
|
||||
started_at=run.started_at, finished_at=run.finished_at, created_at=run.created_at,
|
||||
)
|
||||
|
||||
|
||||
def _release_out(release) -> ReleaseOut: # noqa: ANN001
|
||||
return ReleaseOut(
|
||||
id=release.id, project_id=release.project_id, run_id=release.run_id,
|
||||
sha=release.sha, is_demo_live=release.is_demo_live, created_at=release.created_at,
|
||||
)
|
||||
|
||||
|
||||
def _runtime_out(runtime) -> RuntimeOut: # noqa: ANN001
|
||||
return RuntimeOut(
|
||||
project_id=runtime.project_id, demo_host=runtime.demo_host,
|
||||
demo_url=runtime.demo_url, host_port=runtime.host_port, status=runtime.status,
|
||||
current_release_id=runtime.current_release_id, updated_at=runtime.updated_at,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Runs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/runs", response_model=list[PipelineRunOut])
|
||||
async def list_runs(
|
||||
project_id: str = Query(min_length=1),
|
||||
context: OrgContext = Depends(get_org_context),
|
||||
service: DeliveryService = Depends(get_delivery_service),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
) -> list[PipelineRunOut]:
|
||||
runs = await service.list_runs(
|
||||
organization_id=context.org_id, project_id=project_id, limit=limit, offset=offset
|
||||
)
|
||||
return [_run_out(r) for r in runs]
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}", response_model=PipelineRunDetailOut)
|
||||
async def get_run(
|
||||
run_id: str,
|
||||
context: OrgContext = Depends(get_org_context),
|
||||
service: DeliveryService = Depends(get_delivery_service),
|
||||
) -> PipelineRunDetailOut:
|
||||
run, steps = await service.get_run_detail(organization_id=context.org_id, run_id=run_id)
|
||||
return PipelineRunDetailOut(
|
||||
**_run_out(run).model_dump(),
|
||||
steps=[
|
||||
PipelineStepOut(
|
||||
id=s.id, index=s.index, name=s.name, status=s.status,
|
||||
exit_code=s.exit_code, log_tail=s.log_tail, duration_ms=s.duration_ms,
|
||||
)
|
||||
for s in steps
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@router.post("/runs", response_model=PipelineRunOut, status_code=status.HTTP_201_CREATED)
|
||||
async def start_run(
|
||||
payload: ManualRunRequest,
|
||||
request: Request,
|
||||
context: OrgContext = Depends(get_org_context),
|
||||
service: DeliveryService = Depends(get_delivery_service),
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> PipelineRunOut:
|
||||
"""Re-run the current `dev` head. Never a production deploy."""
|
||||
context.require(CAN_MANAGE_PROJECTS)
|
||||
run = await service.request_manual_run(
|
||||
organization_id=context.org_id, project_id=payload.project_id
|
||||
)
|
||||
ip, ua = client_meta(request)
|
||||
await record_audit(
|
||||
session, action="delivery.run_started", organization_id=context.org_id,
|
||||
actor_user_id=context.user.id, target_type="pipeline_run", target_id=run.id,
|
||||
metadata={"project_id": run.project_id, "sha": run.sha, "trigger": run.trigger},
|
||||
ip=ip, user_agent=ua,
|
||||
)
|
||||
return _run_out(run)
|
||||
|
||||
|
||||
@router.post("/runs/{run_id}/cancel", response_model=PipelineRunOut)
|
||||
async def cancel_run(
|
||||
run_id: str,
|
||||
request: Request,
|
||||
context: OrgContext = Depends(get_org_context),
|
||||
service: DeliveryService = Depends(get_delivery_service),
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> PipelineRunOut:
|
||||
context.require(CAN_MANAGE_PROJECTS)
|
||||
run = await service.cancel_run(organization_id=context.org_id, run_id=run_id)
|
||||
ip, ua = client_meta(request)
|
||||
await record_audit(
|
||||
session, action="delivery.run_cancelled", organization_id=context.org_id,
|
||||
actor_user_id=context.user.id, target_type="pipeline_run", target_id=run.id,
|
||||
ip=ip, user_agent=ua,
|
||||
)
|
||||
return _run_out(run)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Releases & runtime
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/releases", response_model=list[ReleaseOut])
|
||||
async def list_releases(
|
||||
project_id: str = Query(min_length=1),
|
||||
context: OrgContext = Depends(get_org_context),
|
||||
service: DeliveryService = Depends(get_delivery_service),
|
||||
) -> list[ReleaseOut]:
|
||||
releases = await service.list_releases(
|
||||
organization_id=context.org_id, project_id=project_id
|
||||
)
|
||||
return [_release_out(r) for r in releases]
|
||||
|
||||
|
||||
@router.get("/runtime", response_model=RuntimeOut)
|
||||
async def get_runtime(
|
||||
project_id: str = Query(min_length=1),
|
||||
context: OrgContext = Depends(get_org_context),
|
||||
service: DeliveryService = Depends(get_delivery_service),
|
||||
) -> RuntimeOut:
|
||||
runtime = await service.runtime_status(
|
||||
organization_id=context.org_id, project_id=project_id
|
||||
)
|
||||
return _runtime_out(runtime)
|
||||
|
||||
|
||||
@router.post("/releases/rollback", response_model=ReleaseOut)
|
||||
async def rollback_demo(
|
||||
payload: RollbackRequest,
|
||||
request: Request,
|
||||
context: OrgContext = Depends(get_org_context),
|
||||
service: DeliveryService = Depends(get_delivery_service),
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> ReleaseOut:
|
||||
"""Point the demo host back at an earlier release (admin+, audited)."""
|
||||
context.require(CAN_MANAGE_PROJECTS)
|
||||
release = await service.rollback_demo(
|
||||
organization_id=context.org_id,
|
||||
project_id=payload.project_id,
|
||||
release_id=payload.release_id,
|
||||
)
|
||||
ip, ua = client_meta(request)
|
||||
await record_audit(
|
||||
session, action="delivery.demo_rollback", organization_id=context.org_id,
|
||||
actor_user_id=context.user.id, target_type="release", target_id=release.id,
|
||||
metadata={"project_id": release.project_id, "sha": release.sha},
|
||||
ip=ip, user_agent=ua,
|
||||
)
|
||||
return _release_out(release)
|
||||
|
||||
@@ -1,15 +1,135 @@
|
||||
"""Delivery worker — executes queued CI/CD pipeline runs and polls remotes for pushes.
|
||||
"""Delivery worker — executes queued CI/CD runs and polls remotes for pushes.
|
||||
|
||||
Owned by Agent 3 (see docs/plan/PLAN-v2.md). Foundation stub: one no-op pass.
|
||||
Owned by Agent 3 (see docs/plan/PLAN-v2.md). Two jobs per pass:
|
||||
|
||||
1. **Drain the queue.** Every queued run, oldest first, executed to completion.
|
||||
A run is claimed by moving it to `running` before any work happens, so two
|
||||
worker processes never execute the same build.
|
||||
2. **Poll the remotes.** A push made *outside* the platform (a user pushing to
|
||||
their own git, or to our Gitea through its web UI) is invisible to the engine,
|
||||
which only reports pushes it makes itself. Comparing the remote `dev` head
|
||||
against the last successful run's sha is what closes that gap.
|
||||
|
||||
Polling is gated by `DELIVERY_POLL_SECONDS` rather than running on every pass:
|
||||
`git ls-remote` against every project on every tick is a lot of network for a
|
||||
signal that changes rarely.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.workers.base import WorkerOutcome
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.errors import AppError
|
||||
from app.infrastructure.error_log import record_error
|
||||
from app.workers.base import WorkerOutcome, worker_session
|
||||
|
||||
logger = logging.getLogger("agy.delivery.worker")
|
||||
|
||||
|
||||
class DeliveryWorker:
|
||||
name = "delivery"
|
||||
|
||||
async def run_pass(self, *, limit: int = 10) -> WorkerOutcome:
|
||||
return WorkerOutcome(name=self.name)
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
poll_seconds: int | None = None,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
) -> None:
|
||||
self.poll_seconds = int(
|
||||
settings.delivery_poll_seconds if poll_seconds is None else poll_seconds
|
||||
)
|
||||
self._clock = clock
|
||||
self._last_poll: float | None = None
|
||||
|
||||
async def run_pass(self, *, limit: int = 10, poll: bool = True) -> WorkerOutcome:
|
||||
"""One pass: drain the queue, then (maybe) poll the remotes."""
|
||||
outcome = WorkerOutcome(name=self.name)
|
||||
|
||||
from app.application.container import build_delivery_service
|
||||
|
||||
async with worker_session() as session:
|
||||
service = build_delivery_service(session)
|
||||
queued = await service.runs.list_queued(limit=max(1, limit))
|
||||
if not queued:
|
||||
outcome.skipped += 1
|
||||
|
||||
for run in queued:
|
||||
try:
|
||||
finished = await service.execute_run(run.id)
|
||||
except Exception as exc: # noqa: BLE001 - one bad run must not stop the pass
|
||||
logger.exception("delivery: run %s raised", run.id)
|
||||
await record_error(
|
||||
kind="delivery_run_failed",
|
||||
message=str(exc),
|
||||
exc=exc,
|
||||
organization_id=run.organization_id,
|
||||
metadata={"run_id": run.id, "worker": self.name},
|
||||
)
|
||||
outcome.failed += 1
|
||||
continue
|
||||
outcome.processed += 1
|
||||
outcome.note(f"run {finished.id} -> {finished.status}")
|
||||
|
||||
if poll and self._poll_due():
|
||||
polled = await self._poll_remotes(limit=limit)
|
||||
outcome.processed += polled
|
||||
if polled:
|
||||
outcome.note(f"queued {polled} run(s) from remote pushes")
|
||||
|
||||
return outcome
|
||||
|
||||
def _poll_due(self) -> bool:
|
||||
now = self._clock()
|
||||
if self._last_poll is not None and (now - self._last_poll) < self.poll_seconds:
|
||||
return False
|
||||
self._last_poll = now
|
||||
return True
|
||||
|
||||
async def _poll_remotes(self, *, limit: int) -> int:
|
||||
"""Queue a run for every project whose remote `dev` moved without us."""
|
||||
from app.application.container import build_delivery_service
|
||||
|
||||
queued = 0
|
||||
try:
|
||||
async with worker_session() as session:
|
||||
service = build_delivery_service(session)
|
||||
runtimes = await service.runtimes.list_all(limit=limit * 20)
|
||||
for runtime in runtimes:
|
||||
if queued >= limit:
|
||||
break
|
||||
try:
|
||||
run = await service.poll_remote(
|
||||
organization_id=runtime.organization_id,
|
||||
project_id=runtime.project_id,
|
||||
)
|
||||
except AppError:
|
||||
# A project without a repo yet is normal, not an error.
|
||||
continue
|
||||
except Exception as exc: # noqa: BLE001 - keep polling the rest
|
||||
logger.warning(
|
||||
"delivery: poll failed for project %s: %s",
|
||||
runtime.project_id,
|
||||
exc,
|
||||
)
|
||||
await record_error(
|
||||
kind="delivery_poll_failed",
|
||||
message=str(exc),
|
||||
exc=exc,
|
||||
organization_id=runtime.organization_id,
|
||||
metadata={"project_id": runtime.project_id, "worker": self.name},
|
||||
)
|
||||
continue
|
||||
if run is not None:
|
||||
queued += 1
|
||||
logger.info(
|
||||
"delivery: poll queued run %s for %s", run.id, runtime.project_id
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - polling is best-effort housekeeping
|
||||
logger.warning("delivery: remote poll pass failed: %s", exc)
|
||||
return queued
|
||||
|
||||
|
||||
__all__ = ["DeliveryWorker"]
|
||||
|
||||
Reference in New Issue
Block a user