feat(delivery): live CI/CD pipeline + demo runtime (v2 phase A3-3)
A push to `dev` now becomes a live demo, and a failed build never takes the demo down. - Four tables (pipeline_runs, pipeline_steps, releases, app_runtimes) and their migration; alembic upgrade/downgrade/check all clean. - DeliveryService implements DeliveryHooks: on_app_created provisions the demo port + host + first build (idempotently); on_branch_pushed queues a run for `dev` and supersedes older *queued* runs, leaving a running one alone. - The 8-step pipeline. Blue/green is real: the replacement starts on a staging port, is healthchecked, and only then does the route move — then the previous container is stopped. A failure removes the replacement, releases its port and leaves the old container serving. - The demo port ledger is the DATABASE, not the router adapters' in-memory maps: the API and the delivery worker are separate processes. - Runs are claimed with a conditional UPDATE, so two workers never build the same run. - Live logs are batched (1 event / 500 ms, 200 lines) and redacted as a whole tail — per-chunk redaction misses a secret split across pipe reads, and the *stored* tail leaked until this was fixed. - delivery worker: drain the queue, then poll remotes for pushes made outside the engine. - 7 routes; reads need membership, actions need projects.manage and are audited. Tests: 415 API (+21, 1 docker-marked skipped); ruff, mypy, alembic check clean. Report: docs/plan/reports/agent-3-phase-3.md
This commit is contained in:
mjalmoudy94
@@ -0,0 +1,450 @@
|
||||
"""Repositories for CI/CD runs, their steps, releases and the demo runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.domain.base import utcnow
|
||||
from app.domain.delivery.models import (
|
||||
AppRuntime,
|
||||
PipelineRun,
|
||||
PipelineStep,
|
||||
Release,
|
||||
RunStatus,
|
||||
StepStatus,
|
||||
)
|
||||
from app.infrastructure.db.models_delivery import (
|
||||
AppRuntimeRow,
|
||||
PipelineRunRow,
|
||||
PipelineStepRow,
|
||||
ReleaseRow,
|
||||
)
|
||||
|
||||
#: Statuses that count as "in flight" for the one-active-run-per-project rule.
|
||||
ACTIVE_STATUSES = (RunStatus.QUEUED.value, RunStatus.RUNNING.value)
|
||||
|
||||
|
||||
def _run(row: PipelineRunRow) -> PipelineRun:
|
||||
return PipelineRun(
|
||||
id=row.id,
|
||||
organization_id=row.organization_id,
|
||||
project_id=row.project_id,
|
||||
branch=row.branch,
|
||||
sha=row.sha,
|
||||
trigger=row.trigger,
|
||||
status=row.status,
|
||||
error_summary=row.error_summary,
|
||||
started_at=row.started_at,
|
||||
finished_at=row.finished_at,
|
||||
created_at=row.created_at,
|
||||
)
|
||||
|
||||
|
||||
def _step(row: PipelineStepRow) -> PipelineStep:
|
||||
return PipelineStep(
|
||||
id=row.id,
|
||||
run_id=row.run_id,
|
||||
index=row.index,
|
||||
name=row.name,
|
||||
status=row.status,
|
||||
exit_code=row.exit_code,
|
||||
log_tail=row.log_tail,
|
||||
started_at=row.started_at,
|
||||
finished_at=row.finished_at,
|
||||
)
|
||||
|
||||
|
||||
def _release(row: ReleaseRow) -> Release:
|
||||
return Release(
|
||||
id=row.id,
|
||||
organization_id=row.organization_id,
|
||||
project_id=row.project_id,
|
||||
run_id=row.run_id,
|
||||
sha=row.sha,
|
||||
artifact_path=row.artifact_path,
|
||||
is_demo_live=row.is_demo_live,
|
||||
created_at=row.created_at,
|
||||
)
|
||||
|
||||
|
||||
def _runtime(row: AppRuntimeRow) -> AppRuntime:
|
||||
return AppRuntime(
|
||||
project_id=row.project_id,
|
||||
organization_id=row.organization_id,
|
||||
demo_container_id=row.demo_container_id,
|
||||
host_port=row.host_port,
|
||||
demo_host=row.demo_host,
|
||||
status=row.status,
|
||||
current_release_id=row.current_release_id,
|
||||
updated_at=row.updated_at,
|
||||
)
|
||||
|
||||
|
||||
class PipelineRunRepository:
|
||||
def __init__(self, session: AsyncSession) -> None:
|
||||
self.session = session
|
||||
|
||||
async def get(self, run_id: str) -> PipelineRun | None:
|
||||
row = await self.session.get(PipelineRunRow, run_id)
|
||||
return _run(row) if row else None
|
||||
|
||||
async def get_for_org(self, organization_id: str, run_id: str) -> PipelineRun | None:
|
||||
"""Tenant-scoped lookup: a foreign run id is indistinguishable from a missing one."""
|
||||
row = await self.session.scalar(
|
||||
select(PipelineRunRow).where(
|
||||
PipelineRunRow.id == run_id,
|
||||
PipelineRunRow.organization_id == organization_id,
|
||||
)
|
||||
)
|
||||
return _run(row) if row else None
|
||||
|
||||
async def list_for_project(
|
||||
self, *, organization_id: str, project_id: str, limit: int = 50, offset: int = 0
|
||||
) -> list[PipelineRun]:
|
||||
stmt = (
|
||||
select(PipelineRunRow)
|
||||
.where(
|
||||
PipelineRunRow.organization_id == organization_id,
|
||||
PipelineRunRow.project_id == project_id,
|
||||
)
|
||||
.order_by(PipelineRunRow.created_at.desc())
|
||||
.limit(min(max(limit, 1), 200))
|
||||
.offset(max(offset, 0))
|
||||
)
|
||||
return [_run(r) for r in (await self.session.execute(stmt)).scalars()]
|
||||
|
||||
async def active_for_project(self, project_id: str) -> PipelineRun | None:
|
||||
row = await self.session.scalar(
|
||||
select(PipelineRunRow)
|
||||
.where(
|
||||
PipelineRunRow.project_id == project_id,
|
||||
PipelineRunRow.status.in_(ACTIVE_STATUSES),
|
||||
)
|
||||
.order_by(PipelineRunRow.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return _run(row) if row else None
|
||||
|
||||
async def last_succeeded_sha(self, project_id: str) -> str | None:
|
||||
return await self.session.scalar(
|
||||
select(PipelineRunRow.sha)
|
||||
.where(
|
||||
PipelineRunRow.project_id == project_id,
|
||||
PipelineRunRow.status == RunStatus.SUCCEEDED.value,
|
||||
)
|
||||
.order_by(PipelineRunRow.finished_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
async def queue(self, run: PipelineRun) -> PipelineRun:
|
||||
row = PipelineRunRow(
|
||||
id=run.id,
|
||||
organization_id=run.organization_id,
|
||||
project_id=run.project_id,
|
||||
trigger=run.trigger,
|
||||
branch=run.branch,
|
||||
sha=run.sha,
|
||||
status=run.status,
|
||||
error_summary=run.error_summary,
|
||||
started_at=run.started_at,
|
||||
finished_at=run.finished_at,
|
||||
created_at=run.created_at,
|
||||
)
|
||||
self.session.add(row)
|
||||
await self.session.flush()
|
||||
return _run(row)
|
||||
|
||||
async def list_queued(self, *, limit: int = 5) -> list[PipelineRun]:
|
||||
"""Oldest queued run first — the worker's hot query."""
|
||||
stmt = (
|
||||
select(PipelineRunRow)
|
||||
.where(PipelineRunRow.status == RunStatus.QUEUED.value)
|
||||
.order_by(PipelineRunRow.created_at.asc())
|
||||
.limit(min(max(limit, 1), 50))
|
||||
)
|
||||
return [_run(r) for r in (await self.session.execute(stmt)).scalars()]
|
||||
|
||||
async def list_running(self, *, limit: int = 50) -> list[PipelineRun]:
|
||||
stmt = (
|
||||
select(PipelineRunRow)
|
||||
.where(PipelineRunRow.status == RunStatus.RUNNING.value)
|
||||
.order_by(PipelineRunRow.started_at.asc())
|
||||
.limit(min(max(limit, 1), 200))
|
||||
)
|
||||
return [_run(r) for r in (await self.session.execute(stmt)).scalars()]
|
||||
|
||||
async def supersede_queued(self, *, project_id: str, keep_id: str) -> int:
|
||||
"""Mark every *queued* run of a project except ``keep_id`` as superseded.
|
||||
|
||||
Only queued ones: a run that has already started is doing real work and
|
||||
cancelling it mid-flight would leave a half-built container behind.
|
||||
"""
|
||||
result = await self.session.execute(
|
||||
update(PipelineRunRow)
|
||||
.where(
|
||||
PipelineRunRow.project_id == project_id,
|
||||
PipelineRunRow.status == RunStatus.QUEUED.value,
|
||||
PipelineRunRow.id != keep_id,
|
||||
)
|
||||
.values(status=RunStatus.SUPERSEDED.value, finished_at=utcnow())
|
||||
)
|
||||
await self.session.flush()
|
||||
# `session.execute(update(...))` returns a CursorResult at runtime; the
|
||||
# typed signature is the generic Result, which has no `rowcount`.
|
||||
return int(getattr(result, "rowcount", 0) or 0)
|
||||
|
||||
async def claim(self, run_id: str) -> bool:
|
||||
"""Atomically move a *queued* run to running. True when we won the race.
|
||||
|
||||
A conditional UPDATE rather than a read-then-write: two worker processes
|
||||
polling the same queue would otherwise both see `queued` and both start a
|
||||
build, publishing two containers to the same demo port.
|
||||
"""
|
||||
result = await self.session.execute(
|
||||
update(PipelineRunRow)
|
||||
.where(
|
||||
PipelineRunRow.id == run_id,
|
||||
PipelineRunRow.status == RunStatus.QUEUED.value,
|
||||
)
|
||||
.values(status=RunStatus.RUNNING.value, started_at=utcnow())
|
||||
)
|
||||
await self.session.flush()
|
||||
return int(getattr(result, "rowcount", 0) or 0) == 1
|
||||
|
||||
async def set_status(
|
||||
self,
|
||||
run_id: str,
|
||||
status: str,
|
||||
*,
|
||||
error_summary: str | None = None,
|
||||
started: bool = False,
|
||||
finished: bool = False,
|
||||
) -> None:
|
||||
row = await self.session.get(PipelineRunRow, run_id)
|
||||
if row is None:
|
||||
return
|
||||
row.status = status
|
||||
if error_summary is not None:
|
||||
row.error_summary = error_summary
|
||||
if started and row.started_at is None:
|
||||
row.started_at = utcnow()
|
||||
if finished:
|
||||
row.finished_at = utcnow()
|
||||
await self.session.flush()
|
||||
|
||||
async def count_by_status(self, *, organization_id: str) -> dict[str, int]:
|
||||
stmt = (
|
||||
select(PipelineRunRow.status, func.count())
|
||||
.where(PipelineRunRow.organization_id == organization_id)
|
||||
.group_by(PipelineRunRow.status)
|
||||
)
|
||||
return dict((await self.session.execute(stmt)).all())
|
||||
|
||||
|
||||
class PipelineStepRepository:
|
||||
def __init__(self, session: AsyncSession) -> None:
|
||||
self.session = session
|
||||
|
||||
async def list_for_run(self, run_id: str) -> list[PipelineStep]:
|
||||
stmt = (
|
||||
select(PipelineStepRow)
|
||||
.where(PipelineStepRow.run_id == run_id)
|
||||
.order_by(PipelineStepRow.index.asc())
|
||||
)
|
||||
return [_step(r) for r in (await self.session.execute(stmt)).scalars()]
|
||||
|
||||
async def create_many(self, steps: list[PipelineStep]) -> list[PipelineStep]:
|
||||
rows = [
|
||||
PipelineStepRow(
|
||||
id=step.id,
|
||||
run_id=step.run_id,
|
||||
index=step.index,
|
||||
name=step.name,
|
||||
status=step.status,
|
||||
exit_code=step.exit_code,
|
||||
log_tail=step.log_tail,
|
||||
started_at=step.started_at,
|
||||
finished_at=step.finished_at,
|
||||
)
|
||||
for step in steps
|
||||
]
|
||||
self.session.add_all(rows)
|
||||
await self.session.flush()
|
||||
return [_step(r) for r in rows]
|
||||
|
||||
async def mark(
|
||||
self,
|
||||
step_id: str,
|
||||
status: str,
|
||||
*,
|
||||
exit_code: int | None = None,
|
||||
log_tail: str | None = None,
|
||||
started: bool = False,
|
||||
finished: bool = False,
|
||||
) -> None:
|
||||
row = await self.session.get(PipelineStepRow, step_id)
|
||||
if row is None:
|
||||
return
|
||||
row.status = status
|
||||
if exit_code is not None:
|
||||
row.exit_code = exit_code
|
||||
if log_tail is not None:
|
||||
row.log_tail = log_tail
|
||||
if started and row.started_at is None:
|
||||
row.started_at = utcnow()
|
||||
if finished:
|
||||
row.finished_at = utcnow()
|
||||
await self.session.flush()
|
||||
|
||||
|
||||
class ReleaseRepository:
|
||||
def __init__(self, session: AsyncSession) -> None:
|
||||
self.session = session
|
||||
|
||||
async def get(self, release_id: str) -> Release | None:
|
||||
row = await self.session.get(ReleaseRow, release_id)
|
||||
return _release(row) if row else None
|
||||
|
||||
async def get_for_org(self, organization_id: str, release_id: str) -> Release | None:
|
||||
row = await self.session.scalar(
|
||||
select(ReleaseRow).where(
|
||||
ReleaseRow.id == release_id,
|
||||
ReleaseRow.organization_id == organization_id,
|
||||
)
|
||||
)
|
||||
return _release(row) if row else None
|
||||
|
||||
async def live_for_project(self, project_id: str) -> Release | None:
|
||||
row = await self.session.scalar(
|
||||
select(ReleaseRow).where(
|
||||
ReleaseRow.project_id == project_id, ReleaseRow.is_demo_live.is_(True)
|
||||
)
|
||||
)
|
||||
return _release(row) if row else None
|
||||
|
||||
async def list_for_project(
|
||||
self, *, project_id: str, limit: int = 20
|
||||
) -> list[Release]:
|
||||
stmt = (
|
||||
select(ReleaseRow)
|
||||
.where(ReleaseRow.project_id == project_id)
|
||||
.order_by(ReleaseRow.created_at.desc())
|
||||
.limit(min(max(limit, 1), 100))
|
||||
)
|
||||
return [_release(r) for r in (await self.session.execute(stmt)).scalars()]
|
||||
|
||||
async def create(self, release: Release) -> Release:
|
||||
row = ReleaseRow(
|
||||
id=release.id,
|
||||
organization_id=release.organization_id,
|
||||
project_id=release.project_id,
|
||||
run_id=release.run_id,
|
||||
sha=release.sha,
|
||||
artifact_path=release.artifact_path,
|
||||
is_demo_live=release.is_demo_live,
|
||||
created_at=release.created_at,
|
||||
)
|
||||
self.session.add(row)
|
||||
await self.session.flush()
|
||||
return _release(row)
|
||||
|
||||
async def mark_live(self, *, project_id: str, release_id: str) -> None:
|
||||
"""Exactly one live release per project, switched in one transaction."""
|
||||
await self.session.execute(
|
||||
update(ReleaseRow)
|
||||
.where(ReleaseRow.project_id == project_id, ReleaseRow.id != release_id)
|
||||
.values(is_demo_live=False)
|
||||
)
|
||||
await self.session.execute(
|
||||
update(ReleaseRow)
|
||||
.where(ReleaseRow.id == release_id)
|
||||
.values(is_demo_live=True)
|
||||
)
|
||||
await self.session.flush()
|
||||
|
||||
|
||||
class AppRuntimeRepository:
|
||||
def __init__(self, session: AsyncSession) -> None:
|
||||
self.session = session
|
||||
|
||||
async def get(self, project_id: str) -> AppRuntime | None:
|
||||
row = await self.session.get(AppRuntimeRow, project_id)
|
||||
return _runtime(row) if row else None
|
||||
|
||||
async def get_for_org(self, organization_id: str, project_id: str) -> AppRuntime | None:
|
||||
row = await self.session.scalar(
|
||||
select(AppRuntimeRow).where(
|
||||
AppRuntimeRow.project_id == project_id,
|
||||
AppRuntimeRow.organization_id == organization_id,
|
||||
)
|
||||
)
|
||||
return _runtime(row) if row else None
|
||||
|
||||
async def list_for_org(self, organization_id: str) -> list[AppRuntime]:
|
||||
stmt = select(AppRuntimeRow).where(
|
||||
AppRuntimeRow.organization_id == organization_id
|
||||
)
|
||||
return [_runtime(r) for r in (await self.session.execute(stmt)).scalars()]
|
||||
|
||||
async def list_all(self, *, limit: int = 500) -> list[AppRuntime]:
|
||||
"""Every app runtime, for the delivery worker's remote poll.
|
||||
|
||||
Deliberately not tenant-scoped: the poll is platform housekeeping and is
|
||||
never reachable from a tenant-scoped route.
|
||||
"""
|
||||
stmt = select(AppRuntimeRow).limit(min(max(limit, 1), 2_000))
|
||||
return [_runtime(r) for r in (await self.session.execute(stmt)).scalars()]
|
||||
|
||||
async def upsert(self, runtime: AppRuntime) -> AppRuntime:
|
||||
row = await self.session.get(AppRuntimeRow, runtime.project_id)
|
||||
if row is None:
|
||||
row = AppRuntimeRow(
|
||||
project_id=runtime.project_id, organization_id=runtime.organization_id
|
||||
)
|
||||
self.session.add(row)
|
||||
row.demo_container_id = runtime.demo_container_id
|
||||
row.host_port = runtime.host_port
|
||||
row.demo_host = runtime.demo_host
|
||||
row.status = runtime.status
|
||||
row.current_release_id = runtime.current_release_id
|
||||
row.updated_at = utcnow()
|
||||
await self.session.flush()
|
||||
return _runtime(row)
|
||||
|
||||
async def set_status(self, project_id: str, status: str) -> None:
|
||||
row = await self.session.get(AppRuntimeRow, project_id)
|
||||
if row is not None:
|
||||
row.status = status
|
||||
row.updated_at = utcnow()
|
||||
await self.session.flush()
|
||||
|
||||
async def set_live(
|
||||
self,
|
||||
project_id: str,
|
||||
*,
|
||||
container_id: str | None,
|
||||
host_port: int | None,
|
||||
release_id: str | None,
|
||||
status: str = "live",
|
||||
) -> None:
|
||||
"""Commit the blue/green switch: container, port and release together."""
|
||||
row = await self.session.get(AppRuntimeRow, project_id)
|
||||
if row is None:
|
||||
return
|
||||
row.demo_container_id = container_id
|
||||
row.host_port = host_port
|
||||
row.current_release_id = release_id
|
||||
row.status = status
|
||||
row.updated_at = utcnow()
|
||||
await self.session.flush()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ACTIVE_STATUSES",
|
||||
"AppRuntimeRepository",
|
||||
"PipelineRunRepository",
|
||||
"PipelineStepRepository",
|
||||
"ReleaseRepository",
|
||||
"StepStatus",
|
||||
]
|
||||
Reference in New Issue
Block a user