feat(delivery): live CI/CD pipeline + demo runtime (v2 phase A3-3)

A push to `dev` now becomes a live demo, and a failed build never takes the
demo down.

- Four tables (pipeline_runs, pipeline_steps, releases, app_runtimes) and their
  migration; alembic upgrade/downgrade/check all clean.
- DeliveryService implements DeliveryHooks: on_app_created provisions the demo
  port + host + first build (idempotently); on_branch_pushed queues a run for
  `dev` and supersedes older *queued* runs, leaving a running one alone.
- The 8-step pipeline. Blue/green is real: the replacement starts on a staging
  port, is healthchecked, and only then does the route move — then the previous
  container is stopped. A failure removes the replacement, releases its port and
  leaves the old container serving.
- The demo port ledger is the DATABASE, not the router adapters' in-memory maps:
  the API and the delivery worker are separate processes.
- Runs are claimed with a conditional UPDATE, so two workers never build the
  same run.
- Live logs are batched (1 event / 500 ms, 200 lines) and redacted as a whole
  tail — per-chunk redaction misses a secret split across pipe reads, and the
  *stored* tail leaked until this was fixed.
- delivery worker: drain the queue, then poll remotes for pushes made outside
  the engine.
- 7 routes; reads need membership, actions need projects.manage and are audited.

Tests: 415 API (+21, 1 docker-marked skipped); ruff, mypy, alembic check clean.

Report: docs/plan/reports/agent-3-phase-3.md

apps/api/app/workers/delivery.py

@@ -1,15 +1,135 @@
-"""Delivery worker — executes queued CI/CD pipeline runs and polls remotes for pushes.
+"""Delivery worker — executes queued CI/CD runs and polls remotes for pushes.
 
-Owned by Agent 3 (see docs/plan/PLAN-v2.md). Foundation stub: one no-op pass.
+Owned by Agent 3 (see docs/plan/PLAN-v2.md). Two jobs per pass:
+
+1. **Drain the queue.** Every queued run, oldest first, executed to completion.
+   A run is claimed by moving it to `running` before any work happens, so two
+   worker processes never execute the same build.
+2. **Poll the remotes.** A push made *outside* the platform (a user pushing to
+   their own git, or to our Gitea through its web UI) is invisible to the engine,
+   which only reports pushes it makes itself. Comparing the remote `dev` head
+   against the last successful run's sha is what closes that gap.
+
+Polling is gated by `DELIVERY_POLL_SECONDS` rather than running on every pass:
+`git ls-remote` against every project on every tick is a lot of network for a
+signal that changes rarely.
 """
 
 from __future__ import annotations
 
-from app.workers.base import WorkerOutcome
+import logging
+import time
+from collections.abc import Callable
+
+from app.core.config import settings
+from app.core.errors import AppError
+from app.infrastructure.error_log import record_error
+from app.workers.base import WorkerOutcome, worker_session
+
+logger = logging.getLogger("agy.delivery.worker")
 
 
 class DeliveryWorker:
     name = "delivery"
 
-    async def run_pass(self, *, limit: int = 10) -> WorkerOutcome:
-        return WorkerOutcome(name=self.name)
+    def __init__(
+        self,
+        *,
+        poll_seconds: int | None = None,
+        clock: Callable[[], float] = time.monotonic,
+    ) -> None:
+        self.poll_seconds = int(
+            settings.delivery_poll_seconds if poll_seconds is None else poll_seconds
+        )
+        self._clock = clock
+        self._last_poll: float | None = None
+
+    async def run_pass(self, *, limit: int = 10, poll: bool = True) -> WorkerOutcome:
+        """One pass: drain the queue, then (maybe) poll the remotes."""
+        outcome = WorkerOutcome(name=self.name)
+
+        from app.application.container import build_delivery_service
+
+        async with worker_session() as session:
+            service = build_delivery_service(session)
+            queued = await service.runs.list_queued(limit=max(1, limit))
+            if not queued:
+                outcome.skipped += 1
+
+            for run in queued:
+                try:
+                    finished = await service.execute_run(run.id)
+                except Exception as exc:  # noqa: BLE001 - one bad run must not stop the pass
+                    logger.exception("delivery: run %s raised", run.id)
+                    await record_error(
+                        kind="delivery_run_failed",
+                        message=str(exc),
+                        exc=exc,
+                        organization_id=run.organization_id,
+                        metadata={"run_id": run.id, "worker": self.name},
+                    )
+                    outcome.failed += 1
+                    continue
+                outcome.processed += 1
+                outcome.note(f"run {finished.id} -> {finished.status}")
+
+        if poll and self._poll_due():
+            polled = await self._poll_remotes(limit=limit)
+            outcome.processed += polled
+            if polled:
+                outcome.note(f"queued {polled} run(s) from remote pushes")
+
+        return outcome
+
+    def _poll_due(self) -> bool:
+        now = self._clock()
+        if self._last_poll is not None and (now - self._last_poll) < self.poll_seconds:
+            return False
+        self._last_poll = now
+        return True
+
+    async def _poll_remotes(self, *, limit: int) -> int:
+        """Queue a run for every project whose remote `dev` moved without us."""
+        from app.application.container import build_delivery_service
+
+        queued = 0
+        try:
+            async with worker_session() as session:
+                service = build_delivery_service(session)
+                runtimes = await service.runtimes.list_all(limit=limit * 20)
+                for runtime in runtimes:
+                    if queued >= limit:
+                        break
+                    try:
+                        run = await service.poll_remote(
+                            organization_id=runtime.organization_id,
+                            project_id=runtime.project_id,
+                        )
+                    except AppError:
+                        # A project without a repo yet is normal, not an error.
+                        continue
+                    except Exception as exc:  # noqa: BLE001 - keep polling the rest
+                        logger.warning(
+                            "delivery: poll failed for project %s: %s",
+                            runtime.project_id,
+                            exc,
+                        )
+                        await record_error(
+                            kind="delivery_poll_failed",
+                            message=str(exc),
+                            exc=exc,
+                            organization_id=runtime.organization_id,
+                            metadata={"project_id": runtime.project_id, "worker": self.name},
+                        )
+                        continue
+                    if run is not None:
+                        queued += 1
+                        logger.info(
+                            "delivery: poll queued run %s for %s", run.id, runtime.project_id
+                        )
+        except Exception as exc:  # noqa: BLE001 - polling is best-effort housekeeping
+            logger.warning("delivery: remote poll pass failed: %s", exc)
+        return queued
+
+
+__all__ = ["DeliveryWorker"]