mjalmoudy94
a1b8a28dda
A push to `dev` now becomes a live demo, and a failed build never takes the demo down. - Four tables (pipeline_runs, pipeline_steps, releases, app_runtimes) and their migration; alembic upgrade/downgrade/check all clean. - DeliveryService implements DeliveryHooks: on_app_created provisions the demo port + host + first build (idempotently); on_branch_pushed queues a run for `dev` and supersedes older *queued* runs, leaving a running one alone. - The 8-step pipeline. Blue/green is real: the replacement starts on a staging port, is healthchecked, and only then does the route move — then the previous container is stopped. A failure removes the replacement, releases its port and leaves the old container serving. - The demo port ledger is the DATABASE, not the router adapters' in-memory maps: the API and the delivery worker are separate processes. - Runs are claimed with a conditional UPDATE, so two workers never build the same run. - Live logs are batched (1 event / 500 ms, 200 lines) and redacted as a whole tail — per-chunk redaction misses a secret split across pipe reads, and the *stored* tail leaked until this was fixed. - delivery worker: drain the queue, then poll remotes for pushes made outside the engine. - 7 routes; reads need membership, actions need projects.manage and are audited. Tests: 415 API (+21, 1 docker-marked skipped); ruff, mypy, alembic check clean. Report: docs/plan/reports/agent-3-phase-3.md
136 lines
5.3 KiB
Python
136 lines
5.3 KiB
Python
"""Delivery worker — executes queued CI/CD runs and polls remotes for pushes.
|
|
|
|
Owned by Agent 3 (see docs/plan/PLAN-v2.md). Two jobs per pass:
|
|
|
|
1. **Drain the queue.** Every queued run, oldest first, executed to completion.
|
|
A run is claimed by moving it to `running` before any work happens, so two
|
|
worker processes never execute the same build.
|
|
2. **Poll the remotes.** A push made *outside* the platform (a user pushing to
|
|
their own git, or to our Gitea through its web UI) is invisible to the engine,
|
|
which only reports pushes it makes itself. Comparing the remote `dev` head
|
|
against the last successful run's sha is what closes that gap.
|
|
|
|
Polling is gated by `DELIVERY_POLL_SECONDS` rather than running on every pass:
|
|
`git ls-remote` against every project on every tick is a lot of network for a
|
|
signal that changes rarely.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import time
|
|
from collections.abc import Callable
|
|
|
|
from app.core.config import settings
|
|
from app.core.errors import AppError
|
|
from app.infrastructure.error_log import record_error
|
|
from app.workers.base import WorkerOutcome, worker_session
|
|
|
|
logger = logging.getLogger("agy.delivery.worker")
|
|
|
|
|
|
class DeliveryWorker:
|
|
name = "delivery"
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
poll_seconds: int | None = None,
|
|
clock: Callable[[], float] = time.monotonic,
|
|
) -> None:
|
|
self.poll_seconds = int(
|
|
settings.delivery_poll_seconds if poll_seconds is None else poll_seconds
|
|
)
|
|
self._clock = clock
|
|
self._last_poll: float | None = None
|
|
|
|
async def run_pass(self, *, limit: int = 10, poll: bool = True) -> WorkerOutcome:
|
|
"""One pass: drain the queue, then (maybe) poll the remotes."""
|
|
outcome = WorkerOutcome(name=self.name)
|
|
|
|
from app.application.container import build_delivery_service
|
|
|
|
async with worker_session() as session:
|
|
service = build_delivery_service(session)
|
|
queued = await service.runs.list_queued(limit=max(1, limit))
|
|
if not queued:
|
|
outcome.skipped += 1
|
|
|
|
for run in queued:
|
|
try:
|
|
finished = await service.execute_run(run.id)
|
|
except Exception as exc: # noqa: BLE001 - one bad run must not stop the pass
|
|
logger.exception("delivery: run %s raised", run.id)
|
|
await record_error(
|
|
kind="delivery_run_failed",
|
|
message=str(exc),
|
|
exc=exc,
|
|
organization_id=run.organization_id,
|
|
metadata={"run_id": run.id, "worker": self.name},
|
|
)
|
|
outcome.failed += 1
|
|
continue
|
|
outcome.processed += 1
|
|
outcome.note(f"run {finished.id} -> {finished.status}")
|
|
|
|
if poll and self._poll_due():
|
|
polled = await self._poll_remotes(limit=limit)
|
|
outcome.processed += polled
|
|
if polled:
|
|
outcome.note(f"queued {polled} run(s) from remote pushes")
|
|
|
|
return outcome
|
|
|
|
def _poll_due(self) -> bool:
|
|
now = self._clock()
|
|
if self._last_poll is not None and (now - self._last_poll) < self.poll_seconds:
|
|
return False
|
|
self._last_poll = now
|
|
return True
|
|
|
|
async def _poll_remotes(self, *, limit: int) -> int:
|
|
"""Queue a run for every project whose remote `dev` moved without us."""
|
|
from app.application.container import build_delivery_service
|
|
|
|
queued = 0
|
|
try:
|
|
async with worker_session() as session:
|
|
service = build_delivery_service(session)
|
|
runtimes = await service.runtimes.list_all(limit=limit * 20)
|
|
for runtime in runtimes:
|
|
if queued >= limit:
|
|
break
|
|
try:
|
|
run = await service.poll_remote(
|
|
organization_id=runtime.organization_id,
|
|
project_id=runtime.project_id,
|
|
)
|
|
except AppError:
|
|
# A project without a repo yet is normal, not an error.
|
|
continue
|
|
except Exception as exc: # noqa: BLE001 - keep polling the rest
|
|
logger.warning(
|
|
"delivery: poll failed for project %s: %s",
|
|
runtime.project_id,
|
|
exc,
|
|
)
|
|
await record_error(
|
|
kind="delivery_poll_failed",
|
|
message=str(exc),
|
|
exc=exc,
|
|
organization_id=runtime.organization_id,
|
|
metadata={"project_id": runtime.project_id, "worker": self.name},
|
|
)
|
|
continue
|
|
if run is not None:
|
|
queued += 1
|
|
logger.info(
|
|
"delivery: poll queued run %s for %s", run.id, runtime.project_id
|
|
)
|
|
except Exception as exc: # noqa: BLE001 - polling is best-effort housekeeping
|
|
logger.warning("delivery: remote poll pass failed: %s", exc)
|
|
return queued
|
|
|
|
|
|
__all__ = ["DeliveryWorker"]
|