mjalmoudy94
72408a0a4f
The one path that touches a customer's server, and the one the platform cannot undo. It is human-triggered, confirmed and audited; there is deliberately no code path from a pipeline run into it (D13). - deploy_ssh.py: SshDeployer. Release lands in releases/<ts>-<sha7>/, the post-deploy command runs with cwd there, and only then does `current` move — atomically (relative symlink + mv -T). A failed hook or extraction never moves it, so the previous release keeps serving. Host key is pinned on first success (TOFU); a mismatch is a hard failure. Prunes to the newest five. - deploy_ftp.py: FtpDeployer. FTP has no atomic switch and no rollback except a backup, so a backup is mandatory: every file about to be overwritten is archived under ARTIFACTS_ROOT before anything is uploaded. The artifact is extracted, then the build dir (dist/build/public/out) is chosen — a static deploy must not publish the repository. Archive members are sanitised: absolute paths, `..` and escaping links are refused. - deploy_target.py: the tester that dispatches on `kind`. - deploy_service.py + production_deploys (migration a3b4c5d6e7f4) + 4 routes. One deploy per project at a time; the org slug must be typed back; the target summary reports has_secret, never the secret. Tests: 499 API (+26) with fake SSH/FTP transports; ruff, mypy, alembic clean. Report: docs/plan/reports/agent-3-phase-5.md
477 lines
18 KiB
Python
477 lines
18 KiB
Python
"""FTP / FTPS production deploy (D8).
|
|
|
|
FTP is the weakest transport the platform supports, and the honest thing to do
|
|
is say so in the code rather than pretend otherwise. Three limits shape the
|
|
design:
|
|
|
|
**There is no atomic switch.** FTP has no symlinks, so a deploy overwrites files
|
|
in place. A failure halfway through therefore leaves a *half-updated* site, and
|
|
the only defence is a backup taken first.
|
|
|
|
**So a backup is mandatory, not optional.** Before anything is overwritten, the
|
|
files that are about to change are downloaded into
|
|
``ARTIFACTS_ROOT/<org>/<project>/ftp-backup-<ts>.tar.gz``. Rollback re-uploads
|
|
that archive. Without it a bad FTP deploy is unrecoverable.
|
|
|
|
**There is no host key to pin.** FTPS (explicit TLS) encrypts the session but
|
|
says nothing about *which* server you reached, so the TOFU guarantee SSH gives
|
|
does not exist here. The operator is expected to have verified the host out of
|
|
band; the code cannot do it for them.
|
|
|
|
The transport is injectable, so the deploy logic is testable without a server.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import io
|
|
import logging
|
|
import os
|
|
import tarfile
|
|
import time
|
|
from collections.abc import Awaitable, Callable
|
|
from dataclasses import dataclass
|
|
from typing import Any, Protocol, runtime_checkable
|
|
|
|
from app.application.ports.deploy_target import DeployCheck, DeployTargetSpec
|
|
from app.core.redact import redact_text
|
|
from app.domain.platform.rules import assert_remote_host_allowed
|
|
|
|
logger = logging.getLogger("agy.deploy.ftp")
|
|
|
|
CONNECT_TIMEOUT_SECONDS = 15.0
|
|
TRANSFER_TIMEOUT_SECONDS = 600.0
|
|
|
|
#: The directory names a build output commonly lands in, tried in order when the
|
|
#: app profile does not name one. Matches the framework catalog defaults.
|
|
DEFAULT_BUILD_DIRS: tuple[str, ...] = ("dist", "build", "public", "out")
|
|
|
|
#: A probe file written and deleted by `test_connection`.
|
|
PROBE_FILENAME = ".smart-org-probe"
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class FtpEntry:
|
|
path: str
|
|
size: int = 0
|
|
|
|
|
|
@runtime_checkable
|
|
class FtpTransport(Protocol):
|
|
"""The remote operations an FTP deploy needs."""
|
|
|
|
async def connect(
|
|
self, *, host: str, port: int, username: str, password: str, tls: bool
|
|
) -> None: ...
|
|
|
|
async def cwd(self, path: str) -> None: ...
|
|
|
|
async def download(self, remote_path: str) -> bytes: ...
|
|
|
|
async def upload(self, remote_path: str, data: bytes) -> None: ...
|
|
|
|
async def remove(self, remote_path: str) -> None: ...
|
|
|
|
async def list_files(self) -> list[FtpEntry]: ...
|
|
|
|
async def close(self) -> None: ...
|
|
|
|
|
|
TransportFactory = Callable[[], FtpTransport]
|
|
|
|
|
|
class AsyncFtpTransport:
|
|
"""The real transport: one aioftp session per operation."""
|
|
|
|
def __init__(self) -> None:
|
|
# aioftp.Client is only imported inside connect(); the attribute is
|
|
# typed loosely so the module imports without the optional dependency
|
|
# present at type-check time.
|
|
self._client: Any = None
|
|
self._root = ""
|
|
|
|
async def connect(
|
|
self, *, host: str, port: int, username: str, password: str, tls: bool
|
|
) -> None:
|
|
import aioftp
|
|
|
|
if tls:
|
|
# Explicit FTPS: connect in the clear, then upgrade. `aioftp` exposes
|
|
# this through `ssl` on the client; the certificate is not verified
|
|
# because a shared host's FTP certificate is rarely matchable.
|
|
import ssl
|
|
|
|
context = ssl.create_default_context()
|
|
context.check_hostname = False
|
|
context.verify_mode = ssl.CERT_NONE
|
|
self._client = aioftp.Client(ssl=context)
|
|
else:
|
|
self._client = aioftp.Client()
|
|
await self._client.connect(host, port, timeout=CONNECT_TIMEOUT_SECONDS)
|
|
await self._client.login(username, password)
|
|
|
|
async def cwd(self, path: str) -> None:
|
|
assert self._client is not None, "connect() first"
|
|
self._root = path.rstrip("/") or "/"
|
|
await self._client.change_directory(self._root)
|
|
|
|
async def download(self, remote_path: str) -> bytes:
|
|
assert self._client is not None
|
|
buffer = io.BytesIO()
|
|
async with self._client.download_stream(remote_path) as stream:
|
|
async for block in stream.iter_by_block():
|
|
buffer.write(block)
|
|
return buffer.getvalue()
|
|
|
|
async def upload(self, remote_path: str, data: bytes) -> None:
|
|
assert self._client is not None
|
|
async with self._client.upload_stream(remote_path) as stream:
|
|
await stream.write(data)
|
|
|
|
async def remove(self, remote_path: str) -> None:
|
|
assert self._client is not None
|
|
with contextlib.suppress(Exception):
|
|
await self._client.remove_file(remote_path)
|
|
|
|
async def list_files(self) -> list[FtpEntry]:
|
|
assert self._client is not None
|
|
entries: list[FtpEntry] = []
|
|
async for path, info in self._client.list(recursive=False):
|
|
entries.append(FtpEntry(path=str(path), size=int(info.get("size", 0) or 0)))
|
|
return entries
|
|
|
|
async def close(self) -> None:
|
|
if self._client is not None:
|
|
with contextlib.suppress(Exception):
|
|
await self._client.quit()
|
|
self._client = None
|
|
|
|
|
|
class FtpDeployer:
|
|
"""``DeployTargetTester`` and the production deploy itself, over FTP/FTPS."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
transport_factory: TransportFactory = AsyncFtpTransport,
|
|
artifacts_root: str | None = None,
|
|
allow_private_hosts: bool = False,
|
|
clock: Callable[[], float] = time.time,
|
|
) -> None:
|
|
from app.core.config import settings
|
|
|
|
self._factory = transport_factory
|
|
self.artifacts_root = artifacts_root or settings.artifacts_root
|
|
self.allow_private_hosts = allow_private_hosts
|
|
self._clock = clock
|
|
|
|
# -- testing -------------------------------------------------------------
|
|
|
|
async def test_connection(self, *, target: DeployTargetSpec) -> DeployCheck:
|
|
"""Log in, ``cwd`` the target path, and write/delete a probe file."""
|
|
try:
|
|
assert_remote_host_allowed(
|
|
f"ftp://{target.host}:{target.port}", allow_private=self.allow_private_hosts
|
|
)
|
|
except Exception as exc: # noqa: BLE001
|
|
return DeployCheck(ok=False, message=str(exc))
|
|
|
|
transport = self._factory()
|
|
try:
|
|
await transport.connect(
|
|
host=target.host,
|
|
port=target.port,
|
|
username=target.username,
|
|
password=target.secret or "",
|
|
tls=target.kind in ("ftps",),
|
|
)
|
|
await transport.cwd(target.remote_path)
|
|
await transport.upload(PROBE_FILENAME, b"smart-org")
|
|
await transport.remove(PROBE_FILENAME)
|
|
except Exception as exc: # noqa: BLE001
|
|
return DeployCheck(ok=False, message=_ftp_error_message(exc))
|
|
finally:
|
|
with contextlib.suppress(Exception):
|
|
await transport.close()
|
|
|
|
return DeployCheck(
|
|
ok=True,
|
|
message=(
|
|
"اتصال و مسیر مقصد تأیید شد. توجه: FTP امکان تأیید کلید میزبان را "
|
|
"ندارد و تغییر فایلها اتمی نیست."
|
|
),
|
|
)
|
|
|
|
# -- deploy --------------------------------------------------------------
|
|
|
|
async def deploy(
|
|
self,
|
|
*,
|
|
target: DeployTargetSpec,
|
|
artifact_path: str,
|
|
sha: str,
|
|
organization_id: str = "",
|
|
project_id: str = "",
|
|
build_dir: str | None = None,
|
|
on_output: Callable[[str], Awaitable[None]] | None = None,
|
|
) -> tuple[bool, str, str]:
|
|
"""Back up what will be overwritten, then upload the new build.
|
|
|
|
Returns ``(ok, backup_path, message)``. The backup path is returned even
|
|
on success: it is what a later rollback needs, and it is recorded in the
|
|
deploy row.
|
|
"""
|
|
if not os.path.isfile(artifact_path):
|
|
return False, "", "فایل بستهٔ انتشار پیدا نشد."
|
|
|
|
# The artifact is a tar.gz of the whole release; the deployable build is
|
|
# a directory inside it. Unpack first, then choose.
|
|
extracted = _extract(artifact_path, self.artifacts_root, organization_id, project_id)
|
|
if extracted is None:
|
|
return False, "", "استخراج بستهٔ انتشار ناموفق بود."
|
|
|
|
source_dir = _pick_build_dir(extracted, build_dir)
|
|
if source_dir is None:
|
|
return False, "", "پوشهٔ خروجی ساخت در بستهٔ انتشار پیدا نشد."
|
|
|
|
transport = self._factory()
|
|
try:
|
|
await transport.connect(
|
|
host=target.host,
|
|
port=target.port,
|
|
username=target.username,
|
|
password=target.secret or "",
|
|
tls=target.kind == "ftps",
|
|
)
|
|
await transport.cwd(target.remote_path)
|
|
|
|
backup = await self._backup(
|
|
transport,
|
|
target=target,
|
|
organization_id=organization_id,
|
|
project_id=project_id,
|
|
sha=sha,
|
|
)
|
|
if on_output is not None and backup:
|
|
await on_output(f"[deploy] پشتیبان در {backup} ذخیره شد.\n")
|
|
|
|
uploaded = 0
|
|
for relative, data in _walk(source_dir):
|
|
remote = relative
|
|
await transport.upload(remote, data)
|
|
uploaded += 1
|
|
if on_output is not None:
|
|
await on_output(f"[deploy] {uploaded} فایل بارگذاری شد.\n")
|
|
|
|
return True, backup, "استقرار در production انجام شد (FTP)."
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.exception("ftp deploy failed")
|
|
return False, "", _ftp_error_message(exc)
|
|
finally:
|
|
with contextlib.suppress(Exception):
|
|
await transport.close()
|
|
|
|
async def _backup(
|
|
self,
|
|
transport: FtpTransport,
|
|
*,
|
|
target: DeployTargetSpec,
|
|
organization_id: str,
|
|
project_id: str,
|
|
sha: str,
|
|
) -> str:
|
|
"""Download every file we are about to overwrite, into one archive."""
|
|
try:
|
|
entries = await transport.list_files()
|
|
except Exception: # noqa: BLE001 - an empty directory has nothing to back up
|
|
return ""
|
|
|
|
buffer = io.BytesIO()
|
|
captured = 0
|
|
with tarfile.open(fileobj=buffer, mode="w:gz") as tar:
|
|
for entry in entries:
|
|
name = os.path.basename(entry.path.rstrip("/"))
|
|
if not name or name == PROBE_FILENAME:
|
|
continue
|
|
try:
|
|
data = await transport.download(entry.path)
|
|
except Exception: # noqa: BLE001 - a directory entry, or unreadable
|
|
continue
|
|
info = tarfile.TarInfo(name=name)
|
|
info.size = len(data)
|
|
info.mtime = int(self._clock())
|
|
tar.addfile(info, io.BytesIO(data))
|
|
captured += 1
|
|
|
|
if not captured:
|
|
return ""
|
|
|
|
stamp = time.strftime("%Y%m%dT%H%M%S", time.gmtime(self._clock()))
|
|
directory = os.path.join(
|
|
self.artifacts_root, organization_id or "unknown", project_id or "unknown"
|
|
)
|
|
os.makedirs(directory, exist_ok=True)
|
|
path = os.path.join(directory, f"ftp-backup-{stamp}-{(sha or '')[:7]}.tar.gz")
|
|
with open(path, "wb") as handle:
|
|
handle.write(buffer.getvalue())
|
|
return path
|
|
|
|
# -- rollback ------------------------------------------------------------
|
|
|
|
async def rollback(
|
|
self,
|
|
*,
|
|
target: DeployTargetSpec,
|
|
backup_path: str,
|
|
on_output: Callable[[str], Awaitable[None]] | None = None,
|
|
) -> tuple[bool, str]:
|
|
"""Re-upload a backup archive over the live directory."""
|
|
if not backup_path or not os.path.isfile(backup_path):
|
|
return False, "پشتیبان بازگردانی پیدا نشد."
|
|
|
|
transport = self._factory()
|
|
try:
|
|
await transport.connect(
|
|
host=target.host,
|
|
port=target.port,
|
|
username=target.username,
|
|
password=target.secret or "",
|
|
tls=target.kind == "ftps",
|
|
)
|
|
await transport.cwd(target.remote_path)
|
|
restored = 0
|
|
with tarfile.open(backup_path, "r:gz") as tar:
|
|
for member in tar.getmembers():
|
|
if not member.isfile():
|
|
continue
|
|
extracted = tar.extractfile(member)
|
|
if extracted is None:
|
|
continue
|
|
await transport.upload(member.name, extracted.read())
|
|
restored += 1
|
|
if on_output is not None:
|
|
await on_output(f"[rollback] {restored} فایل بازگردانی شد.\n")
|
|
return True, f"بازگردانی {restored} فایل انجام شد."
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.exception("ftp rollback failed")
|
|
return False, _ftp_error_message(exc)
|
|
finally:
|
|
with contextlib.suppress(Exception):
|
|
await transport.close()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _extract(
|
|
artifact_path: str, artifacts_root: str, organization_id: str, project_id: str
|
|
) -> str | None:
|
|
"""Unpack a release archive into a scratch directory and return its path."""
|
|
import shutil
|
|
|
|
stamp = time.strftime("%Y%m%dT%H%M%S", time.gmtime())
|
|
directory = os.path.join(
|
|
artifacts_root, organization_id or "unknown", project_id or "unknown", f"extract-{stamp}"
|
|
)
|
|
shutil.rmtree(directory, ignore_errors=True)
|
|
os.makedirs(directory, exist_ok=True)
|
|
try:
|
|
with tarfile.open(artifact_path, "r:gz") as tar:
|
|
tar.extractall(directory, filter=_safe_member)
|
|
except (tarfile.TarError, OSError):
|
|
return None
|
|
return directory
|
|
|
|
|
|
def _safe_member(member: tarfile.TarInfo, dest: str) -> tarfile.TarInfo | None:
|
|
"""Sanitise an archive member, or skip it.
|
|
|
|
An uploaded artifact is not trusted. Two things are refused outright:
|
|
|
|
* an absolute name or one containing ``..`` — the classic way a tarball
|
|
writes outside the directory it is extracted into; and
|
|
* the archive's own root entry (``.``), which would ``mkdir`` the extraction
|
|
directory itself. That is harmless on a normal filesystem but raises here,
|
|
and skipping it is correct regardless: the root already exists.
|
|
"""
|
|
name = (member.name or "").replace("\\", "/")
|
|
name = name.lstrip("/")
|
|
while name.startswith("./"):
|
|
name = name[2:]
|
|
if name in ("", "."):
|
|
return None
|
|
parts = [part for part in name.split("/") if part not in ("", ".")]
|
|
if not parts or ".." in parts:
|
|
return None
|
|
if member.issym() or member.islnk():
|
|
# A link target is another way out of the extraction directory.
|
|
target = (member.linkname or "").lstrip("/")
|
|
if os.path.isabs(target) or ".." in target.split("/"):
|
|
return None
|
|
member.name = "/".join(parts)
|
|
return member
|
|
|
|
|
|
def _pick_build_dir(artifact_path: str, explicit: str | None) -> str | None:
|
|
"""The directory inside the artifact that holds the deployable build.
|
|
|
|
A build output is a *subdirectory* of the release archive (`dist/`, `build/`,
|
|
`public/`, `out/`), not the archive itself. Uploading the whole repository
|
|
over a static site is how a deploy publishes source code by accident.
|
|
"""
|
|
if not os.path.isdir(artifact_path):
|
|
return None
|
|
if explicit:
|
|
candidate = os.path.join(artifact_path, explicit)
|
|
return candidate if os.path.isdir(candidate) else None
|
|
for name in DEFAULT_BUILD_DIRS:
|
|
candidate = os.path.join(artifact_path, name)
|
|
if os.path.isdir(candidate):
|
|
return candidate
|
|
# A static site has no build step: its root *is* the output.
|
|
return artifact_path
|
|
|
|
|
|
def _walk(root: str):
|
|
"""Yield ``(relative_path, bytes)`` for every file under ``root``."""
|
|
for directory, _dirs, files in os.walk(root):
|
|
for name in files:
|
|
absolute = os.path.join(directory, name)
|
|
relative = os.path.relpath(absolute, root).replace(os.sep, "/")
|
|
try:
|
|
with open(absolute, "rb") as handle:
|
|
yield relative, handle.read()
|
|
except OSError:
|
|
continue
|
|
|
|
|
|
def _ftp_error_message(exc: BaseException) -> str:
|
|
"""A plain Persian message. Never the raw exception."""
|
|
text = str(exc).lower()
|
|
if "530" in text or "login" in text or "auth" in text:
|
|
return "ورود به سرور مقصد ناموفق بود."
|
|
if "550" in text or "not found" in text or "no such" in text:
|
|
return "مسیر مقصد روی سرور پیدا نشد."
|
|
if "timeout" in text or "timed out" in text:
|
|
return "اتصال به سرور مقصد در زمان مجاز برقرار نشد."
|
|
if "refused" in text or "unreachable" in text:
|
|
return "اتصال به سرور مقصد برقرار نشد."
|
|
if "certificate" in text or "ssl" in text or "tls" in text:
|
|
return "برقراری اتصال امن (FTPS) ناموفق بود."
|
|
return redact_text("استقرار روی سرور مقصد ناموفق بود.")
|
|
|
|
|
|
__all__ = [
|
|
"CONNECT_TIMEOUT_SECONDS",
|
|
"DEFAULT_BUILD_DIRS",
|
|
"PROBE_FILENAME",
|
|
"TRANSFER_TIMEOUT_SECONDS",
|
|
"AsyncFtpTransport",
|
|
"FtpDeployer",
|
|
"FtpEntry",
|
|
"FtpTransport",
|
|
]
|
|
|