mjalmoudy94
f1e5df29ce
- Backend: ConnectionManager, Redis-backed bus with in-process fallback, channel authorization, and typed emit helpers wired into workers (ai_runtime, sandbox_gateway), workforce/sandbox/billing/audit/error services. - Frontend: single shared WS client with backoff + heartbeat, RealtimeProvider, useLiveChannel/useLiveEvent hooks, live indicators on task detail, manager chat, dashboard, traces, sandbox terminal, admin errors, and billing. - Contracts: WsEvent/WsServerFrame/WsClientFrame types mirroring the port. All 255 backend tests pass; web typecheck and build pass.
295 lines
10 KiB
Python
295 lines
10 KiB
Python
"""Persist failures so they can be diagnosed after the fact.
|
|
|
|
Three properties make this module different from an ordinary repository call,
|
|
and all three are load-bearing:
|
|
|
|
**It never raises.** This is called from inside exception handlers, workers and
|
|
startup code — the exact places where an exception is already in flight. A
|
|
recorder that throws turns a handled 500 into an unhandled one, or kills a
|
|
worker mid-batch. Every failure here is swallowed and reported on stderr.
|
|
|
|
**It uses its own session.** The caller's session is frequently unusable by the
|
|
time we get here: an ``IntegrityError`` leaves it in a failed transaction where
|
|
every subsequent statement raises ``PendingRollbackError``. Reusing it would
|
|
mean the errors hardest to record — the database errors — are exactly the ones
|
|
that never get recorded. So we open a short-lived session of our own, commit,
|
|
and close.
|
|
|
|
**It redacts before it writes.** Everything passes through
|
|
``app.core.redact`` first. Error payloads are where credentials concentrate.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import re
|
|
import sys
|
|
import traceback
|
|
from collections.abc import Mapping
|
|
from typing import Any
|
|
|
|
from app.core.database import SessionFactory
|
|
from app.core.redact import redact_mapping, redact_text
|
|
from app.domain.audit.models import (
|
|
ErrorEvent,
|
|
ErrorSeverity,
|
|
ErrorSource,
|
|
)
|
|
|
|
logger = logging.getLogger("agy.errors")
|
|
|
|
#: Field caps. An error log that accepts unbounded input is a denial-of-service
|
|
#: vector and a storage leak; a 2 MB stack trace helps nobody.
|
|
MAX_MESSAGE_CHARS = 4_000
|
|
MAX_STACK_CHARS = 16_000
|
|
MAX_METADATA_JSON_CHARS = 20_000
|
|
MAX_KIND_CHARS = 80
|
|
MAX_TYPE_CHARS = 200
|
|
MAX_PATH_CHARS = 512
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fingerprinting — collapse a thousand occurrences into one row
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_UUID_RE = re.compile(
|
|
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}"
|
|
r"-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
|
|
)
|
|
_HEX_RE = re.compile(r"\b[0-9a-fA-F]{8,}\b")
|
|
_NUMBER_RE = re.compile(r"\b\d+\b")
|
|
_QUOTED_RE = re.compile(r"'[^']{0,200}'|\"[^\"]{0,200}\"")
|
|
#: A path segment that is an entity id rather than a route name. All-hex catches
|
|
#: ``new_id()`` (32 hex chars); the digit+length rule catches other id styles.
|
|
#: Route words like ``organizations`` have no digits and survive untouched.
|
|
_ID_SEGMENT_RE = re.compile(r"^(?:[0-9a-fA-F]{8,}|(?=.*\d)[A-Za-z0-9_-]{16,})$")
|
|
|
|
|
|
def _normalise_message(message: str) -> str:
|
|
"""Strip the variable parts of a message so equal bugs hash equally.
|
|
|
|
``"task 4f3a… not found"`` and ``"task 9b1c… not found"`` are one bug with
|
|
two victims; without this they are two unrelated rows.
|
|
"""
|
|
text = _UUID_RE.sub("{uuid}", message)
|
|
text = _HEX_RE.sub("{hex}", text)
|
|
text = _QUOTED_RE.sub("{value}", text)
|
|
return _NUMBER_RE.sub("{n}", text)
|
|
|
|
|
|
def _normalise_path(path: str) -> str:
|
|
"""``/api/orgs/4f3a…/projects`` → ``/api/orgs/{id}/projects``."""
|
|
if not path:
|
|
return ""
|
|
return "/".join(
|
|
"{id}" if _ID_SEGMENT_RE.match(segment) else segment for segment in path.split("/")
|
|
)
|
|
|
|
|
|
def compute_fingerprint(
|
|
*,
|
|
kind: str,
|
|
error_type: str,
|
|
method: str = "",
|
|
path: str = "",
|
|
message: str = "",
|
|
) -> str:
|
|
"""Stable identity for a failure, independent of tenant and occurrence.
|
|
|
|
Deliberately *excludes* organization, user and request id: the same bug
|
|
hitting two tenants is still one bug, and grouping it that way is the whole
|
|
point of a fingerprint.
|
|
"""
|
|
material = "\x1f".join(
|
|
(
|
|
kind.strip().lower(),
|
|
error_type.strip().lower(),
|
|
method.strip().upper(),
|
|
_normalise_path(path).lower(),
|
|
_normalise_message(message.strip())[:500],
|
|
)
|
|
)
|
|
return hashlib.sha256(material.encode("utf-8", "replace")).hexdigest()[:32]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Truncation helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _clip(value: str, limit: int) -> str:
|
|
if len(value) <= limit:
|
|
return value
|
|
return value[:limit] + f"…[+{len(value) - limit} chars]"
|
|
|
|
|
|
def _safe_metadata(metadata: Mapping[str, Any] | None) -> dict[str, Any]:
|
|
"""Redact, then bound. Returns a dict that is always JSON-serialisable."""
|
|
if not metadata:
|
|
return {}
|
|
scrubbed = redact_mapping(metadata)
|
|
try:
|
|
encoded = json.dumps(scrubbed, ensure_ascii=False, default=str)
|
|
except (TypeError, ValueError):
|
|
return {"_note": "metadata was not JSON-serialisable"}
|
|
if len(encoded) <= MAX_METADATA_JSON_CHARS:
|
|
return scrubbed
|
|
# Too large to keep: retain the keys so the shape is still visible.
|
|
return {
|
|
"_note": f"metadata dropped: {len(encoded)} chars exceeds {MAX_METADATA_JSON_CHARS}",
|
|
"_keys": sorted(str(k) for k in scrubbed)[:50],
|
|
}
|
|
|
|
|
|
def format_exception_stack(exc: BaseException) -> str:
|
|
"""``traceback.format_exception`` without the chained-context noise limit."""
|
|
return "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
|
|
|
|
|
|
def _to_stderr(event: ErrorEvent, reason: str) -> None:
|
|
"""Last-resort output when the database write failed.
|
|
|
|
Goes to stderr rather than through ``logging`` so it survives even a
|
|
misconfigured logging setup — the one case where losing the error entirely
|
|
is unacceptable.
|
|
"""
|
|
with contextlib.suppress(Exception):
|
|
sys.stderr.write(
|
|
f"[error-log] could not persist error ({reason}) "
|
|
f"kind={event.kind} type={event.error_type} "
|
|
f"path={event.path} msg={event.message[:300]}\n"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Public API
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def record_error(
|
|
*,
|
|
kind: str,
|
|
message: str,
|
|
exc: BaseException | None = None,
|
|
stack: str = "",
|
|
source: str = ErrorSource.HTTP,
|
|
severity: str = ErrorSeverity.ERROR,
|
|
status_code: int | None = None,
|
|
method: str = "",
|
|
path: str = "",
|
|
organization_id: str | None = None,
|
|
user_id: str | None = None,
|
|
run_id: str | None = None,
|
|
request_id: str = "",
|
|
ip: str = "",
|
|
user_agent: str = "",
|
|
metadata: Mapping[str, Any] | None = None,
|
|
) -> str | None:
|
|
"""Write one error row. Returns its id, or ``None`` if recording failed.
|
|
|
|
Returning ``None`` rather than raising is the contract: callers treat error
|
|
recording as best-effort telemetry, never as a step that can fail a request.
|
|
"""
|
|
try:
|
|
error_type = type(exc).__name__ if exc is not None else ""
|
|
if not stack and exc is not None:
|
|
stack = format_exception_stack(exc)
|
|
if not message and exc is not None:
|
|
message = str(exc) or error_type
|
|
|
|
# Redact *before* fingerprinting is not required, but redact before
|
|
# *storage* absolutely is. Fingerprint the raw message so a secret that
|
|
# happens to appear in a message does not change the grouping.
|
|
fingerprint = compute_fingerprint(
|
|
kind=kind, error_type=error_type, method=method, path=path, message=message
|
|
)
|
|
|
|
event = ErrorEvent(
|
|
kind=_clip(redact_text(kind), MAX_KIND_CHARS),
|
|
message=_clip(redact_text(message), MAX_MESSAGE_CHARS),
|
|
stack=_clip(redact_text(stack), MAX_STACK_CHARS),
|
|
metadata_json=_safe_metadata(metadata),
|
|
organization_id=organization_id,
|
|
user_id=user_id,
|
|
run_id=run_id,
|
|
source=source,
|
|
severity=severity,
|
|
error_type=_clip(redact_text(error_type), MAX_TYPE_CHARS),
|
|
fingerprint=fingerprint,
|
|
status_code=status_code,
|
|
method=_clip(method, 10),
|
|
path=_clip(redact_text(path), MAX_PATH_CHARS),
|
|
request_id=_clip(request_id, 64),
|
|
ip=_clip(redact_text(ip), 64),
|
|
user_agent=_clip(redact_text(user_agent), 255),
|
|
)
|
|
except Exception as build_exc: # pragma: no cover - redaction is defensive
|
|
logger.exception("error-log: could not build the error record: %s", build_exc)
|
|
return None
|
|
|
|
# A dedicated session: the caller's may be mid-rollback, and a failed error
|
|
# write must not roll back whatever the caller was legitimately doing.
|
|
try:
|
|
async with SessionFactory() as session:
|
|
from app.infrastructure.repositories.audit import ObservabilityRepository
|
|
|
|
await ObservabilityRepository(session).record_error(event)
|
|
await session.commit()
|
|
except Exception as write_exc:
|
|
_to_stderr(event, type(write_exc).__name__)
|
|
logger.warning("error-log: persistence failed: %s", write_exc)
|
|
return None
|
|
|
|
# Mirror to the application log so the console and the terminal agree.
|
|
logger.log(
|
|
logging.ERROR if severity != ErrorSeverity.WARNING else logging.WARNING,
|
|
"%s [%s] %s %s -> %s: %s (request_id=%s fingerprint=%s)",
|
|
severity.upper(),
|
|
source,
|
|
method or "-",
|
|
path or "-",
|
|
status_code if status_code is not None else "-",
|
|
event.message[:500],
|
|
request_id or "-",
|
|
fingerprint,
|
|
)
|
|
|
|
# Live push to the platform-owner console. Best-effort like everything here:
|
|
# a failed publish must not turn a recorded error into an unrecorded one.
|
|
from app.application.ports.realtime import emit_error_recorded
|
|
|
|
await emit_error_recorded(
|
|
error_id=event.id,
|
|
kind=event.kind,
|
|
message=event.message,
|
|
severity=str(severity),
|
|
source=str(source),
|
|
organization_id=event.organization_id,
|
|
)
|
|
return event.id
|
|
|
|
|
|
async def record_exception(
|
|
exc: BaseException,
|
|
*,
|
|
kind: str | None = None,
|
|
**context: Any,
|
|
) -> str | None:
|
|
"""``record_error`` for the common case where there is an exception object."""
|
|
return await record_error(
|
|
kind=kind or type(exc).__name__,
|
|
message=str(exc),
|
|
exc=exc,
|
|
**context,
|
|
)
|
|
|
|
|
|
__all__ = [
|
|
"compute_fingerprint",
|
|
"format_exception_stack",
|
|
"record_error",
|
|
"record_exception",
|
|
]
|