agy تنها ارائه‌دهندهٔ مدل؛ حذف بخش ارائه‌دهنده‌ها و افزودن زنجیرهٔ انتخاب مدل

- حذف کامل providerهای mock و openai_compatible: دامنه، آداپوترها، رجیستری، کانفیگ، seed و رابطٔ مدیریت
- بخش «ارائه‌دهنده‌ها» از کنسول حذف شد؛ ورود اکانت‌ها فقط از صفحهٔ اکانت‌های agy
- صفحهٔ مدل‌ها بازطراحی شد: ارائه‌دهنده → اکانت → مدل (کاتالوگ لحظه‌ای و زندهٔ همان اکانت از Google) → افورت استدلال؛ بدون هیچ ورود دستی نام مدل
- مدل‌ها ستون reasoning_effort گرفتند؛ آداپتور آن را به thinkingConfig (بودجهٔ ۱۰۲۴ تا ۳۲۷۶۸) ترجمه می‌کند و فقط به مدل‌های استدلالی می‌فرستد
- مهاجرت c2d3e4f5b6a7: افزودن reasoning_effort و حذف ردیف‌های providerهای قدیمی
- تست‌ها: ۲۴۶ تست بک‌اند و ۵۱ تست فرانت سبز؛ ruff و mypy و tsc و eslint تمیز
This commit is contained in:
mjalmoudy94
2026-10-13 10:01:11 +03:30
parent a2284db4f4
commit da5a2a5d1a
29 changed files with 906 additions and 995 deletions
+110 -13
View File
@@ -1,7 +1,10 @@
"""Shared test fixtures.
Tests run against an in-memory SQLite database and the ``mock`` adapters, so the
suite needs no Postgres, no Redis and no credentials.
Tests run against an in-memory SQLite database with the real agy-shaped adapter
wiring and a fake model provider, so the suite needs no Postgres, no Redis, no
Google and no accounts. agy is the only model provider the platform talks to
(see ``app/infrastructure/adapters/registry.py``); the fake keeps completions
deterministic and offline.
"""
from __future__ import annotations
@@ -96,6 +99,76 @@ async def client(engine, session_factory):
app.dependency_overrides.clear()
# ---------------------------------------------------------------------------
# Fake model provider — the only sanctioned stand-in for agy in tests
# ---------------------------------------------------------------------------
class _FakeModelProvider:
"""Offline stand-in for the agy adapter.
The canned text must read like model output, not like a debug tag: the
provider/model identity is platform-internal and must never surface in
anything a customer can read (see the e2e smoke test).
"""
async def complete(
self,
*,
model,
messages,
system="",
temperature=0.2,
max_tokens=2048,
credential=None,
credentials=None,
):
import asyncio
import time
from app.application.ports import ModelResponse
started = time.perf_counter()
last_user = next((m.content for m in reversed(messages) if m.role == "user"), "")
prompt_tokens = sum(len(m.content.split()) for m in messages)
# A tiny sleep so latency bookkeeping runs a real code path.
await asyncio.sleep(0)
content = (
"این پاسخ آزمایشی است. "
f"درخواست دریافتی: {last_user[:280] or '—'}"
)
return ModelResponse(
content=content,
model_internal_name=model.internal_name,
input_tokens=prompt_tokens + len(system.split()),
output_tokens=len(content.split()),
latency_ms=int((time.perf_counter() - started) * 1000),
raw={},
)
async def health(self):
from app.application.ports import ModelHealth
return ModelHealth(ok=True, detail="fake agy pool")
@pytest.fixture(autouse=True)
def _fake_model_provider(monkeypatch):
"""Swap the real model adapter out of every request path.
``container`` and the health router bind ``get_model_provider`` at import
time, so both import sites are patched; without this a completion test
would open the developer's real database and reach the real Google API.
"""
from app.application import container as di_container
from app.interfaces.http.routers import health as health_router
provider = _FakeModelProvider()
monkeypatch.setattr(di_container, "get_model_provider", lambda: provider)
monkeypatch.setattr(health_router, "get_model_provider", lambda: provider)
return provider
async def signup(
client: AsyncClient, email: str, password: str = "password123", name: str = ""
) -> dict:
@@ -149,10 +222,31 @@ SYSTEM_ROUTING_POLICIES: list[tuple[str, str, bool, str]] = [
("استدلال عمیق", "reasoning", False, "برای تحلیل پیچیده."),
]
#: The mock models ``app/seed.py`` installs: (internal_name, display, capabilities).
SYSTEM_MODELS: list[tuple[str, str, list[str]]] = [
("mock-fast", "Mock Fast", ["text"]),
("mock-reasoning", "Mock Reasoning", ["text", "reasoning", "code"]),
#: The agy models ``app/seed.py`` installs: (internal_name, display,
#: capabilities, reasoning_effort). agy is the only provider, so this is the
#: whole catalogue.
SYSTEM_MODELS: list[tuple[str, str, list[str], str]] = [
("gemini-3-flash", "Gemini 3 Flash", ["text", "code", "long_context"], ""),
(
"gemini-3-pro-high",
"Gemini 3 Pro",
["text", "reasoning", "code", "vision", "long_context"],
"high",
),
("claude-sonnet-4-5", "Claude Sonnet 4.5", ["text", "code", "tools"], ""),
(
"claude-sonnet-4-5-thinking",
"Claude Sonnet 4.5 (Thinking)",
["text", "code", "reasoning"],
"high",
),
(
"claude-opus-4-5-thinking",
"Claude Opus 4.5 (Thinking)",
["text", "reasoning", "code", "long_context"],
"high",
),
("gpt-oss-120b-medium", "GPT-OSS 120B Medium", ["text", "code"], ""),
]
#: The rate cards ``app/seed.py`` installs: (unit, customer_price, internal_cost).
@@ -174,11 +268,11 @@ SYSTEM_SANDBOX_TEMPLATES: list[tuple[str, str, str]] = [
async def seeded(session_factory):
"""Install the platform-level seed rows a fresh install would have.
Agent templates, routing policies, the mock provider/models, the rate cards
and the sandbox templates are all *platform* data (no organization owns
them), so without this an employee cannot be created, a run cannot resolve a
model to route to, usage cannot be priced and no sandbox can be started —
none of which the real seeded install lacks.
Agent templates, routing policies, the agy provider with its model catalog,
the rate cards and the sandbox templates are all *platform* data (no
organization owns them), so without this an employee cannot be created, a
run cannot resolve a model to route to, usage cannot be priced and no
sandbox can be started — none of which the real seeded install lacks.
"""
from app.infrastructure.db.models import (
AgentTemplateRow,
@@ -214,9 +308,11 @@ async def seeded(session_factory):
# surfaces as a ``ProviderUnavailableError`` and a *failed* run.
provider_id = new_id()
session.add(
AiProviderRow(id=provider_id, name="mock", type="mock", base_url="", is_active=True)
AiProviderRow(
id=provider_id, name="agy", type="agy", base_url="", is_active=True
)
)
for internal, display, capabilities in SYSTEM_MODELS:
for internal, display, capabilities, effort in SYSTEM_MODELS:
session.add(
AiModelRow(
provider_id=provider_id,
@@ -225,6 +321,7 @@ async def seeded(session_factory):
capabilities=capabilities,
input_price_per_1m=0.3,
output_price_per_1m=1.2,
reasoning_effort=effort,
is_active=True,
)
)
+42 -94
View File
@@ -3,7 +3,9 @@
Two properties matter more than any feature here:
1. A normal customer can never reach these endpoints.
2. A provider API key can never come back out of the API.
2. Model identity and account material stay inside the console — a customer
never sees a raw model, and nothing that could act as a credential (a Google
refresh token, an API key) ever comes back out.
"""
from __future__ import annotations
@@ -12,146 +14,92 @@ from httpx import AsyncClient
from tests.conftest import auth_header, signup
SECRET_KEY = "sk-super-secret-value-do-not-leak-1234567890"
async def test_normal_user_cannot_list_providers(client: AsyncClient):
async def test_normal_user_cannot_list_models(client: AsyncClient):
token = (await signup(client, "peon@example.com"))["access_token"]
response = await client.get("/api/admin/providers", headers=auth_header(token))
response = await client.get("/api/admin/models", headers=auth_header(token))
assert response.status_code == 403
assert response.json()["error"]["code"] == "permission_denied"
async def test_normal_user_cannot_create_provider(client: AsyncClient):
async def test_normal_user_cannot_create_model(client: AsyncClient):
token = (await signup(client, "peon2@example.com"))["access_token"]
response = await client.post(
"/api/admin/providers",
json={"name": "evil", "type": "mock", "base_url": ""},
"/api/admin/models",
json={"internal_name": "evil-model"},
headers=auth_header(token),
)
assert response.status_code == 403
async def test_admin_endpoints_require_authentication(client: AsyncClient):
response = await client.get("/api/admin/providers")
response = await client.get("/api/admin/models")
assert response.status_code == 401
async def test_platform_owner_can_create_and_list_provider(
async def test_model_creation_resolves_the_agy_provider_itself(
client: AsyncClient, platform_owner_token: str
):
"""The provider is not a client choice: agy is the only one."""
created = await client.post(
"/api/admin/providers",
json={"name": "openrouter", "type": "openrouter", "base_url": "https://openrouter.ai/api/v1"},
"/api/admin/models",
json={
"internal_name": "gemini-3-pro-high",
"capabilities": ["text", "reasoning"],
"input_price_per_1m": 0.3,
"output_price_per_1m": 1.2,
"reasoning_effort": "high",
},
headers=auth_header(platform_owner_token),
)
assert created.status_code == 201, created.text
assert created.json()["name"] == "openrouter"
body = created.json()
assert body["internal_name"] == "gemini-3-pro-high"
# The display name falls back to the model id; nothing is typed twice.
assert body["display_name_for_admin"] == "gemini-3-pro-high"
assert body["reasoning_effort"] == "high"
assert body["provider_id"], "the agy provider must be resolved server-side"
listing = await client.get("/api/admin/providers", headers=auth_header(platform_owner_token))
listing = await client.get("/api/admin/models", headers=auth_header(platform_owner_token))
assert listing.status_code == 200
assert [p["name"] for p in listing.json()] == ["openrouter"]
assert [m["internal_name"] for m in listing.json()] == ["gemini-3-pro-high"]
async def test_duplicate_provider_name_is_rejected(client: AsyncClient, platform_owner_token: str):
payload = {"name": "dupe", "type": "mock", "base_url": ""}
async def test_duplicate_model_name_is_rejected(client: AsyncClient, platform_owner_token: str):
payload = {"internal_name": "claude-sonnet-4-5-thinking", "reasoning_effort": "high"}
first = await client.post(
"/api/admin/providers", json=payload, headers=auth_header(platform_owner_token)
"/api/admin/models", json=payload, headers=auth_header(platform_owner_token)
)
assert first.status_code == 201
second = await client.post(
"/api/admin/providers", json=payload, headers=auth_header(platform_owner_token)
"/api/admin/models", json=payload, headers=auth_header(platform_owner_token)
)
assert second.status_code == 409
async def test_credential_creation_returns_only_a_masked_key(
async def test_unknown_reasoning_effort_is_rejected(
client: AsyncClient, platform_owner_token: str
):
provider = (
await client.post(
"/api/admin/providers",
json={"name": "p1", "type": "openai_compatible", "base_url": "https://x.test/v1"},
headers=auth_header(platform_owner_token),
)
).json()
response = await client.post(
f"/api/admin/providers/{provider['id']}/credentials",
json={"name": "primary", "api_key": SECRET_KEY},
"/api/admin/models",
json={"internal_name": "gemini-3-flash", "reasoning_effort": "ultra"},
headers=auth_header(platform_owner_token),
)
assert response.status_code == 201, response.text
body = response.json()
assert SECRET_KEY not in str(body)
assert body["masked_key"].startswith("•")
assert body["masked_key"].endswith(SECRET_KEY[-4:])
assert response.status_code == 422
async def test_listing_credentials_never_exposes_the_key(
async def test_agy_account_and_model_endpoints_are_owner_only(
client: AsyncClient, platform_owner_token: str
):
provider = (
await client.post(
"/api/admin/providers",
json={"name": "p2", "type": "openai_compatible", "base_url": "https://y.test/v1"},
headers=auth_header(platform_owner_token),
)
).json()
await client.post(
f"/api/admin/providers/{provider['id']}/credentials",
json={"name": "primary", "api_key": SECRET_KEY},
headers=auth_header(platform_owner_token),
)
"""The account catalog is owner-only and never carries token material."""
token = (await signup(client, "peon3@example.com"))["access_token"]
response = await client.get(
f"/api/admin/providers/{provider['id']}/credentials",
headers=auth_header(platform_owner_token),
"/api/admin/agy/accounts/whatever/models", headers=auth_header(token)
)
assert response.status_code == 200
assert SECRET_KEY not in response.text
assert response.json()[0]["masked_key"] == "••••••"
assert response.status_code == 403
async def test_model_creation_and_listing(client: AsyncClient, platform_owner_token: str):
provider = (
await client.post(
"/api/admin/providers",
json={"name": "p3", "type": "mock", "base_url": ""},
headers=auth_header(platform_owner_token),
)
).json()
created = await client.post(
"/api/admin/models",
json={
"provider_id": provider["id"],
"internal_name": "internal-coder-v1",
"display_name_for_admin": "Coder V1",
"capabilities": ["text", "code"],
"input_price_per_1m": 0.3,
"output_price_per_1m": 1.2,
},
headers=auth_header(platform_owner_token),
)
assert created.status_code == 201, created.text
listing = await client.get("/api/admin/models", headers=auth_header(platform_owner_token))
assert listing.status_code == 200
assert listing.json()[0]["internal_name"] == "internal-coder-v1"
async def test_model_requires_existing_provider(client: AsyncClient, platform_owner_token: str):
response = await client.post(
"/api/admin/models",
json={
"provider_id": "does-not-exist",
"internal_name": "x",
"display_name_for_admin": "X",
},
headers=auth_header(platform_owner_token),
)
assert response.status_code == 404
unauthenticated = await client.get("/api/admin/agy/accounts/whatever/models")
assert unauthenticated.status_code == 401
async def test_routing_policy_creation_and_default_swap(
+173 -21
View File
@@ -344,41 +344,42 @@ async def test_seed_creates_agy_catalog(client, seeded_install, session_factory)
"claude-opus-4-5-thinking",
"gpt-oss-120b-medium",
}
# Default deployment keeps routing on mock: agy models wait inactive.
assert all(active is False for _, active in models)
async def test_seed_activates_agy_when_it_is_default(
monkeypatch, client, seeded_install, session_factory
):
from app.core.config import settings
monkeypatch.setattr(settings, "ai_default_provider", "agy")
await seeded_install.seed()
models = await _agy_models(session_factory)
# agy is the only provider: the catalog is routable from the first run.
assert all(active is True for _, active in models)
async def test_seed_deactivates_mock_when_agy_is_default(
monkeypatch, client, seeded_install, session_factory
):
async def test_seed_round_trips_efforts(client, seeded_install, session_factory):
"""The seed's effort column is what the adapter later sends to Google."""
from sqlalchemy import select
from app.core.config import settings
from app.infrastructure.db.models import AiModelRow, AiProviderRow
monkeypatch.setattr(settings, "ai_default_provider", "agy")
await seeded_install.seed()
async with session_factory() as session:
rows = (
await session.execute(
select(AiModelRow)
.join(AiProviderRow, AiModelRow.provider_id == AiProviderRow.id)
.where(AiProviderRow.name == "mock")
.where(AiProviderRow.name == "agy")
)
).scalars()
assert all(row.is_active is False for row in rows)
efforts = {row.internal_name: row.reasoning_effort for row in rows}
assert efforts["gemini-3-pro-high"] == "high"
assert efforts["claude-sonnet-4-5-thinking"] == "high"
assert efforts["gemini-3-flash"] == ""
async def test_seed_leaves_no_legacy_providers(client, seeded_install, session_factory):
"""agy is the only provider row a seeded install may carry."""
from sqlalchemy import select
from app.infrastructure.db.models import AiProviderRow
async with session_factory() as session:
rows = (await session.execute(select(AiProviderRow))).scalars()
types = {row.type for row in rows}
names = {row.name for row in rows}
assert types == {"agy"}
assert "mock" not in names
# ---------------------------------------------------------------------------
@@ -519,6 +520,64 @@ def test_payload_translation(decrypted):
assert all(s["threshold"] == "OFF" for s in payload["request"]["safetySettings"])
def test_effort_becomes_a_thinking_config_on_a_thinking_model():
"""Effort is translated, not forwarded: high → a 16384-token budget."""
account = agy.AgyAccount(
credential_id="c1", email="a@b.c", project_id="proj-1", refresh_token="refresh-1"
)
payload = agy._build_generate_payload(
account,
"claude-sonnet-4-5-thinking",
[ChatMessage(role="user", content="سلام")],
system="",
temperature=0.4,
max_tokens=512,
reasoning_effort="high",
)
generation = payload["request"]["generationConfig"]
assert generation["thinkingConfig"] == {
"thinkingBudget": 16384,
"includeThoughts": True,
}
# The budget must fit inside the answer window.
assert generation["maxOutputTokens"] == 16384 + 8000
def test_effort_is_not_sent_to_a_non_thinking_model():
"""``thinkingConfig`` on a model that cannot think is a hard 400."""
account = agy.AgyAccount(
credential_id="c1", email="a@b.c", project_id="proj-1", refresh_token="refresh-1"
)
payload = agy._build_generate_payload(
account,
"claude-sonnet-4-5",
[ChatMessage(role="user", content="سلام")],
system="",
temperature=0.4,
max_tokens=512,
reasoning_effort="high",
)
assert "thinkingConfig" not in payload["request"]["generationConfig"]
assert payload["request"]["generationConfig"]["maxOutputTokens"] == 512
def test_unknown_effort_falls_back_to_medium():
account = agy.AgyAccount(
credential_id="c1", email="a@b.c", project_id="proj-1", refresh_token="refresh-1"
)
payload = agy._build_generate_payload(
account,
"claude-opus-4-5-thinking",
[ChatMessage(role="user", content="سلام")],
system="",
temperature=0.4,
max_tokens=512,
reasoning_effort="turbo",
)
budget = payload["request"]["generationConfig"]["thinkingConfig"]["thinkingBudget"]
assert budget == 8192
def test_response_parsing_skips_thoughts():
model = AiModel(provider_id="p", internal_name="m", display_name_for_admin="m")
response = agy._parse_generate_response(
@@ -713,3 +772,96 @@ async def test_available_models(google, decrypted):
adapter = agy.AgyModelAdapter(loader=loader)
models = await adapter.available_models()
assert models == ["claude-sonnet-4-5", "gemini-3-flash"]
# ---------------------------------------------------------------------------
# Live per-account model catalog (the console's model dropdown source)
# ---------------------------------------------------------------------------
async def test_available_models_for_one_account(google, decrypted):
"""The console asks for a specific account, so the fetch is per-account."""
adapter = agy.AgyModelAdapter()
models = await adapter.available_models_for(
_agy_credential(email="one@gmail.com")
)
assert models == ["claude-sonnet-4-5", "gemini-3-flash"]
# The list is fetched from Google every call — no cached catalog.
fetches = [c for c in google if c["url"].endswith("fetchAvailableModels")]
assert len(fetches) == 1
async def test_available_models_for_account_without_material():
"""A credential with no refresh token can never list models."""
adapter = agy.AgyModelAdapter()
credential = ModelCredential(
provider_id="p",
name="empty@gmail.com",
encrypted_api_key="PLAINTEXT::{}",
email="empty@gmail.com",
)
with pytest.raises(ProviderUnavailableError):
await adapter.available_models_for(credential)
async def test_account_models_endpoint(client, google, platform_owner_token):
"""GET /accounts/{id}/models returns that account's live catalog."""
started = await client.post(
"/api/admin/agy/accounts/login/start",
headers=auth_header(platform_owner_token),
)
state = parse_qs(urlparse(started.json()["auth_url"]).query)["state"][0]
account_id = (
await client.post(
"/api/admin/agy/accounts/login/complete",
headers=auth_header(platform_owner_token),
json={
"login_id": started.json()["login_id"],
"callback_url": (
f"http://localhost:51121/oauth-callback?code=c&state={state}"
),
},
)
).json()["id"]
response = await client.get(
f"/api/admin/agy/accounts/{account_id}/models",
headers=auth_header(platform_owner_token),
)
assert response.status_code == 200, response.text
assert response.json() == ["claude-sonnet-4-5", "gemini-3-flash"]
missing = await client.get(
"/api/admin/agy/accounts/does-not-exist/models",
headers=auth_header(platform_owner_token),
)
assert missing.status_code == 404
async def test_completion_carries_the_model_effort(monkeypatch, decrypted):
"""The end-to-end path: effort on the model row reaches the payload."""
async def handler(method, url, json_body, headers, data):
if url.endswith("/token"):
return _token_response()
if url.endswith("/v1internal:generateContent"):
return _FakeResponse(json_data=GOOD_GENERATE)
return await default_handler(method, url, json_body, headers, data)
calls = patch_google(monkeypatch, handler)
model = AiModel(
provider_id="p",
internal_name="claude-sonnet-4-5-thinking",
display_name_for_admin="f",
reasoning_effort="low",
)
adapter = agy.AgyModelAdapter()
await adapter.complete(
model=model,
credentials=[_agy_credential(email="one@gmail.com")],
messages=_MESSAGES,
)
generate = [c for c in calls if c["url"].endswith("generateContent")]
assert generate[0]["json"]["request"]["generationConfig"]["thinkingConfig"] == {
"thinkingBudget": 4096,
"includeThoughts": True,
}
+3 -1
View File
@@ -1,7 +1,9 @@
"""§18.3 — the minimal end-to-end smoke path.
One test walks the whole product in order, through the real HTTP API and the
real application services, on the mock adapters:
real application services, against the agy-shaped catalogue (the seeded fixture
installs the agy provider and its models) with the fake model provider from
conftest standing in for the Google round-trip:
signup -> organization -> project -> repo -> preview domain -> sandbox
-> employee -> manager message -> task -> run -> usage -> audit