mjalmoudy94
581f703910
Phase 3 of the AI-workforce engine (TODO-agent-2): - engine/manager.py: ManagerTurns.chat (CREATE_TASK/PAUSE/RESUME/CANCEL, HIRE/UPDATE_EMPLOYEE/FIRE, KNOWLEDGE), triage (ASSIGN/REJECT/ASK_USER), final_review (COMPLETE/REVISE +cycle), error_review (RESOLVE_ERROR/CANCEL), bootstrap_team; post_user_message now delegates to it. - One billing path: call_model/BillingScope in runner.py is the single model entry point (fixes the agybot chat-billing bug). - Team changes capped at 3/turn, employee cap, last manager cannot be fired, hires pick a granted model of the requested tier strictly. - container_engine.py wires the real WorkforceBootstrapPort. - Fixes: REVISE is a body tag (prompt and parser now agree); ASSIGN employee="" means pick-for-me; the runner's protocol-error retry is capped by engine_max_protocol_retries (was unbounded until the turn timeout). - 20 new manager tests + regression tests; report in docs/plan/reports/agent-2-phase-3.md.
432 lines
15 KiB
Python
432 lines
15 KiB
Python
"""The tag protocol: parsing, the role/mode matrix and the per-reply limits.
|
|
|
|
What these tests defend:
|
|
|
|
* a tag pair with brackets inside its body is parsed as *one* tag with a
|
|
literal body — a source file full of ``[[`` must survive a round trip
|
|
* an unknown or disallowed tag never executes: it is dropped and reported so
|
|
the model can correct itself next turn
|
|
* the per-role matrix is exhaustive (every role/mode that acts has a row) and
|
|
the toolkit is granted to working roles only — the manager never edits code
|
|
* limits are enforced: three tasks, three team changes, three knowledge
|
|
items, and the extras are dropped rather than executed
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from app.application.engine.protocol import (
|
|
ALLOWED_TAGS,
|
|
MAX_CREATE_TASKS_PER_REPLY,
|
|
MAX_KNOWLEDGE_PER_TURN,
|
|
MAX_TEAM_CHANGES_PER_TURN,
|
|
ProtocolError,
|
|
TagName,
|
|
assign_of,
|
|
edit_blocks_of,
|
|
knowledge_items,
|
|
knowledge_of,
|
|
parse_reply,
|
|
task_draft_of,
|
|
task_drafts,
|
|
team_change_of,
|
|
team_changes,
|
|
)
|
|
from app.domain.workforce.models import RoleKey
|
|
from app.domain.workforce.pipeline import Mode
|
|
|
|
MANAGER_TRIAGE = {"role": RoleKey.MANAGER, "mode": Mode.TRIAGE}
|
|
MANAGER_FINAL = {"role": RoleKey.MANAGER, "mode": Mode.FINAL}
|
|
MANAGER_CHAT = {"role": RoleKey.MANAGER, "mode": Mode.CHAT}
|
|
DEVELOPER = {"role": RoleKey.DEVELOPER, "mode": Mode.WORK}
|
|
PLANNER = {"role": RoleKey.PLANNER, "mode": Mode.WORK}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The happy path
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_a_paired_tag_carries_its_attrs_and_body():
|
|
reply = parse_reply(
|
|
'سلام [[KNOWLEDGE kind="rule" key="style"]]همیشه از فارسی ساده استفاده کن.[[/KNOWLEDGE]]',
|
|
**MANAGER_TRIAGE,
|
|
)
|
|
assert reply.errors == ()
|
|
assert reply.prose == "سلام"
|
|
(tag,) = reply.of(TagName.KNOWLEDGE)
|
|
assert tag.attr("kind") == "rule"
|
|
assert tag.body == "همیشه از فارسی ساده استفاده کن."
|
|
items = knowledge_items(reply)
|
|
assert items[0].kind == "rule"
|
|
assert items[0].content == "همیشه از فارسی ساده استفاده کن."
|
|
|
|
|
|
def test_a_self_closing_tag_has_an_empty_body():
|
|
reply = parse_reply('[[ASSIGN employee="سارا" pipeline="plan_code_test"/]]', **MANAGER_TRIAGE)
|
|
assert reply.errors == ()
|
|
(tag,) = reply.tags
|
|
assert tag.name is TagName.ASSIGN
|
|
assert tag.body == ""
|
|
assign = assign_of(tag)
|
|
assert assign.employee == "سارا"
|
|
assert assign.pipeline == "plan_code_test"
|
|
|
|
|
|
def test_whitespace_and_case_are_tolerated():
|
|
reply = parse_reply(
|
|
'[[ assign employee = "سارا" / ]]',
|
|
**MANAGER_TRIAGE,
|
|
)
|
|
# Attribute values are read even with loose spacing...
|
|
assert reply.errors == ()
|
|
assert assign_of(reply.first(TagName.ASSIGN)).employee == "سارا"
|
|
|
|
|
|
def test_nested_brackets_in_a_body_stay_literal():
|
|
"""A file's content is not re-parsed as tags: the outer tag wins."""
|
|
body = 'def render():\n return "[[NOT_A_TAG]] and [[/NOT_A_TAG]]"'
|
|
reply = parse_reply(
|
|
f'[[WRITE_FILE path="app/render.ts"]]{body}[[/WRITE_FILE]]',
|
|
**DEVELOPER,
|
|
)
|
|
assert reply.errors == ()
|
|
(tag,) = reply.tags
|
|
assert tag.name is TagName.WRITE_FILE
|
|
assert tag.body == body
|
|
|
|
|
|
def test_several_tags_keep_their_order():
|
|
reply = parse_reply(
|
|
'[[READ_FILE path="a.py"]][[LIST_DIR path="."]][[RUN cmd="pytest" timeout="60"/]]',
|
|
**DEVELOPER,
|
|
)
|
|
assert reply.errors == ()
|
|
assert reply.names() == (TagName.READ_FILE, TagName.LIST_DIR, TagName.RUN)
|
|
assert reply.of(TagName.RUN)[0].attr("cmd") == "pytest"
|
|
assert reply.of(TagName.RUN)[0].attr("timeout") == "60"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Rejections, all of which the model hears about
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_unknown_tags_are_ignored_and_reported():
|
|
reply = parse_reply('[[HACK theme="green"]] payload [[/HACK]]done', **DEVELOPER)
|
|
assert reply.tags == ()
|
|
assert reply.errors == ("ERROR: unknown tag [[HACK]] ignored",)
|
|
# The unknown pair's body does not leak into the prose either.
|
|
assert reply.prose == "done"
|
|
|
|
|
|
def test_a_tag_outside_the_role_matrix_is_refused():
|
|
"""A manager cannot write files, and a developer cannot assign work."""
|
|
reply = parse_reply('[[WRITE_FILE path="x"]]hello[[/WRITE_FILE]]', **MANAGER_TRIAGE)
|
|
assert reply.tags == ()
|
|
assert reply.errors == (
|
|
"ERROR: tag [[WRITE_FILE]] is not allowed for role=manager mode=triage",
|
|
)
|
|
|
|
reply = parse_reply('[[ASSIGN employee="سارا"/]]', **DEVELOPER)
|
|
assert reply.tags == ()
|
|
assert reply.errors == (
|
|
"ERROR: tag [[ASSIGN]] is not allowed for role=developer mode=work",
|
|
)
|
|
|
|
|
|
def test_a_stray_closing_tag_is_reported_once():
|
|
reply = parse_reply("[[/WORK_REPORT]]", **DEVELOPER)
|
|
assert reply.tags == ()
|
|
assert reply.errors == ("ERROR: stray closing tag [[/WORK_REPORT]] ignored",)
|
|
|
|
|
|
def test_an_unclosed_tag_is_ignored_not_half_parsed():
|
|
reply = parse_reply("[[WORK_REPORT]] کار انجام شد بدون بستن", **DEVELOPER)
|
|
assert reply.tags == ()
|
|
assert any("unclosed tag [[WORK_REPORT]]" in e for e in reply.errors)
|
|
|
|
|
|
def test_malformed_attributes_are_reported_and_the_tag_keeps_going():
|
|
reply = parse_reply("[[ASSIGN employee سارا/]]", **MANAGER_TRIAGE)
|
|
assert any("malformed attribute fragment 'employee سارا'" in e for e in reply.errors)
|
|
# The tag itself is kept: the typed reader is what refuses it.
|
|
(tag,) = reply.tags
|
|
assert tag.name is TagName.ASSIGN
|
|
|
|
|
|
def test_an_empty_matrix_cell_refuses_everything():
|
|
"""A role/mode with no row gets no tags at all (deny by default)."""
|
|
reply = parse_reply("[[KNOWLEDGE]]x[[/KNOWLEDGE]]", role=RoleKey.RESEARCHER, mode=Mode.WORK)
|
|
assert reply.tags == ()
|
|
assert "not allowed" in reply.errors[0]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The role/mode matrix
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("role", "mode", "expected"),
|
|
[
|
|
(RoleKey.MANAGER, Mode.TRIAGE, {"ASSIGN", "REJECT", "ASK_USER", "KNOWLEDGE"}),
|
|
(RoleKey.MANAGER, Mode.FINAL, {"COMPLETE", "REVISE", "KNOWLEDGE"}),
|
|
(RoleKey.MANAGER, Mode.ERROR, {"RESOLVE_ERROR", "CANCEL", "KNOWLEDGE"}),
|
|
(
|
|
RoleKey.MANAGER,
|
|
Mode.CHAT,
|
|
{
|
|
"CREATE_TASK",
|
|
"PAUSE_TASK",
|
|
"RESUME_TASK",
|
|
"CANCEL_TASK",
|
|
"HIRE",
|
|
"UPDATE_EMPLOYEE",
|
|
"FIRE",
|
|
"KNOWLEDGE",
|
|
},
|
|
),
|
|
(
|
|
RoleKey.PLANNER,
|
|
Mode.WORK,
|
|
{"PLAN_SPEC", "SPLIT", "ASK_USER"}
|
|
| {"READ_FILE", "WRITE_FILE", "EDIT_FILE", "LIST_DIR", "SEARCH", "RUN"},
|
|
),
|
|
(
|
|
RoleKey.DEVELOPER,
|
|
Mode.WORK,
|
|
{"WORK_REPORT", "ASK_USER"}
|
|
| {"READ_FILE", "WRITE_FILE", "EDIT_FILE", "LIST_DIR", "SEARCH", "RUN"},
|
|
),
|
|
(
|
|
RoleKey.REVIEWER,
|
|
Mode.WORK,
|
|
{"REVIEW_PASS", "REVIEW_CHANGES"}
|
|
| {"READ_FILE", "WRITE_FILE", "EDIT_FILE", "LIST_DIR", "SEARCH", "RUN"},
|
|
),
|
|
(
|
|
RoleKey.TESTER,
|
|
Mode.WORK,
|
|
{"TEST_PASS", "TEST_FAIL"}
|
|
| {"READ_FILE", "WRITE_FILE", "EDIT_FILE", "LIST_DIR", "SEARCH", "RUN"},
|
|
),
|
|
],
|
|
)
|
|
def test_the_matrix_grants_exactly_the_planned_tags(role, mode, expected):
|
|
granted = {tag.value for tag in ALLOWED_TAGS[(role, mode)]}
|
|
assert granted == expected
|
|
|
|
|
|
def test_the_toolkit_is_available_to_working_roles_only():
|
|
toolkit = {"READ_FILE", "WRITE_FILE", "EDIT_FILE", "LIST_DIR", "SEARCH", "RUN"}
|
|
for (role, mode), granted in ALLOWED_TAGS.items():
|
|
has_tools = bool({t.value for t in granted} & toolkit)
|
|
if role is RoleKey.MANAGER:
|
|
assert has_tools is False, (role, mode)
|
|
else:
|
|
assert has_tools is True, (role, mode)
|
|
|
|
|
|
def test_manager_modes_have_distinct_tag_sets():
|
|
"""The same employee must wear different hats in different modes."""
|
|
triage = {t.value for t in ALLOWED_TAGS[(RoleKey.MANAGER, Mode.TRIAGE)]}
|
|
final = {t.value for t in ALLOWED_TAGS[(RoleKey.MANAGER, Mode.FINAL)]}
|
|
chat = {t.value for t in ALLOWED_TAGS[(RoleKey.MANAGER, Mode.CHAT)]}
|
|
assert triage != final != chat
|
|
assert "COMPLETE" in final and "COMPLETE" not in triage
|
|
assert "CREATE_TASK" in chat and "CREATE_TASK" not in triage
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Typed readers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_assign_requires_an_employee_or_a_task():
|
|
tag = parse_reply("[[ASSIGN/]]", **MANAGER_TRIAGE).first(TagName.ASSIGN)
|
|
with pytest.raises(ProtocolError):
|
|
assign_of(tag)
|
|
|
|
|
|
def test_an_empty_employee_means_pick_for_me():
|
|
"""``employee=""`` is a *deliberate* value, unlike an absent attribute."""
|
|
tag = parse_reply('[[ASSIGN employee="" pipeline="plan_code_test"/]]', **MANAGER_TRIAGE).first(
|
|
TagName.ASSIGN
|
|
)
|
|
assign = assign_of(tag)
|
|
assert assign.employee == ""
|
|
assert assign.pipeline == "plan_code_test"
|
|
|
|
|
|
def test_revise_carries_a_body():
|
|
"""The prompt asks for ``[[REVISE]]reason[[/REVISE]]``; so does the parser."""
|
|
reply = parse_reply("[[REVISE]]تستها قرمز است[[/REVISE]]", **MANAGER_FINAL)
|
|
assert reply.errors == ()
|
|
assert reply.first(TagName.REVISE).body == "تستها قرمز است"
|
|
|
|
|
|
def test_assign_refuses_an_unknown_pipeline():
|
|
reply = parse_reply('[[ASSIGN employee="سارا" pipeline="plan_ship_test"/]]', **MANAGER_TRIAGE)
|
|
with pytest.raises(ProtocolError):
|
|
assign_of(reply.first(TagName.ASSIGN))
|
|
|
|
|
|
def test_create_task_reads_dependencies_and_kind():
|
|
reply = parse_reply(
|
|
'[[CREATE_TASK title="نصب فرم ورود" kind="feature" priority="high" depends_on="t1,t2"]]'
|
|
"توضیحات کار[[/CREATE_TASK]]",
|
|
**MANAGER_CHAT,
|
|
)
|
|
(draft,) = task_drafts(reply)
|
|
assert draft.title == "نصب فرم ورود"
|
|
assert draft.kind == "feature"
|
|
assert draft.priority == "high"
|
|
assert draft.depends_on == ("t1", "t2")
|
|
assert draft.description == "توضیحات کار"
|
|
|
|
|
|
def test_create_task_without_a_title_is_refused():
|
|
tag = parse_reply(
|
|
"[[CREATE_TASK kind=\"feature\"]]x[[/CREATE_TASK]]", **MANAGER_CHAT
|
|
).first(TagName.CREATE_TASK)
|
|
with pytest.raises(ProtocolError):
|
|
task_draft_of(tag)
|
|
|
|
|
|
def test_team_changes_keep_their_order_across_the_three_tags():
|
|
reply = parse_reply(
|
|
'[[HIRE role="developer" tier="strong" name="سارا"]]'
|
|
"[[UPDATE_EMPLOYEE employee_id=\"e1\" tier=\"fast\"]]"
|
|
'[[FIRE employee_id="e2"]]',
|
|
**MANAGER_CHAT,
|
|
)
|
|
changes = team_changes(reply)
|
|
assert [c.action for c in changes] == [
|
|
TagName.HIRE,
|
|
TagName.UPDATE_EMPLOYEE,
|
|
TagName.FIRE,
|
|
]
|
|
assert changes[0].name == "سارا"
|
|
assert changes[0].tier == "strong"
|
|
# A model name is never part of the protocol: users pick role + tier.
|
|
for tag in reply.tags:
|
|
assert not any("model" in key for key in tag.attrs), tag
|
|
|
|
|
|
def test_knowledge_defaults_to_a_rule_and_reads_its_key():
|
|
reply = parse_reply(
|
|
'[[KNOWLEDGE key="deploy"]]قبل از پوش تست بگیر.[[/KNOWLEDGE]]', **MANAGER_CHAT
|
|
)
|
|
(item,) = knowledge_items(reply)
|
|
assert item.kind == "rule"
|
|
assert item.key == "deploy"
|
|
assert item.content == "قبل از پوش تست بگیر."
|
|
|
|
|
|
def test_edit_file_body_splits_into_search_replace_blocks():
|
|
body = (
|
|
"before\n"
|
|
"<<<OLD\n"
|
|
"old line\n"
|
|
"===\n"
|
|
"new line\n"
|
|
"NEW>>>\n"
|
|
"after\n"
|
|
"<<<OLD\n"
|
|
"second old\n"
|
|
"===\n"
|
|
"second new\n"
|
|
"NEW>>>"
|
|
)
|
|
reply = parse_reply(f'[[EDIT_FILE path="app/x.py"]]{body}[[/EDIT_FILE]]', **DEVELOPER)
|
|
(tag,) = reply.of(TagName.EDIT_FILE)
|
|
blocks = edit_blocks_of(tag)
|
|
assert [b.old for b in blocks] == ["old line", "second old"]
|
|
assert [b.new for b in blocks] == ["new line", "second new"]
|
|
|
|
|
|
def test_edit_file_without_blocks_is_refused():
|
|
reply = parse_reply(
|
|
"[[EDIT_FILE path=\"a\"]]just text[[/EDIT_FILE]]", **DEVELOPER
|
|
)
|
|
with pytest.raises(ProtocolError):
|
|
edit_blocks_of(reply.first(TagName.EDIT_FILE))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Limits
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_at_most_three_tasks_per_reply():
|
|
tasks = "".join(
|
|
f'[[CREATE_TASK title="کار {i}"]]x[[/CREATE_TASK]]' for i in range(3)
|
|
)
|
|
reply = parse_reply(tasks, **MANAGER_CHAT)
|
|
assert len(task_drafts(reply)) == MAX_CREATE_TASKS_PER_REPLY
|
|
assert reply.errors == ()
|
|
|
|
|
|
def test_a_fourth_task_is_dropped_and_reported():
|
|
tasks = "".join(
|
|
f'[[CREATE_TASK title="کار {i}"]]x[[/CREATE_TASK]]' for i in range(4)
|
|
)
|
|
reply = parse_reply(tasks, **MANAGER_CHAT)
|
|
assert len(task_drafts(reply)) == 3
|
|
assert len(reply.errors) == 1
|
|
assert "too many [[CREATE_TASK]]" in reply.errors[0]
|
|
assert f"max {MAX_CREATE_TASKS_PER_REPLY}" in reply.errors[0]
|
|
|
|
|
|
def test_three_team_changes_per_turn_across_all_three_tags():
|
|
reply = parse_reply(
|
|
'[[HIRE role="developer" name="الف"]]'
|
|
'[[HIRE role="tester" name="ب"]]'
|
|
'[[HIRE role="reviewer" name="ج"]]'
|
|
'[[FIRE employee_id="e9"]]',
|
|
**MANAGER_CHAT,
|
|
)
|
|
assert len(team_changes(reply)) == MAX_TEAM_CHANGES_PER_TURN
|
|
assert any("too many [[FIRE]]" in e for e in reply.errors)
|
|
|
|
|
|
def test_three_knowledge_items_per_turn():
|
|
items = "".join(f'[[KNOWLEDGE key="k{i}"]]متن {i}[[/KNOWLEDGE]]' for i in range(4))
|
|
reply = parse_reply(items, **MANAGER_CHAT)
|
|
assert len(knowledge_items(reply)) == MAX_KNOWLEDGE_PER_TURN
|
|
assert any("too many [[KNOWLEDGE]]" in e for e in reply.errors)
|
|
|
|
|
|
def test_limits_do_not_cross_feed_each_other():
|
|
"""Team changes and knowledge are counted separately from tasks."""
|
|
reply = parse_reply(
|
|
'[[CREATE_TASK title="a"]]x[[/CREATE_TASK]]'
|
|
'[[CREATE_TASK title="b"]]x[[/CREATE_TASK]]'
|
|
'[[HIRE role="developer" name="سارا"]]'
|
|
'[[KNOWLEDGE key="k"]]متن[[/KNOWLEDGE]]',
|
|
**MANAGER_CHAT,
|
|
)
|
|
assert len(task_drafts(reply)) == 2
|
|
assert len(team_changes(reply)) == 1
|
|
assert len(knowledge_items(reply)) == 1
|
|
assert reply.errors == ()
|
|
|
|
|
|
def test_typed_readers_work_on_one_tag_at_a_time():
|
|
"""The runner reads single tags too, not only the whole reply."""
|
|
reply = parse_reply(
|
|
'[[HIRE role="developer" tier="balanced" name="سارا"/]]', **MANAGER_CHAT
|
|
)
|
|
change = team_change_of(reply.first(TagName.HIRE))
|
|
assert change.action is TagName.HIRE
|
|
assert change.role == "developer"
|
|
|
|
reply = parse_reply(
|
|
'[[KNOWLEDGE kind="fact" key="port"]]پورت ۸۰۰۰ است.[[/KNOWLEDGE]]', **MANAGER_CHAT
|
|
)
|
|
item = knowledge_of(reply.first(TagName.KNOWLEDGE))
|
|
assert item.kind == "fact"
|
|
assert item.content == "پورت ۸۰۰۰ است."
|