feat(check): the checker and the contract read a folder's reply

`okf check --payload` takes the reply to one call over a folder as well
as a single payload: every bundle's payload is held to all 19 rules on
its own, a finding is named with its bundle, one every payload carries
alike is reported once, an answer labelled with a bundle its payload
does not describe is `answer_misattributed`, and a reply with no answer
is `payload_invalid`. No rule is added, and a single payload's report is
unchanged. Contract SS 2.5.4 names the folder run and SS 8.11 fixes the
reply; the known-positive moves to 24 620 / delta 592.

The skill text follows: the working method's steps 1 and 4 name the
folder, and the generic skill says to use the server's tools first where
they are registered, with the skill as the supplement. The folder is an
instruction in both generators, never a path: the bundle's parent
written absolute named this checkout, and the test holding generated
commands to no repository path fell on it.

v1.1 order F, part F4.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-21 10:50:14 +02:00
commit 21f9241712
10 changed files with 240 additions and 29 deletions

View file

@ -214,3 +214,57 @@ def test_one_bundle_is_read_as_before(folder: Path) -> None:
payload = json.loads(run.stdout)
assert "answers" not in payload
assert payload["bundle"]["bundle_id"] == "hage"
# --- F4: the checker reads the folder's reply --------------------------------
def _check(tmp_path: Path, reply: object) -> subprocess.CompletedProcess[str]:
from llm_ingestion_okf import skill
skill_path = tmp_path / "SKILL.md"
skill_path.write_text(skill.render_generic(), encoding="utf-8")
payload_path = tmp_path / "reply.json"
payload_path.write_text(json.dumps(reply, ensure_ascii=False), encoding="utf-8")
return _okf("check", "--skill", str(skill_path), "--payload", str(payload_path))
def _reply(folder: Path) -> dict[str, object]:
return mcp_server.call_ask(_surface(folder), {"questions": list(QUESTIONS)})
def test_the_generic_skill_is_conformant_on_a_folders_reply(folder: Path, tmp_path: Path) -> None:
run = _check(tmp_path, _reply(folder))
assert run.returncode == 0, run.stdout
assert run.stdout.startswith("conformant: ")
assert "over 2 payloads" in run.stdout
def test_an_answer_labelled_with_another_bundle_is_a_finding(folder: Path, tmp_path: Path) -> None:
reply = _reply(folder)
answers = reply["answers"]
assert isinstance(answers, list)
answers[0]["bundle_id"] = "hage"
run = _check(tmp_path, reply)
assert run.returncode == 1
assert "answer_misattributed" in run.stdout
def test_a_defect_in_one_answer_is_named_with_its_bundle(folder: Path, tmp_path: Path) -> None:
reply = _reply(folder)
answers = reply["answers"]
assert isinstance(answers, list)
del answers[1]["payload"]["contract"]
run = _check(tmp_path, reply)
assert run.returncode == 1
findings = [line for line in run.stdout.splitlines() if line.startswith(" ")]
assert findings == [
line for line in findings if line.startswith(" contract_unversioned: [hage]")
]
assert len(findings) == 1
def test_a_reply_with_no_answer_is_a_finding_not_a_pass(tmp_path: Path) -> None:
run = _check(tmp_path, {"asked": [], "answers": []})
assert run.returncode == 1
assert "payload_invalid" in run.stdout