feat(check): the checker and the contract read a folder's reply
`okf check --payload` takes the reply to one call over a folder as well as a single payload: every bundle's payload is held to all 19 rules on its own, a finding is named with its bundle, one every payload carries alike is reported once, an answer labelled with a bundle its payload does not describe is `answer_misattributed`, and a reply with no answer is `payload_invalid`. No rule is added, and a single payload's report is unchanged. Contract SS 2.5.4 names the folder run and SS 8.11 fixes the reply; the known-positive moves to 24 620 / delta 592. The skill text follows: the working method's steps 1 and 4 name the folder, and the generic skill says to use the server's tools first where they are registered, with the skill as the supplement. The folder is an instruction in both generators, never a path: the bundle's parent written absolute named this checkout, and the test holding generated commands to no repository path fell on it. v1.1 order F, part F4. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
570496470b
commit
21f9241712
10 changed files with 240 additions and 29 deletions
|
|
@ -214,3 +214,57 @@ def test_one_bundle_is_read_as_before(folder: Path) -> None:
|
|||
payload = json.loads(run.stdout)
|
||||
assert "answers" not in payload
|
||||
assert payload["bundle"]["bundle_id"] == "hage"
|
||||
|
||||
|
||||
# --- F4: the checker reads the folder's reply --------------------------------
|
||||
|
||||
|
||||
def _check(tmp_path: Path, reply: object) -> subprocess.CompletedProcess[str]:
|
||||
from llm_ingestion_okf import skill
|
||||
|
||||
skill_path = tmp_path / "SKILL.md"
|
||||
skill_path.write_text(skill.render_generic(), encoding="utf-8")
|
||||
payload_path = tmp_path / "reply.json"
|
||||
payload_path.write_text(json.dumps(reply, ensure_ascii=False), encoding="utf-8")
|
||||
return _okf("check", "--skill", str(skill_path), "--payload", str(payload_path))
|
||||
|
||||
|
||||
def _reply(folder: Path) -> dict[str, object]:
|
||||
return mcp_server.call_ask(_surface(folder), {"questions": list(QUESTIONS)})
|
||||
|
||||
|
||||
def test_the_generic_skill_is_conformant_on_a_folders_reply(folder: Path, tmp_path: Path) -> None:
|
||||
run = _check(tmp_path, _reply(folder))
|
||||
assert run.returncode == 0, run.stdout
|
||||
assert run.stdout.startswith("conformant: ")
|
||||
assert "over 2 payloads" in run.stdout
|
||||
|
||||
|
||||
def test_an_answer_labelled_with_another_bundle_is_a_finding(folder: Path, tmp_path: Path) -> None:
|
||||
reply = _reply(folder)
|
||||
answers = reply["answers"]
|
||||
assert isinstance(answers, list)
|
||||
answers[0]["bundle_id"] = "hage"
|
||||
run = _check(tmp_path, reply)
|
||||
assert run.returncode == 1
|
||||
assert "answer_misattributed" in run.stdout
|
||||
|
||||
|
||||
def test_a_defect_in_one_answer_is_named_with_its_bundle(folder: Path, tmp_path: Path) -> None:
|
||||
reply = _reply(folder)
|
||||
answers = reply["answers"]
|
||||
assert isinstance(answers, list)
|
||||
del answers[1]["payload"]["contract"]
|
||||
run = _check(tmp_path, reply)
|
||||
assert run.returncode == 1
|
||||
findings = [line for line in run.stdout.splitlines() if line.startswith(" ")]
|
||||
assert findings == [
|
||||
line for line in findings if line.startswith(" contract_unversioned: [hage]")
|
||||
]
|
||||
assert len(findings) == 1
|
||||
|
||||
|
||||
def test_a_reply_with_no_answer_is_a_finding_not_a_pass(tmp_path: Path) -> None:
|
||||
run = _check(tmp_path, {"asked": [], "answers": []})
|
||||
assert run.returncode == 1
|
||||
assert "payload_invalid" in run.stdout
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue