fix(cleanup): the review's defects on the R-a diff
The trekcontinue-stats line_ref follows Phase 5 to 271-275 (the Phase 1.5 cut moved it). gold-eval.test.mjs and its run fixture come back: they are the one end-to-end check that the review contract keeps every seeded finding, and they stay with the scorer until R-b. CHANGELOG: the removed validator leaves the known-issues text; the review's other findings outside R-a's files are listed as known issues. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
fb937ee306
commit
5f42bbdb9e
4 changed files with 73 additions and 5 deletions
17
tests/fixtures/bakeoff-rich/runs/run-perfect.json
vendored
Normal file
17
tests/fixtures/bakeoff-rich/runs/run-perfect.json
vendored
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
[
|
||||
{
|
||||
"reviewer": "brief-conformance-reviewer",
|
||||
"findings": [
|
||||
{ "severity": "BLOCKER", "rule_key": "UNIMPLEMENTED_CRITERION", "file": "lib/handlers/login.mjs", "line": 17 },
|
||||
{ "severity": "MAJOR", "rule_key": "PLAN_EXECUTE_DRIFT", "file": "lib/handlers/login.mjs", "line": 13 }
|
||||
]
|
||||
},
|
||||
{
|
||||
"reviewer": "code-correctness-reviewer",
|
||||
"findings": [
|
||||
{ "severity": "BLOCKER", "rule_key": "SECURITY_INJECTION", "file": "lib/auth/jwt.mjs", "line": 19 },
|
||||
{ "severity": "MAJOR", "rule_key": "MISSING_TEST", "file": "lib/auth/refresh.mjs", "line": 0 },
|
||||
{ "severity": "MINOR", "rule_key": "MISSING_ERROR_HANDLING", "file": "lib/auth/refresh.mjs", "line": 10 }
|
||||
]
|
||||
}
|
||||
]
|
||||
2
tests/fixtures/jsonl-schemas.md
vendored
2
tests/fixtures/jsonl-schemas.md
vendored
|
|
@ -26,7 +26,7 @@
|
|||
| trekexecute-stats (autonomy events) | ts, event, known_event, payload | lib/stats/event-emit.mjs `emit()` | event-emit.mjs:78-99 | payload.profile, payload.phase_models, payload.profile_source | none |
|
||||
| trekexecute-stats (PostToolUse Bash) | ts, session_id, command_excerpt, duration_ms, success | hooks/scripts/post-bash-stats.mjs (Bash PostToolUse) | post-bash-stats.mjs:42-54 | none (hook is plugin-level, not profile-aware) | command_excerpt (CWE-212) |
|
||||
| trekreview-stats | ts, slug, verdict, counts (BLOCKER/MAJOR/MINOR/SUGGESTION), reviewed_files_count, mode, duration_ms | lib/stats/stats-append.mjs (trekreview.md Phase 8) | trekreview.md:682-735 | profile, phase_models, profile_source | none |
|
||||
| trekcontinue-stats | ts, project, next_session_label, status | lib/stats/stats-append.mjs (trekcontinue.md Phase 5) | trekcontinue.md:304-308 | profile, profile_source | none |
|
||||
| trekcontinue-stats | ts, project, next_session_label, status | lib/stats/stats-append.mjs (trekcontinue.md Phase 5) | trekcontinue.md:271-275 | profile, profile_source | none |
|
||||
| token-usage | ts, session_id, scope, model, tokens_input, tokens_output, tokens_cache_creation, tokens_cache_read, cost_usd, is_estimate, price_table_version | lib/stats/token-usage.mjs `captureTokenUsage` (via hooks/scripts/otel-export.mjs Stop hook, opt-in `VOYAGE_TOKEN_METER`) | token-usage.mjs:buildRecord | n/a (SKAL-2 schema) | session_id (stripped at export) |
|
||||
|
||||
The `main-merge-declined` event's `payload.reason` (the autonomy-events row) is one of the literal words of
|
||||
|
|
|
|||
49
tests/lib/gold-eval.test.mjs
Normal file
49
tests/lib/gold-eval.test.mjs
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
// tests/lib/gold-eval.test.mjs
|
||||
// SKAL-1·4b — the offline gold-scored output eval (the scoring RUN).
|
||||
//
|
||||
// This is the third test-census category (see lib/util/test-census.mjs):
|
||||
// neither a behavior unit test nor a doc-consistency pin, but a SCORING RUN —
|
||||
// it feeds a committed agent-run fixture through the deterministic coordinator
|
||||
// contract (4a) and scores the result against the golden corpus at
|
||||
// (file, rule_key) granularity. Offline: committed reviewer payloads, no live
|
||||
// agent spawn, no LLM, no network. The all-agree foundation; the
|
||||
// LLM-in-the-loop eval is the separate 4c tier.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { runContract } from '../../lib/review/coordinator-contract.mjs';
|
||||
import { scoreFindings, scoreVerdict } from '../../lib/review/gold-scorer.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const gold = JSON.parse(readFileSync(join(ROOT, 'tests/fixtures/bakeoff-rich/gold.json'), 'utf-8'));
|
||||
const runPerfect = JSON.parse(
|
||||
readFileSync(join(ROOT, 'tests/fixtures/bakeoff-rich/runs/run-perfect.json'), 'utf-8'),
|
||||
);
|
||||
|
||||
test('gold-eval — run-perfect reproduces gold at (file,rule_key): precision/recall/f1 = 1.0', () => {
|
||||
const result = runContract(runPerfect);
|
||||
const score = scoreFindings(result.findings, gold.findings);
|
||||
assert.equal(score.precision, 1, `precision ${score.precision}; spurious=${JSON.stringify(score.spurious)}`);
|
||||
assert.equal(score.recall, 1, `recall ${score.recall}; missed=${JSON.stringify(score.missed)}`);
|
||||
assert.equal(score.f1, 1);
|
||||
assert.equal(score.tp, gold.findings.length);
|
||||
});
|
||||
|
||||
test('gold-eval — run-perfect coordinator verdict matches gold expected_verdict (BLOCK)', () => {
|
||||
const result = runContract(runPerfect);
|
||||
assert.equal(result.verdict, gold.expected_verdict);
|
||||
assert.ok(scoreVerdict(result.verdict, gold.expected_verdict));
|
||||
});
|
||||
|
||||
test('gold-eval — run-perfect drops nothing (no suppressed, no schema-skipped payloads)', () => {
|
||||
// A clean run is the regression guard: any future contract change that
|
||||
// silently suppresses or skips one of the 5 seeded findings breaks here.
|
||||
const result = runContract(runPerfect);
|
||||
assert.equal(result.skipped.length, 0, `skipped payloads: ${JSON.stringify(result.skipped)}`);
|
||||
assert.equal(result.suppressed.length, 0, `suppressed findings: ${result.suppressed.length}`);
|
||||
assert.equal(result.findings.length, gold.findings.length);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue