fix(storm-measure): check BOTH halves of the activation SC, not just the count delta
The SC asks two things of an effort: high run: that it discovered at least one dimension AND that the dimension list in the output brief is a TRUE SUPERSET of the interview-derived ones. activationCheck computed dimensions - dimensions_baseline >= 1 and returned ok on that alone. A count delta says nothing about membership: a run that dropped two interview dimensions and appended three discovered ones is +1 and passed the check while violating the second half outright. Supersetness was asserted only by Phase 4.5's prose contract that discovery appends - nothing read it. The stats record cannot carry the dimension names that would show it directly: names are free prose, and lib/exporters/field-allowlist.mjs denies prose by omission (question, project_dir, brief_path are already excluded for that reason). So the run attests membership with dimensions_baseline_preserved, a boolean set in a new Phase 4.5 step 5, and the gate refuses to call activation OK without it. An ABSENT attestation fails rather than falling back to the old count-only rule - otherwise legacy rows would keep passing on the defect. Producer side wired end to end: the record format, the measurement-fields prose (five fields -> six), the exporter allowlist, the jsonl-schemas fixture row, and the --activation-check comment in docs/storm-measurement.md. Review finding d2786604458207a5a73478cdcb6a54bbdb92141d (MINOR). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LuGhWAbWyRFBFeemfhxoVv
This commit is contained in:
parent
7d5e4fcbcc
commit
dc245af408
6 changed files with 145 additions and 14 deletions
2
tests/fixtures/jsonl-schemas.md
vendored
2
tests/fixtures/jsonl-schemas.md
vendored
|
|
@ -20,7 +20,7 @@
|
|||
| schema_id | fields | writer_path | line_ref | v4.1 additive | PII |
|
||||
|-----------|--------|-------------|----------|---------------|-----|
|
||||
| trekbrief-stats | ts, task, slug, mode, interview_turns, review_iterations, brief_quality, research_topics, auto_research, auto_result, project_dir | commands/trekbrief.md (orchestrator-emit Phase 7) | trekbrief.md:657-672 | profile, phase_models, profile_source | none |
|
||||
| trekresearch-stats | ts, question, mode, scope, engine, slug, project_dir, brief_path, dimensions, dimensions_baseline, effort, conv_turns, empty_turns, unique_sources, agents_local, agents_external, gemini_used, confidence, contradictions, open_questions | commands/trekresearch.md (orchestrator-emit Stats tracking) | trekresearch.md:634-676 | profile, phase_models, parallel_agents, external_research_enabled, profile_source | none |
|
||||
| trekresearch-stats | ts, question, mode, scope, engine, slug, project_dir, brief_path, dimensions, dimensions_baseline, dimensions_baseline_preserved, effort, conv_turns, empty_turns, unique_sources, agents_local, agents_external, gemini_used, confidence, contradictions, open_questions | commands/trekresearch.md (orchestrator-emit Stats tracking) | trekresearch.md:634-676 | profile, phase_models, parallel_agents, external_research_enabled, profile_source | none |
|
||||
| trekplan-stats | ts, task, mode, slug, brief_path, project_dir, codebase_size, codebase_files, agents_deployed, deep_dives, research_briefs_used, research_scout_used, critic_verdict, guardian_verdict, outcome | commands/trekplan.md (orchestrator-emit Phase 12) | trekplan.md:805-826 | profile, phase_models, parallel_agents, profile_source | none |
|
||||
| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | commands/trekexecute.md (orchestrator-emit Phase 9) | trekexecute.md:1479-1494 | profile, phase_models, profile_source | none |
|
||||
| trekexecute-stats (autonomy events) | ts, event, known_event, payload | lib/stats/event-emit.mjs `emit()` | event-emit.mjs:64-86 | payload.profile, payload.phase_models, payload.profile_source | none |
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import {
|
|||
partitionEligible,
|
||||
measure,
|
||||
decideVerdict,
|
||||
activationCheck,
|
||||
ADOPT_THRESHOLD,
|
||||
DECLINE_THRESHOLD,
|
||||
} from '../../scripts/storm-measure.mjs';
|
||||
|
|
@ -235,6 +236,73 @@ test('decideVerdict: a null gain is insufficient data, never a decline', () => {
|
|||
assert.equal(decideVerdict(0.4, null), 'insufficient-data');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// activationCheck — BOTH halves of the SC, not just the count
|
||||
//
|
||||
// The SC requires that an `effort: high` run discovers at least one dimension
|
||||
// AND that the dimension list in the output brief is a TRUE SUPERSET of the
|
||||
// interview-derived ones. activationCheck only computed dimensions -
|
||||
// dimensions_baseline >= 1, so a run that replaced two interview dimensions with
|
||||
// three discovered ones passed the check while violating the SC's second half.
|
||||
// Supersetness was asserted only by Phase 4.5's prose contract that discovery
|
||||
// appends; nothing read it.
|
||||
//
|
||||
// The stats record carries counts, not names — names are free prose and
|
||||
// field-allowlist.mjs denies prose by omission — so the run attests membership
|
||||
// with a low-cardinality boolean instead, and the gate refuses to call
|
||||
// activation OK without it.
|
||||
// ---------------------------------------------------------------------------
|
||||
function highRun(over = {}) {
|
||||
return JSON.stringify({
|
||||
ts: '2026-08-12T00:00:00.000Z',
|
||||
effort: 'high',
|
||||
unique_sources: 13,
|
||||
dimensions: 8,
|
||||
dimensions_baseline: 5,
|
||||
conv_turns: 3,
|
||||
empty_turns: 0,
|
||||
dimensions_baseline_preserved: true,
|
||||
...over,
|
||||
});
|
||||
}
|
||||
|
||||
test('activationCheck: discovery plus a preserved baseline is activation', () => {
|
||||
const r = activationCheck(parseStats(jsonl(highRun())).records);
|
||||
assert.equal(r.ok, true);
|
||||
assert.equal(r.discovered_dimensions, 3);
|
||||
assert.equal(r.dimensions_baseline_preserved, true);
|
||||
});
|
||||
|
||||
test('activationCheck: a REPLACED baseline is not activation, however many were discovered', () => {
|
||||
const r = activationCheck(parseStats(jsonl(
|
||||
highRun({ dimensions: 8, dimensions_baseline: 5, dimensions_baseline_preserved: false }),
|
||||
)).records);
|
||||
assert.equal(r.ok, false, 'a count delta of +3 says nothing about which dimensions survived');
|
||||
assert.match(r.reason, /superset|baseline/i);
|
||||
});
|
||||
|
||||
test('activationCheck: a run that does not attest baseline membership cannot pass', () => {
|
||||
const rec = JSON.parse(highRun());
|
||||
delete rec.dimensions_baseline_preserved;
|
||||
const r = activationCheck(parseStats(jsonl(JSON.stringify(rec))).records);
|
||||
assert.equal(r.ok, false, 'an absent attestation is not an attestation');
|
||||
assert.match(r.reason, /dimensions_baseline_preserved/);
|
||||
});
|
||||
|
||||
test('activationCheck: no discovery is still not activation even with the baseline preserved', () => {
|
||||
const r = activationCheck(parseStats(jsonl(
|
||||
highRun({ dimensions: 5, dimensions_baseline: 5 }),
|
||||
)).records);
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.discovered_dimensions, 0);
|
||||
});
|
||||
|
||||
test('activationCheck: no effort:high run at all is reported as such', () => {
|
||||
const r = activationCheck(parseStats(jsonl(...STANDARD)).records);
|
||||
assert.equal(r.ok, false);
|
||||
assert.match(r.reason, /high/);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// pre-registration — the thresholds are the doc's, and the doc is committed first
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue