chore(cleanup): R-b R2 — STORM loop, cap hook and the --engine research choice removed
/trekresearch loses Phase 4.5, the bounded Phase 5 loop (Phase 5 keeps its single pass of at most 2 follow-ups), the --engine flag, the deep-research adapter and its output self-check. hooks.json loses the pre-agent-cap PreToolUse entry (README: 9 hooks = 9 scripts). The STORM env vars leave the docs; query-privacy-gate.mjs stays, called by no command, and says so. Files removed: 13 (4 docs, 1 hook, 1 lib module, 1 script, 5 tests, 1 fixture). Files edited: 17. Tests: 2600 (2598 pass / 0 fail / 2 skipped) before, 2467 (2466 pass / 0 fail / 1 skipped, R3 pending) after. Durable check: group R2 green and no longer pending. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
21480e8627
commit
32621dcb63
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
30 changed files with 34 additions and 4070 deletions
|
|
@ -1,343 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
// scripts/storm-measure.mjs
|
||||
// Step 11 — the STORM adoption gate: deterministic Δ accounting over
|
||||
// ${CLAUDE_PLUGIN_DATA}/trekresearch-stats.jsonl.
|
||||
//
|
||||
// What this decides: whether the bounded Phase 5 conversation loop buys enough
|
||||
// extra source/coverage breadth to justify flipping VOYAGE_STORM_ENABLED on by
|
||||
// default in lib/util/research-loop-cap.mjs. Nothing else. The thresholds live
|
||||
// in docs/storm-measurement.md and were committed BEFORE any measurement run —
|
||||
// that pre-registration is the whole point, so this script never invents them.
|
||||
//
|
||||
// What this does NOT measure: outline quality, answer correctness, or operator
|
||||
// satisfaction. It measures breadth (distinct sources, dimensions covered).
|
||||
// A breadth win is necessary for adoption, not sufficient on its own.
|
||||
//
|
||||
// Honesty properties, both load-bearing:
|
||||
// - Runs with empty_turns > 0 are EXCLUDED from the gain and REPORTED. An
|
||||
// empty turn means the loop spent budget and returned nothing; leaving those
|
||||
// in decides adoption on a broken denominator.
|
||||
// - A stats file carrying no `effort` field is a loud error. The silent
|
||||
// failure this prevents is an empty treatment group reading as "no gain",
|
||||
// which would decline the mechanism for a schema reason.
|
||||
//
|
||||
// Zero deps. Node stdlib only.
|
||||
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { isMain } from '../lib/util/is-main.mjs';
|
||||
|
||||
// Pre-registered in docs/storm-measurement.md. Do not tune to fit a result.
|
||||
export const ADOPT_THRESHOLD = 0.30;
|
||||
export const DECLINE_THRESHOLD = 0.15;
|
||||
|
||||
const STATS_FILENAME = 'trekresearch-stats.jsonl';
|
||||
|
||||
// ---- pure core (unit-tested) -------------------------------------------------
|
||||
|
||||
/** @param {number[]} xs @returns {number|null} null for an empty list — 0 would read as a measurement. */
|
||||
export function median(xs) {
|
||||
if (!Array.isArray(xs) || xs.length === 0) return null;
|
||||
const s = [...xs].sort((a, b) => a - b);
|
||||
const mid = s.length >> 1;
|
||||
return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a trekresearch-stats.jsonl body into effort-carrying records.
|
||||
*
|
||||
* Rows predating Step 9 have no `effort` field; they are dropped and counted as
|
||||
* `legacy` rather than silently pooled into the control arm. A file where NO row
|
||||
* carries `effort` throws — see the header note on silent failure.
|
||||
*
|
||||
* @param {string} text
|
||||
* @returns {{records: object[], malformed: number, legacy: number}}
|
||||
*/
|
||||
export function parseStats(text) {
|
||||
const lines = String(text ?? '').split('\n');
|
||||
const records = [];
|
||||
let malformed = 0;
|
||||
let legacy = 0;
|
||||
let parsed = 0;
|
||||
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
let rec;
|
||||
try {
|
||||
rec = JSON.parse(line);
|
||||
} catch {
|
||||
malformed++;
|
||||
continue;
|
||||
}
|
||||
parsed++;
|
||||
if (typeof rec?.effort !== 'string' || rec.effort.length === 0) {
|
||||
legacy++;
|
||||
continue;
|
||||
}
|
||||
records.push(rec);
|
||||
}
|
||||
|
||||
if (parsed === 0) {
|
||||
throw new Error(`storm-measure: no records in stats file (${malformed} malformed line(s)).`);
|
||||
}
|
||||
if (records.length === 0) {
|
||||
throw new Error(
|
||||
`storm-measure: no records carry an \`effort\` field (${legacy} legacy row(s)). ` +
|
||||
`The gate groups on \`effort\`; without it there is no treatment arm to measure. ` +
|
||||
`Re-run the measurement set on a build that emits the Step 9 fields.`,
|
||||
);
|
||||
}
|
||||
return { records, malformed, legacy };
|
||||
}
|
||||
|
||||
/**
|
||||
* Split off the runs that must not count toward a gain.
|
||||
*
|
||||
* A value that cannot be READ as a turn count is excluded, not treated as zero.
|
||||
* `Number('many')` is NaN, and testing `Number.isFinite(empty) && empty > 0`
|
||||
* sent NaN down the eligible branch — so a garbage field silently re-entered the
|
||||
* denominator, in the direction that flatters adoption: the run whose bookkeeping
|
||||
* broke is the run whose numbers deserve the least trust. Absent and null stay
|
||||
* eligible via `?? 0`, because a field that was never written is a genuine zero
|
||||
* on any run where the loop did not arm.
|
||||
*
|
||||
* @param {object[]} records
|
||||
* @returns {{eligible: object[], excluded: number}}
|
||||
*/
|
||||
export function partitionEligible(records) {
|
||||
const eligible = [];
|
||||
let excluded = 0;
|
||||
for (const r of records) {
|
||||
const empty = Number(r.empty_turns ?? 0);
|
||||
if (!Number.isFinite(empty) || empty > 0) excluded++;
|
||||
else eligible.push(r);
|
||||
}
|
||||
return { eligible, excluded };
|
||||
}
|
||||
|
||||
function numbers(records, pick) {
|
||||
return records.map(pick).filter((n) => Number.isFinite(n));
|
||||
}
|
||||
|
||||
/** Relative gain (treatment − control)/control. null when control is absent or zero. */
|
||||
function relGain(control, treatment) {
|
||||
if (control === null || treatment === null || control === 0) return null;
|
||||
return (treatment - control) / control;
|
||||
}
|
||||
|
||||
/**
|
||||
* Full measurement over a parsed record set.
|
||||
*
|
||||
* Treatment arm = `effort: high` (the only effort at which the loop runs).
|
||||
* Control arm = every other effort.
|
||||
*
|
||||
* - sources: between-arm median gain in `unique_sources`.
|
||||
* - dimensions: within-run median gain of (dimensions − dimensions_baseline)
|
||||
* / dimensions_baseline across the treatment arm. It is a
|
||||
* within-run delta by construction, so it needs no control arm —
|
||||
* the control arm's is 0, the loop being inert there.
|
||||
*
|
||||
* @param {object[]} records
|
||||
*/
|
||||
export function measure(records) {
|
||||
const { eligible, excluded } = partitionEligible(records);
|
||||
const treatment = eligible.filter((r) => r.effort === 'high');
|
||||
const control = eligible.filter((r) => r.effort !== 'high');
|
||||
|
||||
const srcControl = median(numbers(control, (r) => Number(r.unique_sources)));
|
||||
const srcTreatment = median(numbers(treatment, (r) => Number(r.unique_sources)));
|
||||
const sourcesGain = relGain(srcControl, srcTreatment);
|
||||
|
||||
const dimDeltas = treatment
|
||||
.map((r) => ({ d: Number(r.dimensions), b: Number(r.dimensions_baseline) }))
|
||||
.filter(({ d, b }) => Number.isFinite(d) && Number.isFinite(b) && b > 0)
|
||||
.map(({ d, b }) => (d - b) / b);
|
||||
const dimensionsGain = median(dimDeltas);
|
||||
|
||||
return {
|
||||
control: { n: control.length, sources: srcControl },
|
||||
treatment: { n: treatment.length, sources: srcTreatment },
|
||||
sources: { control: srcControl, treatment: srcTreatment, gain: sourcesGain },
|
||||
dimensions: { gain: dimensionsGain, n: dimDeltas.length },
|
||||
excluded,
|
||||
verdict: decideVerdict(sourcesGain, dimensionsGain),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Pre-registered mapping, verbatim from the brief: "median forbedring >= 30 %
|
||||
* på (a) eller (b) → adopt. < 15 % → decline." OR on both sides, adopt
|
||||
* evaluated first — so a strong win on one axis is an adopt even when the
|
||||
* other axis sits under the decline bar. A stricter AND rule may well be the
|
||||
* better decision procedure, but changing it here is changing the
|
||||
* pre-registration after the fact, which is the one thing the constraint
|
||||
* exists to prevent.
|
||||
*
|
||||
* @param {number|null} sourcesGain
|
||||
* @param {number|null} dimensionsGain
|
||||
* @returns {'adopt'|'decline'|'inconclusive'|'insufficient-data'}
|
||||
*/
|
||||
export function decideVerdict(sourcesGain, dimensionsGain) {
|
||||
if (sourcesGain === null || sourcesGain === undefined) return 'insufficient-data';
|
||||
if (dimensionsGain === null || dimensionsGain === undefined) return 'insufficient-data';
|
||||
if (sourcesGain >= ADOPT_THRESHOLD || dimensionsGain >= ADOPT_THRESHOLD) return 'adopt';
|
||||
if (sourcesGain < DECLINE_THRESHOLD || dimensionsGain < DECLINE_THRESHOLD) return 'decline';
|
||||
return 'inconclusive';
|
||||
}
|
||||
|
||||
/**
|
||||
* SC activation check, BOTH halves.
|
||||
*
|
||||
* The SC asks two things of an `effort: high` run: that it discovered at least
|
||||
* one dimension, AND that the dimension list in the output brief is a TRUE
|
||||
* SUPERSET of the interview-derived ones. This function used to check only
|
||||
* `dimensions - dimensions_baseline >= 1`, which is a count delta and says
|
||||
* nothing about membership — a run that dropped two interview dimensions and
|
||||
* added three discovered ones passed while violating the second half.
|
||||
* Supersetness was asserted only by Phase 4.5's prose contract that discovery
|
||||
* APPENDS; nothing read it.
|
||||
*
|
||||
* The record cannot carry the dimension names: names are free prose, and
|
||||
* the record denies prose by omission. So the run attests
|
||||
* membership with `dimensions_baseline_preserved`, a low-cardinality boolean set
|
||||
* in Phase 4.5, and this gate refuses to call activation OK without it. An
|
||||
* ABSENT attestation is not an attestation — legacy rows fail here rather than
|
||||
* passing on the old count-only rule.
|
||||
*
|
||||
* @param {object[]} records
|
||||
*/
|
||||
export function activationCheck(records) {
|
||||
const high = records.filter((r) => r.effort === 'high');
|
||||
if (high.length === 0) {
|
||||
return { ok: false, reason: 'no `effort: high` run found in stats', discovered_dimensions: null };
|
||||
}
|
||||
const last = high[high.length - 1];
|
||||
const d = Number(last.dimensions);
|
||||
const b = Number(last.dimensions_baseline);
|
||||
if (!Number.isFinite(d) || !Number.isFinite(b)) {
|
||||
return { ok: false, reason: 'latest high run lacks dimensions/dimensions_baseline', discovered_dimensions: null };
|
||||
}
|
||||
const discovered = d - b;
|
||||
const preserved = last.dimensions_baseline_preserved;
|
||||
|
||||
const base = {
|
||||
ts: last.ts ?? null,
|
||||
dimensions: d,
|
||||
dimensions_baseline: b,
|
||||
discovered_dimensions: discovered,
|
||||
dimensions_baseline_preserved: preserved ?? null,
|
||||
conv_turns: Number(last.conv_turns ?? 0),
|
||||
empty_turns: Number(last.empty_turns ?? 0),
|
||||
};
|
||||
|
||||
if (typeof preserved !== 'boolean') {
|
||||
return {
|
||||
...base,
|
||||
ok: false,
|
||||
reason:
|
||||
'latest high run does not attest `dimensions_baseline_preserved`; the SC needs a true ' +
|
||||
'superset of the interview dimensions, and a count delta cannot show membership',
|
||||
};
|
||||
}
|
||||
if (preserved === false) {
|
||||
return {
|
||||
...base,
|
||||
ok: false,
|
||||
reason:
|
||||
`latest high run discovered ${discovered} dimension(s) but did NOT preserve its interview ` +
|
||||
'baseline, so the final list is not a superset of it',
|
||||
};
|
||||
}
|
||||
if (discovered < 1) {
|
||||
return { ...base, ok: false, reason: 'latest high run discovered no dimensions beyond its baseline' };
|
||||
}
|
||||
return { ...base, ok: true };
|
||||
}
|
||||
|
||||
// ---- CLI shim ----------------------------------------------------------------
|
||||
|
||||
function pct(x) {
|
||||
return x === null ? 'n/a' : `${(x * 100).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function defaultStatsPath(env = process.env) {
|
||||
const dir = env.CLAUDE_PLUGIN_DATA;
|
||||
return dir ? join(dir, STATS_FILENAME) : null;
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const o = { stats: null, json: false, activation: false, help: false };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === '--stats') o.stats = argv[++i];
|
||||
else if (a === '--json') o.json = true;
|
||||
else if (a === '--activation-check') o.activation = true;
|
||||
else if (a === '--help' || a === '-h') o.help = true;
|
||||
else { process.stderr.write(`Unknown argument: ${a}\n`); process.exit(2); }
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
function mainCli() {
|
||||
const o = parseArgs(process.argv.slice(2));
|
||||
if (o.help) {
|
||||
process.stdout.write(
|
||||
'Usage: storm-measure.mjs [--stats FILE] [--activation-check] [--json]\n' +
|
||||
' Default --stats: ${CLAUDE_PLUGIN_DATA}/' + STATS_FILENAME + '\n' +
|
||||
' Thresholds (pre-registered, docs/storm-measurement.md): ' +
|
||||
`adopt >= ${ADOPT_THRESHOLD * 100}%, decline < ${DECLINE_THRESHOLD * 100}%\n`,
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const statsPath = o.stats || defaultStatsPath();
|
||||
if (!statsPath) {
|
||||
process.stderr.write('storm-measure: CLAUDE_PLUGIN_DATA is not set and no --stats FILE was given.\n');
|
||||
process.exit(2);
|
||||
}
|
||||
if (!existsSync(statsPath)) {
|
||||
process.stderr.write(`storm-measure: stats file not found: ${statsPath}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
let parsed;
|
||||
try {
|
||||
parsed = parseStats(readFileSync(statsPath, 'utf-8'));
|
||||
} catch (e) {
|
||||
process.stderr.write(`${e.message}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
if (o.activation) {
|
||||
const res = activationCheck(parsed.records);
|
||||
process.stdout.write(JSON.stringify(res, null, 2) + '\n');
|
||||
process.exit(res.ok ? 0 : 1);
|
||||
}
|
||||
|
||||
const m = measure(parsed.records);
|
||||
|
||||
if (o.json) {
|
||||
process.stdout.write(JSON.stringify({ statsPath, ...m, malformed: parsed.malformed, legacy: parsed.legacy }, null, 2) + '\n');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const L = [];
|
||||
L.push(`STORM adoption gate — ${statsPath}`);
|
||||
L.push(` control (effort != high): n=${m.control.n} median unique_sources=${m.control.sources ?? 'n/a'}`);
|
||||
L.push(` treatment (effort = high): n=${m.treatment.n} median unique_sources=${m.treatment.sources ?? 'n/a'}`);
|
||||
L.push(` excluded (empty_turns > 0): ${m.excluded}`);
|
||||
if (parsed.legacy) L.push(` legacy rows without \`effort\`: ${parsed.legacy}`);
|
||||
if (parsed.malformed) L.push(` malformed lines: ${parsed.malformed}`);
|
||||
L.push('');
|
||||
L.push(` median gain, unique_sources: ${pct(m.sources.gain)}`);
|
||||
L.push(` median gain, dimensions over baseline: ${pct(m.dimensions.gain)} (n=${m.dimensions.n})`);
|
||||
L.push('');
|
||||
L.push(` thresholds: adopt >= ${pct(ADOPT_THRESHOLD)} on EITHER · decline < ${pct(DECLINE_THRESHOLD)} on EITHER · adopt wins ties`);
|
||||
L.push(` VERDICT: ${m.verdict}`);
|
||||
process.stdout.write(L.join('\n') + '\n');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (isMain(import.meta.url)) {
|
||||
mainCli();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue