feat(skills): interviewing - one source for the brief's question round (order 7e)
/trekbrief Phase 3 carried the interview form inline. It now asks for the new skill voyage:interviewing by name (allowed-tools gains Skill) and keeps only its own contract: the size class and budget, the three fixed questions (framing first, class, storage), where each answer lands in the brief, and the answers that decide nothing. Step 3a and the form points of Step 3c move into the skill with their credits (Pocock, Osmani, Spec Kit; licences and line ranges checked at the pinned commits). Red baseline first (3 helper interviews against a fixed answer sheet, Opus 5.5): 10-11 questions in round 1 (3 of 3), 15 of 31 without a recommendation, a hollow answer written in as a decision (2 of 3), unasked scope as a requirement (2 of 3). With the skill: 4, 0 of 12, 0 of 3, 0 of 3. Fact questions and stopping did not fail without it. Also: pointers to voyage:writing-criteria (S3) where criteria are asked and written, both review skill names (S3 + S4) in the brief-reviewer prompt; the intent pins that read moved text now read the skill, same assertions; the Phase 7 line_ref moves with the shorter Phase 3; 7 eval cases (3/2/2). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
31c77fd8d6
commit
f6e18a1d09
25 changed files with 569 additions and 51 deletions
2
tests/fixtures/jsonl-schemas.md
vendored
2
tests/fixtures/jsonl-schemas.md
vendored
|
|
@ -19,7 +19,7 @@
|
|||
|
||||
| schema_id | fields | writer_path | line_ref | v4.1 additive | PII |
|
||||
|-----------|--------|-------------|----------|---------------|-----|
|
||||
| trekbrief-stats | ts, task, slug, mode, interview_turns, review_iterations, brief_quality, research_topics, auto_research, auto_result, intent_approved, project_dir, order_id, order_claimed_at, order_reason | commands/trekbrief.md (orchestrator-emit Phase 7; order_* added by lib/stats/stats-append.mjs) | trekbrief.md:1137-1161 | profile, phase_models, profile_source | none |
|
||||
| trekbrief-stats | ts, task, slug, mode, interview_turns, review_iterations, brief_quality, research_topics, auto_research, auto_result, intent_approved, project_dir, order_id, order_claimed_at, order_reason | commands/trekbrief.md (orchestrator-emit Phase 7; order_* added by lib/stats/stats-append.mjs) | trekbrief.md:1127-1151 | profile, phase_models, profile_source | none |
|
||||
| trekresearch-stats | ts, question, mode, scope, engine, slug, project_dir, brief_path, dimensions, dimensions_baseline, dimensions_baseline_preserved, effort, conv_turns, empty_turns, unique_sources, agents_local, agents_external, gemini_used, confidence, contradictions, open_questions | lib/stats/stats-append.mjs (trekresearch.md Stats tracking) | trekresearch.md:782-808 | profile, phase_models, parallel_agents, external_research_enabled, profile_source | none |
|
||||
| trekplan-stats | ts, task, mode, slug, brief_path, project_dir, codebase_size, codebase_files, agents_deployed, deep_dives, research_briefs_used, research_scout_used, critic_verdict, guardian_verdict, outcome | lib/stats/stats-append.mjs (trekplan.md Phase 12) | trekplan.md:1271-1299 | profile, phase_models, parallel_agents, profile_source | none |
|
||||
| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:2892-2914 | profile, phase_models, profile_source | none |
|
||||
|
|
|
|||
|
|
@ -53,16 +53,24 @@ test('trekbrief: three classes, heaviest in doubt, said aloud, overridable, only
|
|||
assert.match(p3, /`ceremony:/, 'the class is written to the brief (K1)');
|
||||
});
|
||||
|
||||
// The interview form moved into the skill /trekbrief asks for by name (order
|
||||
// 7e, one source): the same assertions, now read where the text lives.
|
||||
const interviewing = () => read('skills/interviewing/SKILL.md');
|
||||
|
||||
test('trekbrief: the round — all answerable questions at once, recommendation first, no fact questions', () => {
|
||||
const p3 = section(brief, 'Phase 3');
|
||||
const iv = interviewing();
|
||||
assert.match(p3, /Use `AskUserQuestion` for every question/);
|
||||
assert.match(p3, /`voyage:interviewing`/, 'Phase 3 asks for the interview form by name');
|
||||
assert.doesNotMatch(brief, /Ask one question at a time/, 'the one-at-a-time rule is replaced by the round');
|
||||
assert.doesNotMatch(iv, /Ask one question at a time/, 'the one-at-a-time rule is replaced by the round');
|
||||
assert.match(p3, /one round/i);
|
||||
assert.match(p3, /recommend(ed|ation)[^.\n]*first/i);
|
||||
assert.match(p3, /never ask[^.\n]*fact/i);
|
||||
assert.match(p3, /Impact × Uncertainty/);
|
||||
assert.match(p3, /[Dd]eferred/);
|
||||
assert.match(p3, /next three answers/);
|
||||
assert.match(iv, /one round/i);
|
||||
assert.match(iv, /recommend(ed|ation)[^.\n]*first/i);
|
||||
assert.match(iv, /never ask[^.\n]*fact/i);
|
||||
assert.match(iv, /Impact × Uncertainty/);
|
||||
assert.match(iv, /[Dd]eferred/);
|
||||
assert.match(iv, /next three answers/);
|
||||
});
|
||||
|
||||
test('trekbrief: the budget for a small task is at most 5 questions and fits one AskUserQuestion call', () => {
|
||||
|
|
@ -75,10 +83,16 @@ test('trekbrief: the budget for a small task is at most 5 questions and fits one
|
|||
});
|
||||
|
||||
test('trekbrief: credits every borrowing of the round and the classes', () => {
|
||||
// The classes stay in /trekbrief with their credits; the round's credits
|
||||
// moved with the round into the skill.
|
||||
const p3 = section(brief, 'Phase 3');
|
||||
for (const src of ['obra/superpowers@8ca22db', 'mattpocock/skills@c55ee46', 'GitHub Spec Kit', 'addyosmani/agent-skills@9d0c60d', 'kiro.dev']) {
|
||||
for (const src of ['obra/superpowers@8ca22db', 'kiro.dev']) {
|
||||
assert.ok(p3.includes(src), `credit for ${src}`);
|
||||
}
|
||||
const iv = interviewing();
|
||||
for (const re of [/mattpocock\/skills ?@c55ee46/, /GitHub Spec Kit/, /addyosmani\/agent-skills ?@9d0c60d/]) {
|
||||
assert.match(iv, re, `credit for ${re}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('trekbrief: hard rule 8 no longer says Phase 3 has no cap', () => {
|
||||
|
|
|
|||
171
tests/lib/skills-brief-prose-pins.test.mjs
Normal file
171
tests/lib/skills-brief-prose-pins.test.mjs
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
// tests/lib/skills-brief-prose-pins.test.mjs
|
||||
// Wave 3, order 7e: the `interviewing` skill and its wiring into /trekbrief.
|
||||
//
|
||||
// What is pinned, and why:
|
||||
// • the skill's form is not pinned here: the form check
|
||||
// (lib/skills/skill-lint.mjs) and the routing test (lib/skills/routing.mjs)
|
||||
// run over every skill in `npm test` and are the authority (contract S1);
|
||||
// the routing test is also what shows the skill does not collide with
|
||||
// /trekbrief (cosine >= 0.75 fails, never lowered);
|
||||
// • the interview form lives in the skill: at most four questions per call,
|
||||
// the recommended answer first, facts looked up, the rest written down,
|
||||
// the stop rule, and the two failures the red baseline showed (an answer
|
||||
// that decides nothing written in as a decision; scope nobody asked for);
|
||||
// • /trekbrief asks for it by name, and for `voyage:writing-criteria` (S3,
|
||||
// order 7c) where the success criteria are written, and names both
|
||||
// review skills (S3 + S4) where it starts brief-reviewer;
|
||||
// • the text that moved is gone from /trekbrief (one source), while what is
|
||||
// /trekbrief's own contract stays: the size class and its budget, the
|
||||
// framing question, the storage question, the approval and the answers
|
||||
// that decide nothing;
|
||||
// • the skill says plainly that the interview's form is prose only.
|
||||
//
|
||||
// S3 and S4 are made by orders 7c and 7d. Their skills need not exist in this
|
||||
// branch: the names are the contract, written here before the merge.
|
||||
// Counted as doc pins by lib/util/test-census.mjs (file name *-prose-pins).
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const read = (rel) => readFileSync(join(ROOT, rel), 'utf8');
|
||||
|
||||
const SKILL = 'skills/interviewing/SKILL.md';
|
||||
const skill = () => {
|
||||
assert.ok(existsSync(join(ROOT, SKILL)), `${SKILL} missing`);
|
||||
return read(SKILL);
|
||||
};
|
||||
const brief = read('commands/trekbrief.md');
|
||||
|
||||
/** Text from `## <heading>` up to the next `## ` heading. */
|
||||
function section(text, heading) {
|
||||
const start = text.indexOf(`\n## ${heading}`);
|
||||
assert.ok(start >= 0, `missing ## ${heading}`);
|
||||
const next = text.indexOf('\n## ', start + 4);
|
||||
return text.slice(start, next < 0 ? undefined : next);
|
||||
}
|
||||
|
||||
/** Text from `### <heading>` up to the next `### ` or `## ` heading. */
|
||||
function subsection(text, heading) {
|
||||
const start = text.indexOf(`\n### ${heading}`);
|
||||
assert.ok(start >= 0, `missing ### ${heading}`);
|
||||
const rest = text.slice(start + 5);
|
||||
const m = rest.match(/\n#{2,3} /);
|
||||
return text.slice(start, m ? start + 5 + m.index : undefined);
|
||||
}
|
||||
|
||||
const flat = (t) => t.replace(/\s+/g, ' ');
|
||||
|
||||
// ---- the skill carries the interview form ----
|
||||
|
||||
test('pin: the interviewing skill exists', () => {
|
||||
skill();
|
||||
});
|
||||
|
||||
test('pin: interviewing — at most four questions per AskUserQuestion call, recommended answer first', () => {
|
||||
const p = flat(section(skill(), 'Process'));
|
||||
assert.match(p, /at most (4|four) questions per `AskUserQuestion` call/i);
|
||||
assert.match(p, /recommended answer first/i);
|
||||
assert.match(p, /\(Recommended\)/);
|
||||
assert.match(p, /consequence of each option/i);
|
||||
});
|
||||
|
||||
test('pin: interviewing — facts are looked up, never asked', () => {
|
||||
const p = flat(section(skill(), 'Process'));
|
||||
assert.match(p, /never ask[^.]*fact/i);
|
||||
});
|
||||
|
||||
test('pin: interviewing — the rest is ranked and written down: assumptions and a Deferred list', () => {
|
||||
const p = flat(section(skill(), 'Process'));
|
||||
assert.match(p, /Impact × Uncertainty/);
|
||||
assert.match(p, /[Dd]eferred/);
|
||||
assert.match(p, /assumed, not asked/);
|
||||
});
|
||||
|
||||
test('pin: interviewing — the stop rule: the next three answers can be predicted', () => {
|
||||
assert.match(flat(section(skill(), 'Process')), /next three answers/);
|
||||
});
|
||||
|
||||
test('pin: interviewing — the two measured failures: an answer that decides nothing, and unasked scope', () => {
|
||||
const p = flat(section(skill(), 'Process'));
|
||||
assert.match(p, /not confirmed — the answer was/);
|
||||
assert.match(p, /never as the person's (choice|decision)/i);
|
||||
assert.match(p, /nobody asked for/i);
|
||||
const flags = section(skill(), 'Red flags');
|
||||
assert.match(flags, /[Ww]hatever you think/);
|
||||
});
|
||||
|
||||
test('pin: interviewing — the interview form is prose only; the brief validator checks the result', () => {
|
||||
const enforced = flat(section(skill(), 'Enforced by'));
|
||||
assert.ok(enforced.includes('lib/validators/brief-validator.mjs'), 'names the brief validator');
|
||||
assert.match(enforced, /prose only/);
|
||||
assert.match(enforced, /not the form of the interview/);
|
||||
});
|
||||
|
||||
test('pin: interviewing — credits its sources with commit and licence', () => {
|
||||
const credits = section(skill(), 'Credits');
|
||||
for (const src of [
|
||||
'mattpocock/skills @c55ee46', 'skills/productivity/grilling/SKILL.md',
|
||||
'addyosmani/agent-skills @9d0c60d', 'skills/interview-me/SKILL.md',
|
||||
'github/spec-kit @2e941fe', 'templates/commands/clarify.md',
|
||||
]) assert.ok(credits.includes(src), `credit ${src}`);
|
||||
assert.equal((credits.match(/\bMIT\b/g) || []).length, 3, 'one licence per source');
|
||||
});
|
||||
|
||||
// ---- /trekbrief asks for the skills by name (S3, S4, interviewing) ----
|
||||
|
||||
test('pin: /trekbrief Phase 3 asks for voyage:interviewing by name, and the command may use the Skill tool', () => {
|
||||
assert.match(section(brief, 'Phase 3'), /`voyage:interviewing`/);
|
||||
assert.match(section(brief, 'Phase 3'), /Call the Skill tool/);
|
||||
assert.match(brief.split('\n---\n')[0], /\nallowed-tools:[^\n]*\bSkill\b/, 'the command may use the Skill tool');
|
||||
});
|
||||
|
||||
test('pin: /trekbrief names voyage:writing-criteria (S3) where the success criteria are asked and written', () => {
|
||||
const bank = subsection(brief, 'Question bank');
|
||||
const sc = bank.slice(bank.indexOf('**Success Criteria**'), bank.indexOf('**Research Plan**'));
|
||||
assert.match(sc, /`voyage:writing-criteria`/);
|
||||
const draft = brief.slice(brief.indexOf('**Step 4a'), brief.indexOf('**Step 4b'));
|
||||
assert.match(draft, /\*\*Success Criteria:\*\*[^\n]*\n?[^\n]*`voyage:writing-criteria`/);
|
||||
});
|
||||
|
||||
test('pin: /trekbrief names both review skills where it starts brief-reviewer (S3 + S4)', () => {
|
||||
const step = brief.slice(brief.indexOf('**Step 4c'), brief.indexOf('**Step 4d'));
|
||||
assert.match(step, /`voyage:writing-criteria`/);
|
||||
assert.match(step, /`voyage:reviewing-adversarially`/);
|
||||
});
|
||||
|
||||
// ---- one source: the moved text is gone from /trekbrief ----
|
||||
|
||||
const MOVED = [
|
||||
'Never ask the operator a fact a helper can look up',
|
||||
'Rank the rest by Impact × Uncertainty',
|
||||
'The tool takes at most 4 questions per call',
|
||||
"Stop asking as soon as you can predict the operator's next three answers",
|
||||
'Each question is a full sentence in plain words',
|
||||
'The round, the recommended answer first and "never ask a fact": mattpocock/skills@c55ee46',
|
||||
'Impact × Uncertainty and the Deferred list: GitHub Spec Kit',
|
||||
'The stop rule and the answers that decide nothing: addyosmani/agent-skills@9d0c60d',
|
||||
];
|
||||
|
||||
test('pin: the interview text that moved into the skill is no longer in /trekbrief', () => {
|
||||
const b = flat(brief);
|
||||
for (const s of MOVED) assert.ok(!b.includes(s), `still in commands/trekbrief.md: «${s}»`);
|
||||
});
|
||||
|
||||
// ---- what stays: /trekbrief's own contract ----
|
||||
|
||||
test('pin: /trekbrief keeps its own contract — class, budget, framing first, storage, hollow answers, approval', () => {
|
||||
const p3 = flat(section(brief, 'Phase 3'));
|
||||
assert.match(p3, /### Step 3b — Ceremony by size/);
|
||||
assert.match(p3, /\*\*Budget per class\*\*/);
|
||||
assert.match(p3, /framing[^\n]*first/i);
|
||||
assert.match(p3, /artifact_store/);
|
||||
for (const w of ['høres bra ut', 'gjør som du vil']) assert.ok(p3.includes(w), `Phase 3 keeps «${w}»`);
|
||||
assert.match(p3, /empty reply/);
|
||||
assert.match(section(brief, 'Phase 2.5'), /hollow answer/i);
|
||||
assert.match(section(brief, 'Phase 4h'), /høres bra ut/);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue