feat(ost): v5.6 C — output-style scanner (CA-OST, count 13→14)
New orchestrated scanner output-style-scanner.mjs — first new family since SKL. Three findings, each pinned to a CONFIRMED V-row of the steering-model plan + re-verified against code.claude.com/docs/en/output-styles: - CA-OST-001 (medium, V10): user/project custom style missing keep-coding-instructions:true (default false) → silently strips built-in software-engineering instructions when active. Scoped to user/project. - CA-OST-002 (low, V11): plugin style with force-for-plugin:true overrides the user's selected outputStyle. Verifiseringsplikt correction — the plan bullet said "project/user style," but force-for-plugin is plugin-styles-only per the docs, so the check keys on source==='plugin'. - CA-OST-003 (medium): settings outputStyle matching no built-in (Default/Explanatory/Learning/Proactive, case-insensitive) nor discovered custom style → dead config. Byte-stability — a scanner addition, not a field addition. Growing the scanners array + scanners_ok cannot be hidden by a field strip, but re-seeding frozen v5.0.0 (the SKL precedent) would now bake in B2's hotspot triple + claudeMd drift. So, per the B2 lesson, frozen v5.0.0 snapshots are PRESERVED and the OST entry is stripped at compare time via new tests/helpers/strip-added-scanner.mjs (wired into json/raw-backcompat + the Step 5/6 humanizer tests); only SC-5 default-output is regenerated (additive OST entry, diff reviewed). OST is fixture-gated (no output styles on marketplace-medium / hermetic HOME → silent). Wiring: orchestrator; humanizer (OST→Configuration mistake) + humanizer-data OST family (title-coupled); scoring (OST→Settings, keeps 10 areas). Suite 1012→1023 (+11). Badges: scanners 14, tests 1023, TRANSLATIONS families 15. Lore swept: README, CLAUDE.md, scanner-internals, humanizer.md. self-audit A/A. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
43d8873339
commit
e3b044a476
20 changed files with 557 additions and 47 deletions
169
tests/scanners/output-style-scanner.test.mjs
Normal file
169
tests/scanners/output-style-scanner.test.mjs
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
/**
|
||||
* OST scanner tests — output-style validation (v5.6 C).
|
||||
*
|
||||
* Each finding has a positive fixture (fires) and a negative fixture (silent),
|
||||
* per the v5.5+ plan acceptance criteria. The scanner reads the active config
|
||||
* (project + user + plugin output styles, settings cascade) via HOME, so every
|
||||
* test builds a hermetic temp HOME + repo, overrides process.env.HOME, runs,
|
||||
* then restores and cleans up — never touching real user state.
|
||||
*
|
||||
* CA-OST-001 user/project custom style missing keep-coding-instructions:true (medium)
|
||||
* CA-OST-002 plugin output style with force-for-plugin:true (low)
|
||||
* CA-OST-003 settings outputStyle resolving to a non-existent style (medium)
|
||||
*/
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { join } from 'node:path';
|
||||
import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { resetCounter } from '../../scanners/lib/output.mjs';
|
||||
import { scan } from '../../scanners/output-style-scanner.mjs';
|
||||
|
||||
function tmp(prefix) {
|
||||
return mkdtemp(join(tmpdir(), `ca-ost-${prefix}-`));
|
||||
}
|
||||
|
||||
async function writeStyle(dir, name, frontmatter) {
|
||||
await mkdir(dir, { recursive: true });
|
||||
const fm = frontmatter ?? `name: ${name}\ndescription: test style`;
|
||||
await writeFile(join(dir, `${name}.md`), `---\n${fm}\n---\n\nStyle body for ${name}.\n`, 'utf-8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a hermetic HOME + repo from a spec, run the OST scanner, restore + clean.
|
||||
* @param {object} opts
|
||||
* @param {Array<{name:string, frontmatter?:string}>} [opts.projectStyles]
|
||||
* @param {Array<{name:string, frontmatter?:string}>} [opts.userStyles]
|
||||
* @param {Array<{plugin:string, name:string, frontmatter?:string}>} [opts.pluginStyles]
|
||||
* @param {object} [opts.projectSettings]
|
||||
*/
|
||||
async function runOst(opts = {}) {
|
||||
const home = await tmp('home');
|
||||
const repo = await tmp('repo');
|
||||
|
||||
for (const s of opts.projectStyles || []) {
|
||||
await writeStyle(join(repo, '.claude', 'output-styles'), s.name, s.frontmatter);
|
||||
}
|
||||
for (const s of opts.userStyles || []) {
|
||||
await writeStyle(join(home, '.claude', 'output-styles'), s.name, s.frontmatter);
|
||||
}
|
||||
for (const s of opts.pluginStyles || []) {
|
||||
const root = join(home, '.claude', 'plugins', 'marketplaces', 'mp', 'plugins', s.plugin);
|
||||
await mkdir(join(root, '.claude-plugin'), { recursive: true });
|
||||
await writeFile(
|
||||
join(root, '.claude-plugin', 'plugin.json'),
|
||||
JSON.stringify({ name: s.plugin, version: '1.0.0' }),
|
||||
'utf-8',
|
||||
);
|
||||
await writeStyle(join(root, 'output-styles'), s.name, s.frontmatter);
|
||||
}
|
||||
if (opts.projectSettings) {
|
||||
await mkdir(join(repo, '.claude'), { recursive: true });
|
||||
await writeFile(join(repo, '.claude', 'settings.json'), JSON.stringify(opts.projectSettings), 'utf-8');
|
||||
}
|
||||
|
||||
const originalHome = process.env.HOME;
|
||||
const originalProfile = process.env.USERPROFILE;
|
||||
process.env.HOME = home;
|
||||
process.env.USERPROFILE = home;
|
||||
resetCounter();
|
||||
try {
|
||||
return await scan(repo, { files: [] });
|
||||
} finally {
|
||||
process.env.HOME = originalHome;
|
||||
process.env.USERPROFILE = originalProfile;
|
||||
await rm(home, { recursive: true, force: true });
|
||||
await rm(repo, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
const has = (result, re) => result.findings.find(f => re.test(String(f.title || '')));
|
||||
|
||||
describe('OST scanner — basic structure', () => {
|
||||
it('reports scanner prefix OST', async () => {
|
||||
const result = await runOst({ projectStyles: [{ name: 'plain' }] });
|
||||
assert.equal(result.scanner, 'OST');
|
||||
assert.equal(result.status, 'ok');
|
||||
});
|
||||
|
||||
it('finding IDs match CA-OST-NNN', async () => {
|
||||
const result = await runOst({ projectStyles: [{ name: 'plain' }] });
|
||||
for (const f of result.findings) {
|
||||
assert.match(f.id, /^CA-OST-\d{3}$/);
|
||||
}
|
||||
});
|
||||
|
||||
it('is fixture-gated — no output styles, no settings → zero findings', async () => {
|
||||
const result = await runOst({});
|
||||
assert.equal(result.findings.length, 0, `expected silent; got: ${result.findings.map(f => f.title).join(' | ')}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('OST scanner — CA-OST-001 keep-coding-instructions', () => {
|
||||
it('flags a project custom style missing keep-coding-instructions:true (medium)', async () => {
|
||||
const result = await runOst({ projectStyles: [{ name: 'myco' }] });
|
||||
const f = has(result, /coding instructions/i);
|
||||
assert.ok(f, `expected OST-001; got: ${result.findings.map(x => x.title).join(' | ')}`);
|
||||
assert.equal(f.severity, 'medium');
|
||||
assert.match(String(f.file), /myco\.md$/);
|
||||
});
|
||||
|
||||
it('flags a user custom style missing the flag (source=user in evidence)', async () => {
|
||||
const result = await runOst({ userStyles: [{ name: 'mine' }] });
|
||||
const f = has(result, /coding instructions/i);
|
||||
assert.ok(f);
|
||||
assert.match(String(f.evidence), /source=user/);
|
||||
});
|
||||
|
||||
it('does NOT flag a style that sets keep-coding-instructions:true', async () => {
|
||||
const result = await runOst({
|
||||
projectStyles: [{ name: 'keeper', frontmatter: 'name: keeper\ndescription: test\nkeep-coding-instructions: true' }],
|
||||
});
|
||||
assert.equal(has(result, /coding instructions/i), undefined,
|
||||
`expected silent; got: ${result.findings.map(x => x.title).join(' | ')}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('OST scanner — CA-OST-002 force-for-plugin', () => {
|
||||
it('flags a plugin style with force-for-plugin:true (low, names the plugin)', async () => {
|
||||
const result = await runOst({
|
||||
pluginStyles: [{ plugin: 'styler', name: 'forced', frontmatter: 'name: forced\ndescription: test\nforce-for-plugin: true' }],
|
||||
});
|
||||
const f = has(result, /override/i);
|
||||
assert.ok(f, `expected OST-002; got: ${result.findings.map(x => x.title).join(' | ')}`);
|
||||
assert.equal(f.severity, 'low');
|
||||
assert.match(String(f.evidence), /styler/);
|
||||
});
|
||||
|
||||
it('does NOT flag a plugin style without force-for-plugin (and OST-001 stays project/user-only)', async () => {
|
||||
const result = await runOst({
|
||||
pluginStyles: [{ plugin: 'styler', name: 'plain', frontmatter: 'name: plain\ndescription: test' }],
|
||||
});
|
||||
assert.equal(result.findings.length, 0,
|
||||
`expected silent for a plain plugin style; got: ${result.findings.map(x => x.title).join(' | ')}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('OST scanner — CA-OST-003 dead outputStyle', () => {
|
||||
it('flags settings.outputStyle that resolves to no known style (medium)', async () => {
|
||||
const result = await runOst({ projectSettings: { outputStyle: 'Ghostwriter' } });
|
||||
const f = has(result, /does not exist/i);
|
||||
assert.ok(f, `expected OST-003; got: ${result.findings.map(x => x.title).join(' | ')}`);
|
||||
assert.equal(f.severity, 'medium');
|
||||
assert.match(String(f.evidence), /Ghostwriter/);
|
||||
});
|
||||
|
||||
it('does NOT flag a built-in style name (Explanatory)', async () => {
|
||||
const result = await runOst({ projectSettings: { outputStyle: 'Explanatory' } });
|
||||
assert.equal(has(result, /does not exist/i), undefined);
|
||||
});
|
||||
|
||||
it('does NOT flag an outputStyle matching an existing custom style', async () => {
|
||||
const result = await runOst({
|
||||
projectStyles: [{ name: 'myco', frontmatter: 'name: myco\ndescription: test\nkeep-coding-instructions: true' }],
|
||||
projectSettings: { outputStyle: 'myco' },
|
||||
});
|
||||
assert.equal(has(result, /does not exist/i), undefined,
|
||||
`expected no dead-config finding; got: ${result.findings.map(x => x.title).join(' | ')}`);
|
||||
});
|
||||
});
|
||||
|
|
@ -7,6 +7,7 @@ import { promisify } from 'node:util';
|
|||
import { readFile, unlink } from 'node:fs/promises';
|
||||
import { hermeticEnv } from '../helpers/hermetic-home.mjs';
|
||||
import { stripHotspotLoadPattern } from '../helpers/strip-hotspot-load-pattern.mjs';
|
||||
import { stripAddedScanners, stripAddedScannerStderr } from '../helpers/strip-added-scanner.mjs';
|
||||
|
||||
const exec = promisify(execFile);
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
|
@ -39,12 +40,16 @@ function normalizePosture(p) {
|
|||
}
|
||||
}
|
||||
}
|
||||
return stripHotspotLoadPattern(out);
|
||||
return stripAddedScanners(stripHotspotLoadPattern(out));
|
||||
}
|
||||
|
||||
/** Strip time-varying durations (Xms) so progress lines compare verbatim across runs. */
|
||||
/**
|
||||
* Strip time-varying durations (Xms) so progress lines compare verbatim across
|
||||
* runs, and drop the additive [OST] progress line (v5.6 C) so a 14-scanner live
|
||||
* run matches the frozen 13-scanner v5.0.0 stderr scorecard.
|
||||
*/
|
||||
function normalizeStderr(s) {
|
||||
return s.replace(/\(\d+ms\)/g, '(0ms)');
|
||||
return stripAddedScannerStderr(s).replace(/\(\d+ms\)/g, '(0ms)');
|
||||
}
|
||||
|
||||
async function runPosture(flags) {
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import { promisify } from 'node:util';
|
|||
import { readFile, unlink } from 'node:fs/promises';
|
||||
import { hermeticEnv } from '../helpers/hermetic-home.mjs';
|
||||
import { stripHotspotLoadPattern } from '../helpers/strip-hotspot-load-pattern.mjs';
|
||||
import { stripAddedScanners } from '../helpers/strip-added-scanner.mjs';
|
||||
|
||||
const exec = promisify(execFile);
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
|
@ -37,7 +38,7 @@ function normalizeEnvelope(env) {
|
|||
}
|
||||
}
|
||||
}
|
||||
return stripHotspotLoadPattern(out);
|
||||
return stripAddedScanners(stripHotspotLoadPattern(out));
|
||||
}
|
||||
|
||||
async function runOrchestrator(flags) {
|
||||
|
|
@ -102,7 +103,7 @@ describe('scan-orchestrator humanizer wiring (Step 5)', () => {
|
|||
assert.ok(actual.meta, 'meta present');
|
||||
assert.ok(Array.isArray(actual.scanners), 'scanners array present');
|
||||
assert.ok(actual.aggregate, 'aggregate present');
|
||||
assert.equal(actual.scanners.length, 13, 'all 13 scanners present');
|
||||
assert.equal(actual.scanners.length, 14, 'all 14 scanners present');
|
||||
});
|
||||
|
||||
it('preserves scanner shape (scanner/status/findings/counts)', async () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue