135 lines
7 KiB
JavaScript
135 lines
7 KiB
JavaScript
// tests/kb-update/test-scan-adversarial-content.test.mjs
|
||
// Layer B create-guard CLI (G6 §8 / R6 punkt d): scripts/kb-update/scan-adversarial-content.mjs
|
||
// is the sibling to validate-kb-file.mjs — the gate any generator (kb-update, generate-skills)
|
||
// calls over candidate files. A file that BLOCKs is never written / never committed; a file
|
||
// that WARNs is flagged for the operator (never auto-committed). Exit: 0 clean, 1 block, 2 warn.
|
||
//
|
||
// scanPaths is dependency-injected (readFile + detect), so these tests feed synthetic content
|
||
// and a fake detector — they never touch disk nor llm-security. FAIL-CLOSED behaviour (a read
|
||
// error or an unavailable detector BLOCKS, never silently passes) is asserted here because it
|
||
// is the whole point of a security gate.
|
||
|
||
import { test } from 'node:test';
|
||
import assert from 'node:assert/strict';
|
||
import { mkdtempSync, writeFileSync, rmSync, readdirSync } from 'node:fs';
|
||
import { tmpdir } from 'node:os';
|
||
import { join } from 'node:path';
|
||
import { scanPaths } from '../../scripts/kb-update/scan-adversarial-content.mjs';
|
||
import { detectAdversarial } from '../../scripts/kb-update/lib/adversarial-detect.mjs';
|
||
|
||
const CLEAN = '# T\n\n**Source:** https://learn.microsoft.com/azure/x\n\nProse.\n';
|
||
|
||
function reader(map) {
|
||
return (p) => {
|
||
if (!(p in map)) throw new Error(`ENOENT: ${p}`);
|
||
return map[p];
|
||
};
|
||
}
|
||
// Fake detectors keyed by returning canned raw findings regardless of content.
|
||
const detectClean = async () => [];
|
||
const detectBlock = async () => [{ class: 'injection', severity: 'critical', line: 5, evidence: 'spoofed tag: <system>' }];
|
||
const detectWarn = async () => [{ class: 'injection', severity: 'high', line: 5, evidence: 'imperative in prose' }];
|
||
|
||
test('scanPaths — all clean → ok:true, no block, no warn', async () => {
|
||
const r = await scanPaths(['a.md', 'b.md'], { readFile: reader({ 'a.md': CLEAN, 'b.md': CLEAN }), detect: detectClean });
|
||
assert.equal(r.ok, true);
|
||
assert.equal(r.blocked, false);
|
||
assert.equal(r.warned, false);
|
||
assert.equal(r.results.length, 2);
|
||
assert.ok(r.results.every((x) => x.disposition === 'clean'));
|
||
});
|
||
|
||
test('scanPaths — a blocked file fails the gate (ok:false, blocked:true)', async () => {
|
||
const r = await scanPaths(['bad.md'], { readFile: reader({ 'bad.md': CLEAN }), detect: detectBlock });
|
||
assert.equal(r.ok, false);
|
||
assert.equal(r.blocked, true);
|
||
assert.equal(r.results[0].disposition, 'block');
|
||
});
|
||
|
||
test('scanPaths — a warn-only file is flagged, not clean (warned:true, blocked:false)', async () => {
|
||
const r = await scanPaths(['warn.md'], { readFile: reader({ 'warn.md': CLEAN }), detect: detectWarn });
|
||
assert.equal(r.ok, false);
|
||
assert.equal(r.blocked, false);
|
||
assert.equal(r.warned, true);
|
||
assert.equal(r.results[0].disposition, 'warn');
|
||
});
|
||
|
||
test('scanPaths — one blocked file in a batch fails the whole gate', async () => {
|
||
const detect = async (content) => (content === 'POISON' ? detectBlock() : []);
|
||
const r = await scanPaths(['ok.md', 'bad.md'], { readFile: reader({ 'ok.md': CLEAN, 'bad.md': 'POISON' }), detect });
|
||
assert.equal(r.blocked, true);
|
||
assert.equal(r.results.find((x) => x.path === 'ok.md').disposition, 'clean');
|
||
assert.equal(r.results.find((x) => x.path === 'bad.md').disposition, 'block');
|
||
});
|
||
|
||
test('scanPaths — an unreadable path FAILS CLOSED (blocked), not thrown', async () => {
|
||
const r = await scanPaths(['missing.md'], { readFile: () => { throw new Error('ENOENT'); }, detect: detectClean });
|
||
assert.equal(r.blocked, true);
|
||
assert.equal(r.results[0].disposition, 'block');
|
||
assert.ok(r.results[0].findings.some((f) => /read|enoent/i.test(f.evidence || '')));
|
||
});
|
||
|
||
test('scanPaths — a detector that throws (llm-security unavailable) FAILS CLOSED (blocked)', async () => {
|
||
const r = await scanPaths(['x.md'], {
|
||
readFile: reader({ 'x.md': CLEAN }),
|
||
detect: async () => { throw new Error('LLM_SECURITY_UNAVAILABLE'); },
|
||
});
|
||
assert.equal(r.blocked, true);
|
||
assert.equal(r.results[0].disposition, 'block');
|
||
assert.ok(r.results[0].findings.some((f) => /llm_security|scanner/i.test(f.evidence || '')));
|
||
});
|
||
|
||
test('scanPaths — no paths → ok:true (nothing to gate)', async () => {
|
||
const r = await scanPaths([], { readFile: reader({}), detect: detectClean });
|
||
assert.equal(r.ok, true);
|
||
assert.deepEqual(r.results, []);
|
||
});
|
||
|
||
// --- detectAdversarial: pre-write unicode scan (G6 Layer B bugfix, ingestion-brief §8) ----
|
||
// Unlike the scanPaths tests above (fake detectors, never touch disk/llm-security), these
|
||
// exercise the REAL bridge. The bug: the disk-based unicode scanner only ran when opts.path
|
||
// was supplied, so a candidate scanned as CONTENT (the pre-write ingestion gate) got zero
|
||
// unicode coverage — violating the committed §8 acceptance test (zero-width/bidi rejected
|
||
// BEFORE write). The fix materialises a temp file so the same scanner runs pre-write. These
|
||
// tests are skipped when the llm-security sibling is absent (fresh clone) so the maintainer
|
||
// suite stays green either way.
|
||
let llmSecurityAvailable = false;
|
||
try {
|
||
await detectAdversarial('probe'); // loads the detectors; throws LLM_SECURITY_UNAVAILABLE if absent
|
||
llmSecurityAvailable = true;
|
||
} catch { /* sibling not resolvable — the integration tests below skip */ }
|
||
|
||
const ZW_BIDI = 'Ren tekst med zero-width og bidi override.\n';
|
||
const PLAIN = 'Helt ren prosatekst uten anomalier.\n';
|
||
const advTmpDirs = () => readdirSync(tmpdir()).filter((n) => n.startsWith('kb-adv-'));
|
||
|
||
test('detectAdversarial flags zero-width/bidi in content WITHOUT a path (pre-write §8)', { skip: !llmSecurityAvailable }, async () => {
|
||
const findings = await detectAdversarial(ZW_BIDI);
|
||
const unicode = findings.filter((f) => f.class === 'unicode');
|
||
assert.ok(unicode.length >= 2, `expected ≥2 unicode findings pre-write, got ${JSON.stringify(findings)}`);
|
||
assert.ok(unicode.some((f) => f.subtype === 'zero-width'), 'zero-width carrier must be caught pre-write');
|
||
assert.ok(unicode.some((f) => f.subtype === 'bidi'), 'bidi override must be caught pre-write');
|
||
});
|
||
|
||
test('detectAdversarial returns no unicode findings for clean content WITHOUT a path', { skip: !llmSecurityAvailable }, async () => {
|
||
const findings = await detectAdversarial(PLAIN);
|
||
assert.equal(findings.filter((f) => f.class === 'unicode').length, 0);
|
||
});
|
||
|
||
test('detectAdversarial disk-path route is unchanged (regression)', { skip: !llmSecurityAvailable }, async () => {
|
||
const d = mkdtempSync(join(tmpdir(), 'kbreg-'));
|
||
const p = join(d, 'candidate.md');
|
||
try {
|
||
writeFileSync(p, ZW_BIDI, 'utf8');
|
||
const viaPath = await detectAdversarial(ZW_BIDI, { path: p });
|
||
assert.ok(viaPath.filter((f) => f.class === 'unicode').length >= 2, 'explicit-path route must still flag unicode');
|
||
} finally {
|
||
rmSync(d, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test('detectAdversarial cleans up its pre-write temp file (no leak)', { skip: !llmSecurityAvailable }, async () => {
|
||
const before = advTmpDirs().length;
|
||
await detectAdversarial(ZW_BIDI);
|
||
assert.equal(advTmpDirs().length, before, 'pre-write temp dir must be removed in finally');
|
||
});
|