feat(ms-ai-architect): Enhet A2 — Layer B baseline-adjudikering: innholds-basert allowlist (75 funn adjudikert, TDD) + 4 korpus-defekter fikset (2 homoglyph-ord, 2 U+00AD-filer); korpus 389/389 exit 0 [skip-docs]
This commit is contained in:
parent
135f34eb9b
commit
af6c31c4c1
11 changed files with 1048 additions and 15 deletions
216
tests/kb-update/test-layerb-allowlist.test.mjs
Normal file
216
tests/kb-update/test-layerb-allowlist.test.mjs
Normal file
|
|
@ -0,0 +1,216 @@
|
|||
// tests/kb-update/test-layerb-allowlist.test.mjs
|
||||
// Enhet A2 — the Layer B ALLOWLIST: adjudicated, per-file/line exceptions to the
|
||||
// adversarial-content gate (flag → human ONCE, then deterministically green).
|
||||
//
|
||||
// Design under test (docs/ingestion-security-brief-2026-07.md §8 / STATE Enhet A2):
|
||||
// - An allowlist ENTRY pins {path, class, evidence, tier, match} where `match` is the
|
||||
// exact TRIMMED content of the adjudicated line. Matching is CONTENT-based, not
|
||||
// line-number-based: a line that merely moves stays green; a line whose content
|
||||
// changes RESURFACES as WARN/BLOCK (fail-safe — re-adjudication required).
|
||||
// - The scanner is NOT weakened: only findings that match an adjudicated entry on
|
||||
// all four axes (class, evidence, tier, trimmed content) are downgraded to 'allow'.
|
||||
// - A missing or malformed allowlist file yields an EMPTY allowlist — the gate stays
|
||||
// fully strict (an allowlist can only be relaxed by a well-formed, committed file).
|
||||
//
|
||||
// Pure/DI throughout: no disk, no llm-security.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { classifyFindings } from '../../scripts/kb-update/lib/adversarial-scan.mjs';
|
||||
import { scanPaths, loadAllowlist } from '../../scripts/kb-update/scan-adversarial-content.mjs';
|
||||
import { runGate } from '../../scripts/kb-update/pre-commit-scan.mjs';
|
||||
|
||||
// A KB-style doc. Line map (1-indexed):
|
||||
// 1 # Title
|
||||
// 2 (blank)
|
||||
// 3 Prose documenting the attack: "ignore previous instructions" as an example.
|
||||
// 4 (blank)
|
||||
// 5 - [AI-systemer og styring](#ai-systemer-og-styring)
|
||||
const DOC =
|
||||
'# Title\n\n' +
|
||||
'Prose documenting the attack: "ignore previous instructions" as an example.\n\n' +
|
||||
'- [AI-systemer og styring](#ai-systemer-og-styring)\n';
|
||||
|
||||
const INJ_CRITICAL = {
|
||||
class: 'injection',
|
||||
severity: 'critical',
|
||||
line: 3,
|
||||
evidence: 'override: ignore previous instructions',
|
||||
};
|
||||
const INJ_ANCHOR = {
|
||||
class: 'injection',
|
||||
severity: 'medium',
|
||||
line: 5,
|
||||
evidence: 'markdown: injection payload in link anchor text',
|
||||
};
|
||||
|
||||
const ALLOW_CRITICAL = {
|
||||
path: 'skills/x/references/y/doc.md',
|
||||
class: 'injection',
|
||||
evidence: 'override: ignore previous instructions',
|
||||
tier: 'authored-doc',
|
||||
match: 'Prose documenting the attack: "ignore previous instructions" as an example.',
|
||||
line: 3,
|
||||
reason: 'security teaching: documents the attack pattern',
|
||||
adjudicated: '2026-07-18',
|
||||
};
|
||||
const ALLOW_ANCHOR = {
|
||||
path: 'skills/x/references/y/doc.md',
|
||||
class: 'injection',
|
||||
evidence: 'markdown: injection payload in link anchor text',
|
||||
tier: 'authored-doc',
|
||||
match: '- [AI-systemer og styring](#ai-systemer-og-styring)',
|
||||
line: 5,
|
||||
reason: 'benign TOC link; anchor text contains "system"',
|
||||
adjudicated: '2026-07-18',
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// classifyFindings + opts.allow — the pure matching core
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('classifyFindings — a fully matching allow entry downgrades the finding to allow, file goes clean', () => {
|
||||
const { disposition, findings } = classifyFindings(DOC, [INJ_CRITICAL], {
|
||||
allow: [ALLOW_CRITICAL],
|
||||
});
|
||||
assert.equal(findings[0].disposition, 'allow');
|
||||
assert.equal(disposition, 'clean');
|
||||
});
|
||||
|
||||
test('classifyFindings — allow requires exact trimmed-content match (changed line resurfaces)', () => {
|
||||
const { disposition, findings } = classifyFindings(DOC, [INJ_CRITICAL], {
|
||||
allow: [{ ...ALLOW_CRITICAL, match: 'Some OTHER line content entirely.' }],
|
||||
});
|
||||
assert.equal(findings[0].disposition, 'block');
|
||||
assert.equal(disposition, 'block');
|
||||
});
|
||||
|
||||
test('classifyFindings — allow matches on content, not line number (moved line stays green)', () => {
|
||||
// Same flagged content, but the line moved: prepend two lines.
|
||||
const moved = '<!-- new preamble -->\n\n' + DOC;
|
||||
const shifted = { ...INJ_CRITICAL, line: 5 };
|
||||
const { disposition, findings } = classifyFindings(moved, [shifted], {
|
||||
allow: [ALLOW_CRITICAL], // entry still says line 3 — must not matter
|
||||
});
|
||||
assert.equal(findings[0].disposition, 'allow');
|
||||
assert.equal(disposition, 'clean');
|
||||
});
|
||||
|
||||
test('classifyFindings — allow requires matching evidence', () => {
|
||||
const { findings } = classifyFindings(DOC, [INJ_CRITICAL], {
|
||||
allow: [{ ...ALLOW_CRITICAL, evidence: 'identity: pretend you are' }],
|
||||
});
|
||||
assert.equal(findings[0].disposition, 'block');
|
||||
});
|
||||
|
||||
test('classifyFindings — allow requires matching tier (code-sample entry does not cover authored prose)', () => {
|
||||
const { findings } = classifyFindings(DOC, [INJ_CRITICAL], {
|
||||
allow: [{ ...ALLOW_CRITICAL, tier: 'code-sample' }],
|
||||
});
|
||||
assert.equal(findings[0].disposition, 'block');
|
||||
});
|
||||
|
||||
test('classifyFindings — allow requires matching class', () => {
|
||||
const { findings } = classifyFindings(DOC, [INJ_CRITICAL], {
|
||||
allow: [{ ...ALLOW_CRITICAL, class: 'encoded' }],
|
||||
});
|
||||
assert.equal(findings[0].disposition, 'block');
|
||||
});
|
||||
|
||||
test('classifyFindings — non-allowlisted sibling finding in the same file still flags', () => {
|
||||
const { disposition, findings } = classifyFindings(DOC, [INJ_CRITICAL, INJ_ANCHOR], {
|
||||
allow: [ALLOW_CRITICAL], // only the critical one is adjudicated
|
||||
});
|
||||
assert.equal(findings[0].disposition, 'allow');
|
||||
assert.equal(findings[1].disposition, 'warn');
|
||||
assert.equal(disposition, 'warn');
|
||||
});
|
||||
|
||||
test('classifyFindings — empty/absent allow behaves exactly as before (gate unweakened)', () => {
|
||||
const a = classifyFindings(DOC, [INJ_CRITICAL], {});
|
||||
const b = classifyFindings(DOC, [INJ_CRITICAL], { allow: [] });
|
||||
assert.equal(a.disposition, 'block');
|
||||
assert.equal(b.disposition, 'block');
|
||||
assert.equal(a.findings[0].disposition, 'block');
|
||||
assert.equal(b.findings[0].disposition, 'block');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// loadAllowlist — fail-safe loading (missing/malformed → empty = fully strict)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('loadAllowlist — well-formed file yields its entries', () => {
|
||||
const entries = loadAllowlist({
|
||||
readFile: () => JSON.stringify({ entries: [ALLOW_CRITICAL] }),
|
||||
});
|
||||
assert.equal(entries.length, 1);
|
||||
assert.equal(entries[0].evidence, 'override: ignore previous instructions');
|
||||
});
|
||||
|
||||
test('loadAllowlist — missing file yields empty (gate stays strict)', () => {
|
||||
const entries = loadAllowlist({
|
||||
readFile: () => {
|
||||
throw new Error('ENOENT');
|
||||
},
|
||||
});
|
||||
assert.deepEqual(entries, []);
|
||||
});
|
||||
|
||||
test('loadAllowlist — malformed JSON yields empty (never a crash, never a weaker gate)', () => {
|
||||
const entries = loadAllowlist({ readFile: () => '{ not json' });
|
||||
assert.deepEqual(entries, []);
|
||||
});
|
||||
|
||||
test('loadAllowlist — non-array entries yields empty', () => {
|
||||
const entries = loadAllowlist({ readFile: () => JSON.stringify({ entries: 'nope' }) });
|
||||
assert.deepEqual(entries, []);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// scanPaths — allowlist injection + per-path scoping
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('scanPaths — injected allowlist turns an adjudicated file clean (ok=true)', async () => {
|
||||
const res = await scanPaths(['skills/x/references/y/doc.md'], {
|
||||
readFile: () => DOC,
|
||||
detect: async () => [INJ_CRITICAL],
|
||||
allowlist: [ALLOW_CRITICAL],
|
||||
});
|
||||
assert.equal(res.ok, true);
|
||||
assert.equal(res.blocked, false);
|
||||
assert.equal(res.results[0].disposition, 'clean');
|
||||
assert.equal(res.results[0].findings[0].disposition, 'allow');
|
||||
});
|
||||
|
||||
test('scanPaths — allowlist entries are scoped to their path (no cross-file bleed)', async () => {
|
||||
const res = await scanPaths(['skills/OTHER/references/z/other.md'], {
|
||||
readFile: () => DOC,
|
||||
detect: async () => [INJ_CRITICAL],
|
||||
allowlist: [ALLOW_CRITICAL], // pinned to skills/x/references/y/doc.md
|
||||
});
|
||||
assert.equal(res.blocked, true);
|
||||
assert.equal(res.results[0].disposition, 'block');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// runGate (pre-commit) — the commit boundary honours the same allowlist
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('runGate — adjudicated staged file passes the commit gate (exit 0)', async () => {
|
||||
const res = await runGate(['skills/x/references/y/doc.md'], {
|
||||
readFile: () => DOC,
|
||||
detect: async () => [INJ_CRITICAL],
|
||||
allowlist: [ALLOW_CRITICAL],
|
||||
});
|
||||
assert.equal(res.exitCode, 0);
|
||||
assert.equal(res.blocked, false);
|
||||
});
|
||||
|
||||
test('runGate — non-adjudicated finding still blocks the commit', async () => {
|
||||
const res = await runGate(['skills/x/references/y/doc.md'], {
|
||||
readFile: () => DOC,
|
||||
detect: async () => [INJ_CRITICAL, INJ_ANCHOR],
|
||||
allowlist: [ALLOW_CRITICAL],
|
||||
});
|
||||
assert.equal(res.exitCode, 2); // the anchor WARN is not adjudicated → flag → human
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue