feat(ms-ai-architect): RX-OPS1 — deterministisk Layer B commit-gate (pre-commit-scan) + runbook Forutsetning 0 [skip-docs]
pre-commit-scan.mjs mekaniserer G6 §8: scan-adversarial-content over stagede skills/**/*.md ved commit-grensen, uansett modell-adferd. Kun ingestion-flaten (skills/**/*.md); BLOCK/WARN aborterer commit (WARN = flag->human). DI'd (selectStagedKbFiles + runGate), 9 nye tester. E2E: clean->0, seeded adversarial->BLOCK exit 1, tom->0. Runbook Forutsetning 0 = §5b Layer A-gate + untrusted-data-ramme: judge-side fencing UTELATT bevisst (ville vaere judge-bump per brief §5b.4 / Non-goal §7); foreground-fetch + Layer B er de kompenserende kontrollene. Suite 910->919 exit 0 (+9). validate-plugin.sh 250/0.
This commit is contained in:
parent
f094e242d1
commit
6125f933be
3 changed files with 222 additions and 0 deletions
|
|
@ -8,6 +8,12 @@ v3 er adoptert baseline, målt **P 100,0 % / R 92,9 % / 0 FP** på G5b-korrigert
|
||||||
|
|
||||||
**Forhåndsregistrert adopsjonsgate (låst FØR fan-out):** adopter v3.1 KUN hvis den **holder P = 100 % OG løfter R over 92,9 %** (mot G5b-gull). Enhver ny FP feller den → behold v3. Rapportens egen «GATE: PASS» (R≥0,70/P≥0,60) er kun gulvet, IKKE adopsjonsbaren.
|
**Forhåndsregistrert adopsjonsgate (låst FØR fan-out):** adopter v3.1 KUN hvis den **holder P = 100 % OG løfter R over 92,9 %** (mot G5b-gull). Enhver ny FP feller den → behold v3. Rapportens egen «GATE: PASS» (R≥0,70/P≥0,60) er kun gulvet, IKKE adopsjonsbaren.
|
||||||
|
|
||||||
|
## Forutsetning 0 (R7-gate — FØR alt annet)
|
||||||
|
|
||||||
|
Denne kjøringen fetcher untrusted innhold (MS Learn → judge-subagenter). **Layer A-aktiveringsprotokollen `docs/ingestion-security-brief-2026-07.md` §5b (pkt. 1–5) MÅ være grønn før første fetch** — den er en hard R7-gate, ikke bare en aktiveringsregel. Kort: aktiver `post-mcp-verify`-hooken, verifiser at den fyrer på ÉN foreground-fetch (ellers kompenserende foreground-skann), og bekreft at `llm-security`-substratet løser (ellers fail-closer Layer B og BLOCKer alt).
|
||||||
|
|
||||||
|
**Untrusted-data-ramme (hvorfor judgen IKKE fences):** det fetchede innholdet prompt-fences **ikke** inn i den frosne v3.1-judgen. Å legge fencing rundt `<FILE>`/`<CLAIMS>` (som ligger *inne i* den frosne malen — det finnes intet skall utenfor judgens synsfelt) ville endre teksten judgen prosesserer = en judge-bump (Non-goal §7, ville kreve re-måling av P/R). De kompenserende kontrollene er i stedet: (a) **foreground-fetch obligatorisk** (lar Layer A-hooken + operatørens øye se innholdet før judgen), og (b) **Layer B ved commit-grensen** — `scripts/kb-update/pre-commit-scan.mjs` kjører den deterministiske adversarial-skannen over stagede `skills/**/*.md` og BLOCKer commit på BLOCK/WARN, uansett modell-adferd. Judgen forblir frosset.
|
||||||
|
|
||||||
## Forutsetninger (verifiser FØRST — premiss-sjekk)
|
## Forutsetninger (verifiser FØRST — premiss-sjekk)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|
|
||||||
105
scripts/kb-update/pre-commit-scan.mjs
Normal file
105
scripts/kb-update/pre-commit-scan.mjs
Normal file
|
|
@ -0,0 +1,105 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
// pre-commit-scan.mjs — RX-OPS1: deterministic Layer B enforcement at the commit boundary.
|
||||||
|
//
|
||||||
|
// Mechanizes G6 §8 / R6 punkt (d): the adversarial-content scan (scan-adversarial-content.mjs)
|
||||||
|
// runs over the STAGED untrusted-ingestion surface (skills/**/*.md) BEFORE a commit lands,
|
||||||
|
// regardless of model behaviour. A generator (kb-update apply, generate-skills, a future R7
|
||||||
|
// judge-pass) may already call Layer B in-process; this wrapper makes it a hard, model-independent
|
||||||
|
// gate at the git boundary as well — a poisoned candidate that slips past the in-process call is
|
||||||
|
// still stopped here.
|
||||||
|
//
|
||||||
|
// It scans ONLY staged skills/**/*.md — the surface fed by MS Learn fetches (docs/, scripts/,
|
||||||
|
// tests/ are first-party and out of scope, matching the ingestion brief). It maps the scan
|
||||||
|
// disposition to a commit-blocking exit code: any BLOCK or WARN aborts the commit (a WARN is
|
||||||
|
// "flag → human, never auto-committed", so it must stop an unattended commit too).
|
||||||
|
//
|
||||||
|
// This is Layer B ONLY. It deliberately does NOT touch the frozen groundedness judge: per the
|
||||||
|
// ingestion brief §5b.4, fencing the fetched content into the judge would be a judge-bump
|
||||||
|
// (Non-goal §7). Foreground fetch + this scan are the compensating controls; the judge is
|
||||||
|
// untouched. See docs/v3.1-fanout-runbook.md "Forutsetning 0" and docs/ingestion-security-brief-2026-07.md §5b.
|
||||||
|
//
|
||||||
|
// Install (dev): symlink or copy as .git/hooks/pre-commit, or wire via a PreToolUse Bash gate.
|
||||||
|
// Exit code: 0 = clean (safe to commit); 1 = at least one BLOCK; 2 = at least one WARN.
|
||||||
|
|
||||||
|
import { execFileSync } from 'node:child_process';
|
||||||
|
import { realpathSync } from 'node:fs';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
import { scanPaths } from './scan-adversarial-content.mjs';
|
||||||
|
|
||||||
|
// Repo-relative, POSIX-separated paths as emitted by `git diff --cached --name-only`.
|
||||||
|
// Anchored at skills/ so a sibling like notskills/ never matches; exact `.md` only (not .mdx).
|
||||||
|
const KB_RE = /^skills\/.+\.md$/;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Keep only the staged files on the untrusted-ingestion surface (skills/**\/*.md).
|
||||||
|
* @param {string[]} names — raw staged path list
|
||||||
|
* @returns {string[]}
|
||||||
|
*/
|
||||||
|
export function selectStagedKbFiles(names) {
|
||||||
|
return (names ?? []).filter((n) => KB_RE.test(n));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run the Layer B gate over the staged path list. Pure + dependency-injected (scanPaths' deps
|
||||||
|
* pass straight through), so it is unit-testable without git, disk, or llm-security.
|
||||||
|
*
|
||||||
|
* @param {string[]} stagedNames — raw `git diff --cached` path list
|
||||||
|
* @param {object} [deps] — forwarded to scanPaths ({readFile, detect})
|
||||||
|
* @returns {Promise<{exitCode: 0|1|2, blocked: boolean, warned: boolean, kb: string[], results: object[]}>}
|
||||||
|
*/
|
||||||
|
export async function runGate(stagedNames, deps = {}) {
|
||||||
|
const kb = selectStagedKbFiles(stagedNames);
|
||||||
|
const { blocked, warned, results } = await scanPaths(kb, deps);
|
||||||
|
const exitCode = blocked ? 1 : warned ? 2 : 0;
|
||||||
|
return { exitCode, blocked, warned, kb, results };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Raw staged path list from git. NUL-separated (-z) so filenames with spaces/newlines survive.
|
||||||
|
* @param {object} [deps] — {exec} injectable for tests
|
||||||
|
* @returns {string[]}
|
||||||
|
*/
|
||||||
|
function stagedNames(deps = {}) {
|
||||||
|
const exec =
|
||||||
|
deps.exec ??
|
||||||
|
(() => execFileSync('git', ['diff', '--cached', '--name-only', '--diff-filter=ACM', '-z'], { encoding: 'utf8' }));
|
||||||
|
return exec().split('\0').filter(Boolean);
|
||||||
|
}
|
||||||
|
|
||||||
|
function report({ kb, results }) {
|
||||||
|
if (kb.length === 0) {
|
||||||
|
process.stdout.write('Layer B pre-commit scan: no staged skills/**/*.md — nothing to gate.\n');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
process.stdout.write(`Layer B pre-commit scan: ${kb.length} staged KB file(s).\n`);
|
||||||
|
for (const r of results) {
|
||||||
|
if (r.disposition === 'clean') {
|
||||||
|
process.stdout.write(` OK ${r.path}\n`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const marker = r.disposition === 'block' ? 'BLOCK' : 'WARN ';
|
||||||
|
process.stdout.write(` ${marker} ${r.path}\n`);
|
||||||
|
for (const f of r.findings) {
|
||||||
|
if (f.disposition === 'clean' || f.disposition === 'pass') continue;
|
||||||
|
const tier = f.tier ? ` [${f.tier}]` : '';
|
||||||
|
process.stdout.write(` ${(f.disposition || 'flag').toUpperCase()} ${f.class}/${f.subtype ?? f.severity}${tier} line ${f.line}: ${f.evidence}\n`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const { exitCode, kb, results, blocked, warned } = await runGate(stagedNames());
|
||||||
|
report({ kb, results });
|
||||||
|
if (blocked) process.stderr.write('\nCommit BLOCKED: adversarial content in a staged KB file. Nothing was committed.\n');
|
||||||
|
else if (warned) process.stderr.write('\nCommit HELD: a staged KB file needs operator review (flag → human). Nothing was committed.\n');
|
||||||
|
process.exit(exitCode);
|
||||||
|
}
|
||||||
|
|
||||||
|
const isMain = (() => {
|
||||||
|
try {
|
||||||
|
return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
if (isMain) main();
|
||||||
111
tests/kb-update/test-pre-commit-scan.test.mjs
Normal file
111
tests/kb-update/test-pre-commit-scan.test.mjs
Normal file
|
|
@ -0,0 +1,111 @@
|
||||||
|
// tests/kb-update/test-pre-commit-scan.test.mjs
|
||||||
|
// RX-OPS1 — deterministic Layer B enforcement at the commit boundary
|
||||||
|
// (scripts/kb-update/pre-commit-scan.mjs). This wraps scan-adversarial-content's scanPaths
|
||||||
|
// with two new responsibilities that are the actual point of the gate:
|
||||||
|
// (1) select ONLY staged skills/**/*.md (the untrusted-ingestion surface) from the raw
|
||||||
|
// `git diff --cached` list — never scan docs/, scripts/, tests/, or non-.md files;
|
||||||
|
// (2) map the scan disposition to a commit-blocking exit code — 0 clean, 1 block, 2 warn —
|
||||||
|
// so BOTH a BLOCK and a WARN abort the commit (a WARN is "flag → human, never
|
||||||
|
// auto-commit", so it must stop an unattended commit too).
|
||||||
|
//
|
||||||
|
// The git invocation and the detector are dependency-injected, so these tests exercise the
|
||||||
|
// selection + exit-code contract without touching git, disk, or llm-security. FAIL-CLOSED
|
||||||
|
// behaviour is inherited from scanPaths and re-asserted here at the gate layer.
|
||||||
|
|
||||||
|
import { test } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { selectStagedKbFiles, runGate } from '../../scripts/kb-update/pre-commit-scan.mjs';
|
||||||
|
|
||||||
|
const CLEAN = '# T\n\n**Source:** https://learn.microsoft.com/azure/x\n\nProse.\n';
|
||||||
|
|
||||||
|
function reader(map) {
|
||||||
|
return (p) => {
|
||||||
|
if (!(p in map)) throw new Error(`ENOENT: ${p}`);
|
||||||
|
return map[p];
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const detectClean = async () => [];
|
||||||
|
const detectBlock = async () => [{ class: 'injection', severity: 'critical', line: 5, evidence: 'spoofed tag: <system>' }];
|
||||||
|
const detectWarn = async () => [{ class: 'injection', severity: 'high', line: 5, evidence: 'imperative in prose' }];
|
||||||
|
|
||||||
|
// --- selection: only staged skills/**/*.md reach the scanner ---------------------------
|
||||||
|
|
||||||
|
test('selectStagedKbFiles — keeps skills/**/*.md, drops everything else', () => {
|
||||||
|
const staged = [
|
||||||
|
'skills/ms-ai-engineering/references/rag/patterns.md', // kept
|
||||||
|
'skills/ms-ai-advisor/SKILL.md', // kept
|
||||||
|
'docs/v3.1-fanout-runbook.md', // dropped (not under skills/)
|
||||||
|
'scripts/kb-update/pre-commit-scan.mjs', // dropped (not .md)
|
||||||
|
'tests/kb-update/test-pre-commit-scan.test.mjs', // dropped
|
||||||
|
'skills/x/notes.mdx', // dropped (.mdx, not .md)
|
||||||
|
'skills/x/README', // dropped (no extension)
|
||||||
|
'notskills/y.md', // dropped (prefix, not anchored)
|
||||||
|
];
|
||||||
|
assert.deepEqual(selectStagedKbFiles(staged), [
|
||||||
|
'skills/ms-ai-engineering/references/rag/patterns.md',
|
||||||
|
'skills/ms-ai-advisor/SKILL.md',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('selectStagedKbFiles — empty / nullish input → []', () => {
|
||||||
|
assert.deepEqual(selectStagedKbFiles([]), []);
|
||||||
|
assert.deepEqual(selectStagedKbFiles(undefined), []);
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- exit-code contract: BLOCK→1, WARN→2, clean→0 --------------------------------------
|
||||||
|
|
||||||
|
test('runGate — no staged KB files → exit 0 (nothing to gate)', async () => {
|
||||||
|
const r = await runGate(['docs/x.md', 'scripts/y.mjs'], { readFile: reader({}), detect: detectClean });
|
||||||
|
assert.equal(r.exitCode, 0);
|
||||||
|
assert.deepEqual(r.kb, []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('runGate — all staged KB files clean → exit 0', async () => {
|
||||||
|
const staged = ['skills/a/x.md', 'skills/b/y.md'];
|
||||||
|
const r = await runGate(staged, { readFile: reader({ 'skills/a/x.md': CLEAN, 'skills/b/y.md': CLEAN }), detect: detectClean });
|
||||||
|
assert.equal(r.exitCode, 0);
|
||||||
|
assert.equal(r.blocked, false);
|
||||||
|
assert.equal(r.warned, false);
|
||||||
|
assert.deepEqual(r.kb, staged);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('runGate — a BLOCKED staged KB file → exit 1 (commit aborts)', async () => {
|
||||||
|
const r = await runGate(['skills/a/bad.md'], { readFile: reader({ 'skills/a/bad.md': CLEAN }), detect: detectBlock });
|
||||||
|
assert.equal(r.exitCode, 1);
|
||||||
|
assert.equal(r.blocked, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('runGate — a WARN-only staged KB file → exit 2 (commit aborts, flag→human)', async () => {
|
||||||
|
const r = await runGate(['skills/a/warn.md'], { readFile: reader({ 'skills/a/warn.md': CLEAN }), detect: detectWarn });
|
||||||
|
assert.equal(r.exitCode, 2);
|
||||||
|
assert.equal(r.warned, true);
|
||||||
|
assert.equal(r.blocked, false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('runGate — a poisoned NON-KB staged file is not scanned → gate stays 0', async () => {
|
||||||
|
// docs/evil.md is staged and would BLOCK if scanned, but it is outside skills/ → ignored.
|
||||||
|
const detect = async (content) => (content === 'POISON' ? detectBlock() : []);
|
||||||
|
const r = await runGate(['docs/evil.md', 'skills/a/ok.md'], {
|
||||||
|
readFile: reader({ 'docs/evil.md': 'POISON', 'skills/a/ok.md': CLEAN }),
|
||||||
|
detect,
|
||||||
|
});
|
||||||
|
assert.equal(r.exitCode, 0);
|
||||||
|
assert.deepEqual(r.kb, ['skills/a/ok.md']);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('runGate — BLOCK takes precedence over WARN in a mixed batch → exit 1', async () => {
|
||||||
|
const detect = async (content) => (content === 'B' ? detectBlock() : content === 'W' ? detectWarn() : []);
|
||||||
|
const r = await runGate(['skills/a/w.md', 'skills/a/b.md'], {
|
||||||
|
readFile: reader({ 'skills/a/w.md': 'W', 'skills/a/b.md': 'B' }),
|
||||||
|
detect,
|
||||||
|
});
|
||||||
|
assert.equal(r.exitCode, 1);
|
||||||
|
assert.equal(r.blocked, true);
|
||||||
|
assert.equal(r.warned, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('runGate — an unreadable staged KB file FAILS CLOSED → exit 1', async () => {
|
||||||
|
const r = await runGate(['skills/a/gone.md'], { readFile: () => { throw new Error('ENOENT'); }, detect: detectClean });
|
||||||
|
assert.equal(r.exitCode, 1);
|
||||||
|
assert.equal(r.blocked, true);
|
||||||
|
});
|
||||||
Loading…
Add table
Add a link
Reference in a new issue