fix(llm-security): scanner false-positives/negatives — trigger, toxic-flow, policy-loader (#38-#41,#57,#26)
#38 toxic-flow matched trifecta-leg keywords with bare includes(), so substrings ('url' in 'curl', 'key' in 'monkey', 'auth' in 'author') fabricated CRITICAL trifectas on benign components; now word-boundary matched. #40 TRG-broad fired HIGH on a bare any/all/every anywhere ('fix any lint errors'); the universal-claim regex now requires genuine universal phrasing. #41 TRG-baiting substring-matched ('any file' in 'many files'); now boundary-anchored. #39 the broad-name list missed multi-char generic names (helper/assistant/auto/general/agent/tool); widened coherently so it does not reintroduce #40. #57 the '(recovered from obfuscation)' label compared raw against a lowercased normal form, firing on any uppercase char; now gated on an explicit decode-changed flag. #26 (same file) getPolicyValue used 'key in sectionObj' with no type guard, so a scalar section override in policy.json (e.g. {"injection":"block"}) threw an uncaught TypeError; now guarded to fall back to the default. Suite 1931/0. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TcQyMTQfyrsAapaCMPxTtQ
This commit is contained in:
parent
196517f38a
commit
f3aaf5479f
6 changed files with 339 additions and 13 deletions
|
|
@ -81,6 +81,7 @@ const DEFAULT_POLICY = Object.freeze({
|
|||
],
|
||||
broad_single_words: [
|
||||
'run', 'do', 'go', 'help', 'fix', 'use', 'get', 'set', 'all', 'any', 'it', 'this', 'that',
|
||||
'helper', 'assistant', 'auto', 'general', 'agent', 'tool',
|
||||
],
|
||||
},
|
||||
// SIG — known-bad-identity signature engine. Toggle families or point at a
|
||||
|
|
@ -180,7 +181,10 @@ export function loadPolicy(projectRoot) {
|
|||
export function getPolicyValue(section, key, defaultValue, projectRoot) {
|
||||
const policy = loadPolicy(projectRoot);
|
||||
const sectionObj = policy[section];
|
||||
if (sectionObj && key in sectionObj) return sectionObj[key];
|
||||
// v7.8.3 (#26): a scalar section override in policy.json (e.g.
|
||||
// {"injection": "block"}) survives deepMerge — guard before `in` so it
|
||||
// falls back to the default instead of throwing a TypeError.
|
||||
if (sectionObj && typeof sectionObj === 'object' && key in sectionObj) return sectionObj[key];
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -64,6 +64,26 @@ const EXFIL_KEYWORDS = [
|
|||
'network', 'api', 'endpoint', 'transfer', 'exfil',
|
||||
];
|
||||
|
||||
/**
|
||||
* Compile a keyword into a word-boundary-anchored regex (#38), so 'url' no
|
||||
* longer matches inside 'curl', 'key' inside 'monkey', 'auth' inside
|
||||
* 'author', or 'api' inside 'rapidly'. Boundary guards are only applied
|
||||
* where the keyword edge is a word character (so '.env' still matches
|
||||
* 'config.env'); a trailing plural 's' is allowed ('credentials', 'tokens').
|
||||
*/
|
||||
function keywordToRegex(kw) {
|
||||
const escaped = String(kw).split(/\s+/)
|
||||
.map(w => w.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
|
||||
.join('\\s+');
|
||||
const lead = /^[a-z0-9_$]/i.test(kw) ? '(?<![a-z0-9_$])' : '';
|
||||
const trail = /[a-z0-9_]$/i.test(kw) ? 's?(?![a-z0-9_])' : '';
|
||||
return new RegExp(lead + escaped + trail, 'i');
|
||||
}
|
||||
|
||||
const INPUT_KEYWORD_MATCHERS = INPUT_KEYWORDS.map(kw => ({ kw, re: keywordToRegex(kw) }));
|
||||
const SENSITIVE_KEYWORD_MATCHERS = SENSITIVE_KEYWORDS.map(kw => ({ kw, re: keywordToRegex(kw) }));
|
||||
const EXFIL_KEYWORD_MATCHERS = EXFIL_KEYWORDS.map(kw => ({ kw, re: keywordToRegex(kw) }));
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Hook guard patterns — known hooks that mitigate exfil paths
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -234,9 +254,9 @@ function classifyTrifectaLegs(components, priorResults, mcpPresent) {
|
|||
comp.inputEvidence.push('$ARGUMENTS in command body');
|
||||
}
|
||||
|
||||
// Keyword-based
|
||||
for (const kw of INPUT_KEYWORDS) {
|
||||
if (comp.description.includes(kw) || comp.body.includes(kw)) {
|
||||
// Keyword-based (word-boundary matched, #38)
|
||||
for (const { kw, re } of INPUT_KEYWORD_MATCHERS) {
|
||||
if (re.test(comp.description) || re.test(comp.body)) {
|
||||
comp.hasInputSurface = true;
|
||||
comp.inputEvidence.push(`keyword "${kw}"`);
|
||||
break;
|
||||
|
|
@ -267,8 +287,8 @@ function classifyTrifectaLegs(components, priorResults, mcpPresent) {
|
|||
}
|
||||
}
|
||||
|
||||
for (const kw of SENSITIVE_KEYWORDS) {
|
||||
if (comp.description.includes(kw) || comp.body.includes(kw)) {
|
||||
for (const { kw, re } of SENSITIVE_KEYWORD_MATCHERS) {
|
||||
if (re.test(comp.description) || re.test(comp.body)) {
|
||||
comp.hasDataAccess = true;
|
||||
comp.accessEvidence.push(`keyword "${kw}"`);
|
||||
break;
|
||||
|
|
@ -290,8 +310,8 @@ function classifyTrifectaLegs(components, priorResults, mcpPresent) {
|
|||
comp.exfilEvidence.push(`delegation: ${matched.join(', ')} (can spawn capable sub-agents)`);
|
||||
}
|
||||
|
||||
for (const kw of EXFIL_KEYWORDS) {
|
||||
if (comp.description.includes(kw) || comp.body.includes(kw)) {
|
||||
for (const { kw, re } of EXFIL_KEYWORD_MATCHERS) {
|
||||
if (re.test(comp.description) || re.test(comp.body)) {
|
||||
comp.hasExfilSink = true;
|
||||
comp.exfilEvidence.push(`keyword "${kw}"`);
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -44,10 +44,20 @@ const DEFAULT_BUILTIN_NAMES = [
|
|||
|
||||
const DEFAULT_BROAD_SINGLE_WORDS = [
|
||||
'run', 'do', 'go', 'help', 'fix', 'use', 'get', 'set', 'all', 'any', 'it', 'this', 'that',
|
||||
'helper', 'assistant', 'auto', 'general', 'agent', 'tool',
|
||||
];
|
||||
|
||||
// Bare universal-applicability claim words used only for the TRG-broad combo.
|
||||
const UNIVERSAL_CLAIM_RE = /\b(any|all|every|everything|anything|universal|whenever|always|no matter what)\b/i;
|
||||
// Universal-applicability claim used only for the TRG-broad combo. A bare
|
||||
// quantifier inside a scoped noun phrase ("any lint errors", "all tests") is
|
||||
// NOT a universal claim (#40) — the quantifier must pair with a universal
|
||||
// noun ("any request"), an unbounded pronoun ("anything"), or an
|
||||
// unconditional-invocation phrase ("always invoke", "no matter what").
|
||||
const UNIVERSAL_CLAIM_RE = new RegExp(
|
||||
'\\b(?:anything|everything|no matter what|universal(?:ly)?'
|
||||
+ '|(?:any|all|every)\\s+(?:request|task|prompt|question|message|input|situation|scenario|purpose|context|case|time|user)s?'
|
||||
+ '|always\\s+(?:use|invoke|activate|apply|run|trigger)'
|
||||
+ '|whenever\\s+(?:the\\s+user|you|asked|possible)'
|
||||
+ ')\\b', 'i');
|
||||
|
||||
// Invisible / zero-width characters used to split keywords (ZWSP, ZWNJ, ZWJ,
|
||||
// word-joiner, BOM/zero-width-no-break-space).
|
||||
|
|
@ -76,6 +86,22 @@ function normalizeText(s) {
|
|||
return normalizeForScan(foldHomoglyphs(noZeroWidth)).toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Compile a baiting phrase into a word-boundary-anchored regex (#41), so
|
||||
* "any file" no longer matches inside "many files" nor "all files" inside
|
||||
* "install files". Boundary anchors are only applied where the phrase edge
|
||||
* is a word character; internal whitespace matches flexibly.
|
||||
*/
|
||||
function phraseToRegex(phrase) {
|
||||
const trimmed = String(phrase).trim();
|
||||
const escaped = trimmed.split(/\s+/)
|
||||
.map(w => w.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
|
||||
.join('\\s+');
|
||||
const lead = /^[a-z0-9]/i.test(trimmed) ? '\\b' : '';
|
||||
const trail = /[a-z0-9]$/i.test(trimmed) ? '\\b' : '';
|
||||
return new RegExp(lead + escaped + trail, 'i');
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan command/agent/skill definitions for trigger/activation abuse.
|
||||
*
|
||||
|
|
@ -95,6 +121,9 @@ export async function scan(targetPath, discovery) {
|
|||
.map(n => String(n).toLowerCase());
|
||||
const broadWords = (getPolicyValue('trg', 'broad_single_words', DEFAULT_BROAD_SINGLE_WORDS, targetPath) || [])
|
||||
.map(w => String(w).toLowerCase());
|
||||
const baitingMatchers = baitingPhrases
|
||||
.filter(Boolean)
|
||||
.map(p => ({ phrase: p, re: phraseToRegex(p) }));
|
||||
|
||||
for (const fileInfo of discovery.files) {
|
||||
if (!isTriggerFile(fileInfo.relPath)) continue;
|
||||
|
|
@ -111,9 +140,15 @@ export async function scan(targetPath, discovery) {
|
|||
const nameLower = name.toLowerCase();
|
||||
const rawDesc = typeof fm.description === 'string' ? fm.description : '';
|
||||
const normDesc = normalizeText(rawDesc);
|
||||
const plainDesc = rawDesc.toLowerCase();
|
||||
const fileType = fileTypeOf(fileInfo.relPath);
|
||||
|
||||
const isBroadName = broadWords.includes(nameLower) || nameLower.length <= 2;
|
||||
// Broad name: a listed generic word, a very short name, or a compound
|
||||
// name made entirely of generic words ("helper-agent") (#39).
|
||||
const nameTokens = nameLower.split(/[^a-z0-9]+/).filter(Boolean);
|
||||
const isBroadName = broadWords.includes(nameLower)
|
||||
|| nameLower.length <= 2
|
||||
|| (nameTokens.length > 0 && nameTokens.every(t => broadWords.includes(t)));
|
||||
const hasUniversalClaim = UNIVERSAL_CLAIM_RE.test(normDesc);
|
||||
|
||||
// TRG-shadow — name collides with a built-in command/tool.
|
||||
|
|
@ -147,14 +182,19 @@ export async function scan(targetPath, discovery) {
|
|||
}
|
||||
|
||||
// TRG-baiting — maximally-activating phrases in the (decoded) description.
|
||||
const matched = baitingPhrases.filter(p => p && normDesc.includes(p));
|
||||
const matched = baitingMatchers.filter(m => m.re.test(normDesc)).map(m => m.phrase);
|
||||
// "(recovered from obfuscation)" only when a matched phrase is NOT
|
||||
// findable in the plain lowercased raw text — i.e. the decode pipeline
|
||||
// actually recovered it (#57). Mere case differences do not count.
|
||||
const recovered = baitingMatchers.some(m =>
|
||||
m.re.test(normDesc) && !m.re.test(plainDesc));
|
||||
if (matched.length > 0) {
|
||||
findings.push(finding({
|
||||
scanner: 'TRG',
|
||||
severity: SEVERITY.MEDIUM,
|
||||
title: `Activation baiting phrase in ${fileType} description`,
|
||||
description: `The ${fileType} "${name}" description uses maximally-activating phrasing that baits the `
|
||||
+ `model into invoking it indiscriminately${rawDesc === normDesc ? '' : ' (recovered from obfuscation)'}.`,
|
||||
+ `model into invoking it indiscriminately${recovered ? ' (recovered from obfuscation)' : ''}.`,
|
||||
file: fileInfo.relPath,
|
||||
evidence: `phrases: ${matched.join(', ')}`,
|
||||
owasp: 'LLM06',
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue