ktg-plugin-marketplace/plugins/ai-psychosis/tests/prompt-analyzer.test.mjs

522 lines
21 KiB
JavaScript

import { describe, it, afterEach } from 'node:test';
import assert from 'node:assert/strict';
import { runHook, setupTestDir, cleanupTestDir, createStateFile, readState } from './test-helper.mjs';
let dir;
function freshState() {
return {
start_epoch: Math.floor(Date.now() / 1000) - 60,
start_iso: '2026-01-01T10:00:00Z',
tool_count: 0, edit_count: 0,
last_event_epoch: 0, burst_count: 0,
dep_flags: 0, esc_flags: 0, fatigue_flags: 0, val_flags: 0,
pushback_count: 0, domain_context: null,
last_warning_epoch: 0,
};
}
function runPrompt(prompt, stateOverrides = {}) {
dir = setupTestDir();
createStateFile(dir, 'p1', { ...freshState(), ...stateOverrides });
runHook('prompt-analyzer.mjs', { session_id: 'p1', prompt }, dir);
return readState(dir, 'p1');
}
afterEach(() => { if (dir) cleanupTestDir(dir); });
// --- Dependency patterns (7 positive, 7 negative) ---
describe('dependency patterns', () => {
it('matches "tell me what to do"', () => {
const s = runPrompt('please tell me what to do');
assert.equal(s.dep_flags, 1);
});
it('does not match "I told him the plan"', () => {
const s = runPrompt('I told him the plan');
assert.equal(s.dep_flags, 0);
});
it('matches "what should I do"', () => {
const s = runPrompt('what should I do next?');
assert.equal(s.dep_flags, 1);
});
it('does not match "I know what to build"', () => {
const s = runPrompt('I know what to build');
assert.equal(s.dep_flags, 0);
});
it('matches "am I right"', () => {
const s = runPrompt('am I right about this?');
assert.equal(s.dep_flags, 1);
});
it('does not match "I turned the server right"', () => {
const s = runPrompt('I turned the server on');
assert.equal(s.dep_flags, 0);
});
it('matches "you understand me"', () => {
const s = runPrompt('you understand me well');
assert.equal(s.dep_flags, 1);
});
it('does not match "do you understand merging?"', () => {
const s = runPrompt('do you understand merging?');
assert.equal(s.dep_flags, 0);
});
it('matches "you\'re the only"', () => {
const s = runPrompt("you're the only one who gets it");
assert.equal(s.dep_flags, 1);
});
it('does not match "the only option is refactoring"', () => {
const s = runPrompt('the only option is refactoring');
assert.equal(s.dep_flags, 0);
});
it('matches "can I do this"', () => {
const s = runPrompt('can I do this alone?');
assert.equal(s.dep_flags, 1);
});
it('does not match "we can implement this later"', () => {
const s = runPrompt('we can implement this later');
assert.equal(s.dep_flags, 0);
});
it('matches "I need you to decide"', () => {
const s = runPrompt('I need you to decide for me');
assert.equal(s.dep_flags, 1);
});
it('does not match "we need to deploy soon"', () => {
const s = runPrompt('we need to deploy soon');
assert.equal(s.dep_flags, 0);
});
});
// --- Escalation patterns (6 positive, 6 negative) ---
describe('escalation patterns', () => {
it('matches "definitely" as word', () => {
const s = runPrompt('this is definitely wrong');
assert.equal(s.esc_flags, 1);
});
it('does not match "definitively"', () => {
const s = runPrompt('this is definitively proven');
assert.equal(s.esc_flags, 0);
});
it('matches "clearly" as word', () => {
const s = runPrompt('clearly this is the issue');
assert.equal(s.esc_flags, 1);
});
it('does not match "nuclear unclear"', () => {
const s = runPrompt('nuclear unclear situation');
assert.equal(s.esc_flags, 0);
});
it('matches "this proves"', () => {
const s = runPrompt('this proves my point');
assert.equal(s.esc_flags, 1);
});
it('does not match "prove this theorem"', () => {
const s = runPrompt('prove this theorem');
assert.equal(s.esc_flags, 0);
});
it('matches "obviously" as word', () => {
const s = runPrompt('obviously we should refactor');
assert.equal(s.esc_flags, 1);
});
it('does not match "not an obvious choice"', () => {
const s = runPrompt('not an obvious choice');
assert.equal(s.esc_flags, 0);
});
it('matches "without a doubt"', () => {
const s = runPrompt('without a doubt this works');
assert.equal(s.esc_flags, 1);
});
it('does not match "I have some doubt"', () => {
const s = runPrompt('I have some doubt about it');
assert.equal(s.esc_flags, 0);
});
it('matches "this confirms"', () => {
const s = runPrompt('this confirms the theory');
assert.equal(s.esc_flags, 1);
});
it('does not match "please confirm the deploy"', () => {
const s = runPrompt('please confirm the deploy');
assert.equal(s.esc_flags, 0);
});
});
// --- Fatigue patterns (7 positive, 7 negative) ---
describe('fatigue patterns', () => {
it('matches "tired"', () => {
const s = runPrompt("I'm tired of debugging");
assert.equal(s.fatigue_flags, 1);
});
it('does not match "retired"', () => {
const s = runPrompt('I retired last year');
assert.equal(s.fatigue_flags, 0);
});
it('matches "exhausted"', () => {
const s = runPrompt("I'm exhausted.");
assert.equal(s.fatigue_flags, 1);
});
it('does not match "exhaustive"', () => {
const s = runPrompt('the options were exhaustive');
assert.equal(s.fatigue_flags, 0);
});
it('matches "can\'t think"', () => {
const s = runPrompt("I can't think straight");
assert.equal(s.fatigue_flags, 1);
});
it('does not match "I can think clearly"', () => {
const s = runPrompt('I can think of a solution');
assert.equal(s.fatigue_flags, 0);
});
it('matches "been at this"', () => {
const s = runPrompt("I've been at this all day");
assert.equal(s.fatigue_flags, 1);
});
it('does not match "haven\'t been at home"', () => {
const s = runPrompt("I haven't been at home");
assert.equal(s.fatigue_flags, 0);
});
it('matches "it\'s late"', () => {
const s = runPrompt("it's late, wrapping up");
assert.equal(s.fatigue_flags, 1);
});
it('does not match "the latest version"', () => {
const s = runPrompt('the latest version is good');
assert.equal(s.fatigue_flags, 0);
});
it('matches "should sleep"', () => {
const s = runPrompt('I should sleep');
assert.equal(s.fatigue_flags, 1);
});
it('does not match "sleep mode is enabled"', () => {
const s = runPrompt('enable sleep mode');
assert.equal(s.fatigue_flags, 0);
});
it('matches "hours now"', () => {
const s = runPrompt('been going for hours now');
assert.equal(s.fatigue_flags, 1);
});
it('does not match "hourly updates"', () => {
const s = runPrompt('hourly updates are fine');
assert.equal(s.fatigue_flags, 0);
});
});
// --- Validation patterns (5 positive, 5 negative) ---
describe('validation patterns', () => {
it('matches "right?"', () => {
const s = runPrompt('this works, right?');
assert.equal(s.val_flags, 1);
});
it('does not match "turn right"', () => {
const s = runPrompt('turn right at the fork');
assert.equal(s.val_flags, 0);
});
it('matches "don\'t you think"', () => {
const s = runPrompt("don't you think so?");
assert.equal(s.val_flags, 1);
});
it('does not match "I don\'t think so"', () => {
const s = runPrompt("I don't think so");
assert.equal(s.val_flags, 0);
});
it('matches "you agree"', () => {
const s = runPrompt('you agree with me');
assert.equal(s.val_flags, 1);
});
it('does not match "if parties agree"', () => {
const s = runPrompt('if parties agree on terms');
assert.equal(s.val_flags, 0);
});
it('matches "correct?"', () => {
const s = runPrompt('is this correct?');
assert.equal(s.val_flags, 1);
});
it('does not match "correct the typo"', () => {
const s = runPrompt("I'll correct the typo");
assert.equal(s.val_flags, 0);
});
it('matches "isn\'t it"', () => {
const s = runPrompt('good approach, isn\'t it');
assert.equal(s.val_flags, 1);
});
it('does not match "it isn\'t working"', () => {
const s = runPrompt("it isn't working yet");
assert.equal(s.val_flags, 0);
});
});
// --- Threshold and cooldown tests (6 cases) ---
describe('thresholds and cooldowns', () => {
it('warns at dependency soft threshold (2 flags)', () => {
dir = setupTestDir();
createStateFile(dir, 'p1', { ...freshState(), dep_flags: 1 });
const out = runHook('prompt-analyzer.mjs', { session_id: 'p1', prompt: 'tell me what to do' }, dir);
assert.ok(out.hookSpecificOutput?.additionalContext?.includes('Dependency language noticed'));
});
it('warns hard at dependency threshold (5 flags)', () => {
dir = setupTestDir();
createStateFile(dir, 'p1', { ...freshState(), dep_flags: 4 });
const out = runHook('prompt-analyzer.mjs', { session_id: 'p1', prompt: 'tell me what to do' }, dir);
assert.ok(out.hookSpecificOutput?.additionalContext?.includes('INTERACTION AWARENESS'));
});
it('fatigue bypasses cooldown', () => {
dir = setupTestDir();
createStateFile(dir, 'p1', { ...freshState(), last_warning_epoch: Math.floor(Date.now() / 1000) });
const out = runHook('prompt-analyzer.mjs', { session_id: 'p1', prompt: "I'm tired" }, dir);
assert.ok(out.hookSpecificOutput?.additionalContext?.includes('Fatigue language detected'));
});
it('cooldown suppresses non-fatigue warning', () => {
dir = setupTestDir();
createStateFile(dir, 'p1', { ...freshState(), dep_flags: 4, last_warning_epoch: Math.floor(Date.now() / 1000) });
const out = runHook('prompt-analyzer.mjs', { session_id: 'p1', prompt: 'tell me what to do' }, dir);
assert.equal(out.continue, true);
assert.ok(!out.hookSpecificOutput);
});
it('warns at escalation threshold (3 flags)', () => {
dir = setupTestDir();
createStateFile(dir, 'p1', { ...freshState(), esc_flags: 2 });
const out = runHook('prompt-analyzer.mjs', { session_id: 'p1', prompt: 'this is definitely the issue' }, dir);
assert.ok(out.hookSpecificOutput?.additionalContext?.includes('Escalation language detected'));
});
it('warns at validation threshold (3 flags)', () => {
dir = setupTestDir();
createStateFile(dir, 'p1', { ...freshState(), val_flags: 2 });
const out = runHook('prompt-analyzer.mjs', { session_id: 'p1', prompt: 'this is correct, right?' }, dir);
assert.ok(out.hookSpecificOutput?.additionalContext?.includes('Validation-seeking pattern'));
});
});
// --- v1.1.0 pushback + domain regex (regex-only unit tests) ---
// Local copies of patterns in hooks/scripts/prompt-analyzer.mjs.
// Step 3 adds integration tests via runPrompt; integration tests catch
// pattern divergence between source and tests.
const pbReactivePatterns = [
/^are you sure\??/i,
/\bi'?m not convinced\b/i,
/\bthat doesn'?t (?:seem|feel) right\b/i,
/\bthat'?s not (?:quite )?what i meant\b/i,
/\blet me add (?:some )?context\b/i,
/\bactually,? (?:my situation|i)\b/i,
/(?:^|[.!?]\s+)i (?:believe|think) (?:you'?re|that'?s) wrong\b/i,
/\bi don'?t agree(?: with you)?\b/i,
/\bare you absolutely sure\b/i,
];
const pbPreemptivePatterns = [
/\bsteelman\b/i,
/\bplay (?:the )?devil'?s advocate\b/i,
/\bargue against (?:this|my)\b/i,
];
const domainRelationshipPatterns = [
/\b(?:my|our) (?:partner|spouse|wife|husband|girlfriend|boyfriend)\b/i,
/\bin our relationship\b/i,
/\b(?:dating|breakup|divorce)\b/i,
/\bromantic(?:ally)? (?:involved|interested)\b/i,
];
function matchesAny(patterns, text) {
return patterns.some((p) => p.test(text));
}
describe('pushback reactive patterns', () => {
it('matches "are you sure?"', () => assert.ok(matchesAny(pbReactivePatterns, 'are you sure?')));
it('does not match "tell me what to do" (no pushback)', () => assert.equal(matchesAny(pbReactivePatterns, 'tell me what to do'), false));
it("matches \"i'm not convinced\"", () => assert.ok(matchesAny(pbReactivePatterns, "i'm not convinced this works")));
it('does not match "i am convinced" (no negation)', () => assert.equal(matchesAny(pbReactivePatterns, 'i am convinced this works'), false));
it('matches "that doesn\'t seem right"', () => assert.ok(matchesAny(pbReactivePatterns, "that doesn't seem right to me")));
it('does not match "that seems right" (positive sense)', () => assert.equal(matchesAny(pbReactivePatterns, 'that seems right to me'), false));
it('matches "that\'s not what I meant"', () => assert.ok(matchesAny(pbReactivePatterns, "that's not what I meant by that")));
it('does not match "I meant exactly that"', () => assert.equal(matchesAny(pbReactivePatterns, 'I meant exactly that'), false));
it('matches "let me add context"', () => assert.ok(matchesAny(pbReactivePatterns, 'let me add context — the issue is X')));
it('does not match "I added context to the function"', () => assert.equal(matchesAny(pbReactivePatterns, 'I added context to the function'), false));
it('matches "actually, my situation is different"', () => assert.ok(matchesAny(pbReactivePatterns, 'actually, my situation is different')));
it('does not match "actually that approach works"', () => assert.equal(matchesAny(pbReactivePatterns, 'actually that approach works'), false));
it("matches \"I think you're wrong\"", () => assert.ok(matchesAny(pbReactivePatterns, "I think you're wrong about this")));
it("does not match \"I think we're wrong\" (different pronoun)", () => assert.equal(matchesAny(pbReactivePatterns, "I think we're wrong here"), false));
it("matches \"I don't agree\"", () => assert.ok(matchesAny(pbReactivePatterns, "I don't agree with that conclusion")));
it('does not match "I agree with you"', () => assert.equal(matchesAny(pbReactivePatterns, 'I agree with you fully'), false));
it('matches "are you absolutely sure"', () => assert.ok(matchesAny(pbReactivePatterns, 'are you absolutely sure about that')));
it('does not match "we are sure of the answer" (no questioning frame)', () => assert.equal(matchesAny(pbReactivePatterns, 'we are sure of the answer'), false));
});
describe('pushback preemptive patterns', () => {
it('matches "steelman"', () => assert.ok(matchesAny(pbPreemptivePatterns, 'please steelman this argument')));
it('does not match "steel manufacturing" (no whole-word match)', () => assert.equal(matchesAny(pbPreemptivePatterns, 'the steel manufacturing report'), false));
it("matches \"play devil's advocate\"", () => assert.ok(matchesAny(pbPreemptivePatterns, "can you play devil's advocate here")));
it('does not match "play music" (different verb object)', () => assert.equal(matchesAny(pbPreemptivePatterns, 'play music while coding'), false));
it('matches "argue against this"', () => assert.ok(matchesAny(pbPreemptivePatterns, 'argue against this proposal')));
it('does not match "they argue with each other"', () => assert.equal(matchesAny(pbPreemptivePatterns, 'they argue with each other'), false));
});
describe('domain relationship patterns', () => {
it('matches "my partner won\'t listen"', () => assert.ok(matchesAny(domainRelationshipPatterns, "my partner won't listen")));
it('matches "in our relationship"', () => assert.ok(matchesAny(domainRelationshipPatterns, 'in our relationship things changed')));
it('matches "considering divorce"', () => assert.ok(matchesAny(domainRelationshipPatterns, 'considering divorce after years')));
it('matches "romantically involved"', () => assert.ok(matchesAny(domainRelationshipPatterns, 'we are romantically involved')));
it('does not match "function relationship between input and output" (technical false-positive)', () => assert.equal(matchesAny(domainRelationshipPatterns, 'function relationship between input and output'), false));
it('does not match "database relationship mapping" (technical false-positive)', () => assert.equal(matchesAny(domainRelationshipPatterns, 'database relationship mapping'), false));
it('does not match "the data is updating" (no dating word boundary)', () => assert.equal(matchesAny(domainRelationshipPatterns, 'the data is updating in real time'), false));
it('does not match "romantic comedy film" (no involved/interested suffix)', () => assert.equal(matchesAny(domainRelationshipPatterns, 'watching a romantic comedy film'), false));
});
// --- v1.1.0 integration: pushback + valence + domain through prompt-analyzer.mjs ---
describe('pushback integration (state accumulation + same-invocation valence)', () => {
it('counts reactive pushback alone (no fatigue/escalation)', () => {
const s = runPrompt('are you sure?');
assert.equal(s.pushback_count, 1);
assert.equal(s.fatigue_flags, 0);
assert.equal(s.esc_flags, 0);
});
it('counts preemptive pushback alone', () => {
const s = runPrompt('please steelman this argument');
assert.equal(s.pushback_count, 1);
});
it('SUPPRESSES pushback when fatigue marker is in same invocation (valence guard)', () => {
const s = runPrompt("are you sure? I'm exhausted by all this");
assert.equal(s.pushback_count, 0, 'pushback must be suppressed when fatigue is co-present');
assert.equal(s.fatigue_flags, 1);
});
it('sets domain_context to ["relationship"] on positive match (v1.2 array shape)', () => {
const s = runPrompt("my partner won't listen to me");
assert.deepEqual(s.domain_context, ['relationship']);
});
it('keeps domain_context null on technical "function relationship" (false-positive guard)', () => {
const s = runPrompt('function relationship between input and output');
// No domainHit → state.domain_context stays as fresh-state null (untouched).
assert.equal(s.domain_context, null);
});
});
// --- v1.2 pushback alert contract (domain-aware re-contextualization) ---
//
// Step 12 of v1.2.0 ADDS the pushback alert with domain awareness baked in.
// Replaces the v1.1.0 "count but never alert" contract test.
//
// Behavior:
// - HIGH_SYCOPHANCY_DOMAINS (relationship, spirituality): alert at count >= 2
// - INFO_DOMAINS (legal, parenting, health, financial, professional): NO alert
// — pushback in info-seeking domains is healthy self-advocacy.
// - Empty / unknown domain: conservative default alert.
function runPromptCapture(prompt, stateOverrides = {}) {
dir = setupTestDir();
createStateFile(dir, 'p1', { ...freshState(), ...stateOverrides });
const out = runHook('prompt-analyzer.mjs', { session_id: 'p1', prompt }, dir);
const state = readState(dir, 'p1');
return { state, out };
}
describe('pushback alert (v1.2 domain-aware contract)', () => {
it('accumulates pushback_count over 5 sequential prompts', () => {
dir = setupTestDir();
createStateFile(dir, 'p1', { ...freshState(), domain_context: ['relationship'] });
const prompts = [
'are you sure?',
"I'm not convinced",
"that doesn't seem right",
"actually, I think you're wrong",
"are you absolutely sure?",
];
for (const p of prompts) {
runHook('prompt-analyzer.mjs', { session_id: 'p1', prompt: p }, dir);
}
const s = readState(dir, 'p1');
assert.equal(s.pushback_count, 5, 'count accumulates across calls');
});
it('3 pushbacks + relationship → alert (HIGH_SYCOPHANCY)', () => {
const { state, out } = runPromptCapture('are you absolutely sure?', {
domain_context: ['relationship'],
pushback_count: 2, // becomes 3
});
assert.equal(state.pushback_count, 3);
assert.match(out.hookSpecificOutput.additionalContext, /pushback re-contextualization/);
});
it('3 pushbacks + parenting → NO alert (INFO_DOMAIN, healthy self-advocacy)', () => {
const { out } = runPromptCapture("I'm not convinced", {
domain_context: ['parenting'],
pushback_count: 2,
});
// Suppress pushback alert; nothing else should fire here either.
assert.equal(out.hookSpecificOutput, undefined,
'parenting pushback is healthy self-advocacy — no alert');
});
it('3 pushbacks + [relationship, legal] → alert (mixed: any HIGH_SYCOPHANCY wins)', () => {
const { out } = runPromptCapture('are you absolutely sure?', {
domain_context: ['relationship', 'legal'],
pushback_count: 2,
});
assert.match(out.hookSpecificOutput.additionalContext, /pushback re-contextualization/);
});
it('3 pushbacks + empty domain → alert (conservative default)', () => {
const { out } = runPromptCapture('are you absolutely sure?', {
domain_context: [],
pushback_count: 2,
});
assert.match(out.hookSpecificOutput.additionalContext, /pushback/);
});
it('1 pushback + relationship → NO alert (sub-threshold)', () => {
const { out } = runPromptCapture("are you sure?", {
domain_context: ['relationship'],
pushback_count: 0,
});
assert.equal(out.hookSpecificOutput, undefined,
'sub-threshold (count<2) — no alert even in HIGH_SYCOPHANCY');
});
it('5 pushbacks across info-only domains [legal, health] → NO alert', () => {
const { out } = runPromptCapture("I'm not convinced", {
domain_context: ['legal', 'health'],
pushback_count: 4,
});
assert.equal(out.hookSpecificOutput, undefined,
'all-info domains never alert pushback regardless of count');
});
});