refactor(marketplace): split cc-architect from ultraplan-local into its own plugin
Extract `/ultra-cc-architect-local` and `/ultra-skill-author-local` plus all 7 supporting agents, the `cc-architect-catalog` skill (13 files), the `ngram-overlap.mjs` IP-hygiene script, and the skill-factory test fixtures from `ultraplan-local` v2.4.0 into a new `ultra-cc-architect` plugin v0.1.0. Why: ultraplan-local had drifted into containing two distinct domains — a universal planning pipeline (brief → research → plan → execute) and a Claude-Code-specific architecture phase. Keeping them together forced users to inherit an unfinished CC-feature catalog (~11 seeds) when they only wanted the planning pipeline, and locked the catalog and the pipeline into the same release cadence. The architect was already optional and decoupled at the code level — only one filesystem touchpoint remained (auto-discovery of `architecture/overview.md`), which already handles absence gracefully. Plugin manifests: - ultraplan-local: 2.4.0 → 3.0.0 (description + keywords updated) - ultra-cc-architect: new at 0.1.0 (pre-release; catalog is thin, Fase 2/3 of skill-factory unbuilt, decision-layer empty, fallback list still needed) What stays in ultraplan-local: brief/research/plan/execute commands, all 19 planning agents, security hooks, plan auto-discovery of `architecture/overview.md` (filesystem-level contract, not code-level). What moved (28 files via git mv, R100 — full history preserved): - 2 commands, 8 agents, 1 skill catalog (13 files), 2 scripts, 8 fixtures Documentation updates: plugin CLAUDE.md and README.md for both plugins, root README.md (added ultra-cc-architect section, updated ultraplan-local section), root CLAUDE.md (added ultra-cc-architect to repo-struktur), marketplace.json (registered ultra-cc-architect), ultraplan-local CHANGELOG.md (v3.0.0 entry with migration guidance). Test verification: ngram-overlap.test.mjs passes 23/23 from new location. Memory updated: feedback_no_architect_until_v3.md now points at the new plugin and reframes the threshold around catalog maturity rather than an ultraplan-local milestone. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen
parent
97c5c9d934
commit
ab504bdf8c
48 changed files with 627 additions and 177 deletions
|
|
@ -1,251 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
// ngram-overlap.mjs — IP-hygiene check for skill-factory drafts.
|
||||
//
|
||||
// Computes word-n-gram containment similarity between a draft and source,
|
||||
// plus a longest-consecutive-shingle-run signal. Verdict drives whether the
|
||||
// draft passes IP-hygiene (`accepted`), warrants human review
|
||||
// (`needs-review`), or must be rejected as too-close-to-source (`rejected`).
|
||||
//
|
||||
// Algorithm (research brief 01 §Recommendation):
|
||||
// - Word 5-gram containment: |shingles(draft) ∩ shingles(source)| / |shingles(draft)|
|
||||
// - Longest-run secondary signal: max consecutive draft shingles also in source
|
||||
// - Verdict thresholds:
|
||||
// containment <0.15 AND longestRun <8 → accepted
|
||||
// containment ≥0.35 OR longestRun ≥15 → rejected
|
||||
// otherwise → needs-review
|
||||
// - Short-text fallback: shingles n=4 when min(words) <500; verdict
|
||||
// `needs-review` with `reason: too-short-to-score` when min(words) <300.
|
||||
//
|
||||
// Pure Node stdlib. No npm dependencies.
|
||||
//
|
||||
// CLI:
|
||||
// node scripts/ngram-overlap.mjs <draft-path> <source-path>
|
||||
// node scripts/ngram-overlap.mjs --help
|
||||
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { argv, exit, stdout, stderr } from 'node:process';
|
||||
|
||||
// === Constants (research-backed defaults; see research brief 01 §Open Questions) ===
|
||||
export const SHINGLE_N_DEFAULT = 5;
|
||||
export const SHINGLE_N_FALLBACK = 4;
|
||||
export const SHORT_FALLBACK_THRESHOLD = 500;
|
||||
export const TOO_SHORT_THRESHOLD = 300;
|
||||
export const ACCEPT_CONTAINMENT_MAX = 0.15;
|
||||
export const REJECT_CONTAINMENT_MIN = 0.35;
|
||||
export const ACCEPT_RUN_MAX = 8;
|
||||
export const REJECT_RUN_MIN = 15;
|
||||
|
||||
const USAGE = `Usage: node scripts/ngram-overlap.mjs <draft-path> <source-path>
|
||||
|
||||
Computes word-n-gram containment overlap of draft against source.
|
||||
Outputs JSON to stdout. Exit 0 on success, non-zero only on I/O error.
|
||||
|
||||
Verdict bands:
|
||||
accepted containment <${ACCEPT_CONTAINMENT_MAX} AND longestRun <${ACCEPT_RUN_MAX}
|
||||
needs-review between bands, or min words <${TOO_SHORT_THRESHOLD}
|
||||
rejected containment >=${REJECT_CONTAINMENT_MIN} OR longestRun >=${REJECT_RUN_MIN}
|
||||
`;
|
||||
|
||||
// === Markdown stripping (research brief 01 §step 2) ===
|
||||
// Strict order: frontmatter, fenced code, inline code, block quotes, links,
|
||||
// images, emphasis, headings, hr, table pipes.
|
||||
export function stripMarkdown(text) {
|
||||
let t = text;
|
||||
// YAML frontmatter (only at file start)
|
||||
t = t.replace(/^---\n[\s\S]*?\n---\n?/, '');
|
||||
// Fenced code blocks ```lang ... ```
|
||||
t = t.replace(/```[\s\S]*?```/g, ' ');
|
||||
// Inline code `...`
|
||||
t = t.replace(/`[^`\n]*`/g, ' ');
|
||||
// Block quotes (line-leading >)
|
||||
t = t.replace(/^>\s?/gm, '');
|
||||
// Markdown links [text](url) → text
|
||||
t = t.replace(/\[([^\]]*)\]\([^)]*\)/g, '$1');
|
||||
// Images  → alt (handled before link strip if alt present;
|
||||
// but link strip already removed; so handle remaining bang-prefix)
|
||||
t = t.replace(/!\[([^\]]*)\]/g, '$1');
|
||||
// Emphasis: ** _ * ~~ (anchored by char, non-greedy)
|
||||
t = t.replace(/\*\*([^*]+)\*\*/g, '$1');
|
||||
t = t.replace(/__([^_]+)__/g, '$1');
|
||||
t = t.replace(/\*([^*\n]+)\*/g, '$1');
|
||||
t = t.replace(/_([^_\n]+)_/g, '$1');
|
||||
t = t.replace(/~~([^~]+)~~/g, '$1');
|
||||
// Heading markers (line-leading #+)
|
||||
t = t.replace(/^#{1,6}\s+/gm, '');
|
||||
// Horizontal rules
|
||||
t = t.replace(/^[-*_]{3,}\s*$/gm, ' ');
|
||||
// Table pipes
|
||||
t = t.replace(/\|/g, ' ');
|
||||
return t;
|
||||
}
|
||||
|
||||
// === Tokenization ===
|
||||
// NFKC normalize, lowercase, split on Unicode letter/number runs.
|
||||
export function tokenize(text) {
|
||||
const stripped = stripMarkdown(text);
|
||||
const normalized = stripped.normalize('NFKC').toLowerCase();
|
||||
// Match runs of letters/numbers (Unicode-aware via /u flag)
|
||||
const matches = normalized.match(/[\p{L}\p{N}]+/gu);
|
||||
return matches || [];
|
||||
}
|
||||
|
||||
// === Shingles (n-grams of words) ===
|
||||
export function shingles(tokens, n) {
|
||||
if (tokens.length < n) return [];
|
||||
const out = [];
|
||||
for (let i = 0; i <= tokens.length - n; i++) {
|
||||
out.push(tokens.slice(i, i + n).join(' '));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// === Overlap metrics ===
|
||||
//
|
||||
// Returns {containment, longestRun, draftShingleCount, sharedCount}.
|
||||
// Containment = |draft ∩ source| / |draft| (asymmetric: how much of draft
|
||||
// is reused from source). Longest-run = max consecutive draft shingles also
|
||||
// in source.
|
||||
export function overlap(draftTokens, sourceTokens, n) {
|
||||
const draftShingles = shingles(draftTokens, n);
|
||||
const sourceShingles = shingles(sourceTokens, n);
|
||||
if (draftShingles.length === 0) {
|
||||
return { containment: 0, longestRun: 0, draftShingleCount: 0, sharedCount: 0 };
|
||||
}
|
||||
const sourceSet = new Set(sourceShingles);
|
||||
let shared = 0;
|
||||
let longest = 0;
|
||||
let current = 0;
|
||||
for (const sh of draftShingles) {
|
||||
if (sourceSet.has(sh)) {
|
||||
shared += 1;
|
||||
current += 1;
|
||||
if (current > longest) longest = current;
|
||||
} else {
|
||||
current = 0;
|
||||
}
|
||||
}
|
||||
const containment = shared / draftShingles.length;
|
||||
return {
|
||||
containment,
|
||||
longestRun: longest,
|
||||
draftShingleCount: draftShingles.length,
|
||||
sharedCount: shared,
|
||||
};
|
||||
}
|
||||
|
||||
// === Verdict dispatch ===
|
||||
export function verdict(metrics, opts = {}) {
|
||||
const {
|
||||
acceptContainmentMax = ACCEPT_CONTAINMENT_MAX,
|
||||
rejectContainmentMin = REJECT_CONTAINMENT_MIN,
|
||||
acceptRunMax = ACCEPT_RUN_MAX,
|
||||
rejectRunMin = REJECT_RUN_MIN,
|
||||
} = opts;
|
||||
const { containment, longestRun } = metrics;
|
||||
if (containment >= rejectContainmentMin || longestRun >= rejectRunMin) {
|
||||
const reasons = [];
|
||||
if (containment >= rejectContainmentMin) {
|
||||
reasons.push(`containment ${containment.toFixed(3)} >= ${rejectContainmentMin}`);
|
||||
}
|
||||
if (longestRun >= rejectRunMin) {
|
||||
reasons.push(`longestRun ${longestRun} >= ${rejectRunMin}`);
|
||||
}
|
||||
return { verdict: 'rejected', reasons };
|
||||
}
|
||||
if (containment < acceptContainmentMax && longestRun < acceptRunMax) {
|
||||
return { verdict: 'accepted', reasons: [] };
|
||||
}
|
||||
const reasons = [];
|
||||
if (containment >= acceptContainmentMax) {
|
||||
reasons.push(`containment ${containment.toFixed(3)} in [${acceptContainmentMax}, ${rejectContainmentMin})`);
|
||||
}
|
||||
if (longestRun >= acceptRunMax) {
|
||||
reasons.push(`longestRun ${longestRun} in [${acceptRunMax}, ${rejectRunMin})`);
|
||||
}
|
||||
return { verdict: 'needs-review', reasons };
|
||||
}
|
||||
|
||||
// === Top-level analysis ===
|
||||
export function analyze(draftText, sourceText) {
|
||||
const draftTokens = tokenize(draftText);
|
||||
const sourceTokens = tokenize(sourceText);
|
||||
const minWords = Math.min(draftTokens.length, sourceTokens.length);
|
||||
const n = minWords < SHORT_FALLBACK_THRESHOLD ? SHINGLE_N_FALLBACK : SHINGLE_N_DEFAULT;
|
||||
|
||||
if (minWords < TOO_SHORT_THRESHOLD) {
|
||||
return {
|
||||
verdict: 'needs-review',
|
||||
reasons: [`min word count ${minWords} < ${TOO_SHORT_THRESHOLD}`],
|
||||
reason: 'too-short-to-score',
|
||||
containment: 0,
|
||||
longestRun: 0,
|
||||
thresholds: {
|
||||
accept: ACCEPT_CONTAINMENT_MAX,
|
||||
reject: REJECT_CONTAINMENT_MIN,
|
||||
minRun: REJECT_RUN_MIN,
|
||||
},
|
||||
shingleSize: n,
|
||||
draftWords: draftTokens.length,
|
||||
sourceWords: sourceTokens.length,
|
||||
draftShingles: 0,
|
||||
sharedShingles: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const metrics = overlap(draftTokens, sourceTokens, n);
|
||||
const v = verdict(metrics);
|
||||
|
||||
return {
|
||||
verdict: v.verdict,
|
||||
reasons: v.reasons,
|
||||
containment: metrics.containment,
|
||||
longestRun: metrics.longestRun,
|
||||
thresholds: {
|
||||
accept: ACCEPT_CONTAINMENT_MAX,
|
||||
reject: REJECT_CONTAINMENT_MIN,
|
||||
minRun: REJECT_RUN_MIN,
|
||||
},
|
||||
shingleSize: n,
|
||||
draftWords: draftTokens.length,
|
||||
sourceWords: sourceTokens.length,
|
||||
draftShingles: metrics.draftShingleCount,
|
||||
sharedShingles: metrics.sharedCount,
|
||||
};
|
||||
}
|
||||
|
||||
// === CLI entry ===
|
||||
async function main() {
|
||||
const args = argv.slice(2);
|
||||
if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
|
||||
stdout.write(USAGE);
|
||||
exit(0);
|
||||
}
|
||||
if (args.length !== 2) {
|
||||
stderr.write(`Error: expected 2 arguments (draft-path, source-path), got ${args.length}\n`);
|
||||
stderr.write(USAGE);
|
||||
exit(2);
|
||||
}
|
||||
const [draftPath, sourcePath] = args;
|
||||
let draftText, sourceText;
|
||||
try {
|
||||
draftText = await readFile(draftPath, 'utf8');
|
||||
} catch (err) {
|
||||
stderr.write(`Error reading draft ${draftPath}: ${err.message}\n`);
|
||||
exit(1);
|
||||
}
|
||||
try {
|
||||
sourceText = await readFile(sourcePath, 'utf8');
|
||||
} catch (err) {
|
||||
stderr.write(`Error reading source ${sourcePath}: ${err.message}\n`);
|
||||
exit(1);
|
||||
}
|
||||
const result = analyze(draftText, sourceText);
|
||||
stdout.write(JSON.stringify(result, null, 2) + '\n');
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// Only run main when invoked as CLI (not when imported)
|
||||
const invokedAsScript = import.meta.url === `file://${process.argv[1]}`;
|
||||
if (invokedAsScript) {
|
||||
main();
|
||||
}
|
||||
|
|
@ -1,281 +0,0 @@
|
|||
// node:test suite for scripts/ngram-overlap.mjs
|
||||
//
|
||||
// Run: node --test scripts/ngram-overlap.test.mjs
|
||||
//
|
||||
// Covers: identical text, disjoint text, partial overlap bands,
|
||||
// longest-run override, fenced-code stripping, short-source fallback,
|
||||
// markdown-emphasis stripping, fixture integration.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { analyze, tokenize, shingles, overlap, verdict, stripMarkdown } from './ngram-overlap.mjs';
|
||||
|
||||
// === Fixtures (inline prose to control word counts and overlap) ===
|
||||
|
||||
// 600+ word source on a generic technical topic (Claude Code hooks).
|
||||
// Reused across multiple tests with different drafts.
|
||||
const SOURCE_LONG = (() => {
|
||||
const sentences = [
|
||||
'Hooks in Claude Code allow you to intercept events emitted by the agent runtime',
|
||||
'These events fire at specific lifecycle points such as before a tool call runs',
|
||||
'or after the agent completes a turn or when a session starts up for the first time',
|
||||
'A hook is configured by adding an entry to the settings file under the hooks key',
|
||||
'Each hook entry binds a matcher pattern to a shell command that the runtime executes',
|
||||
'The matcher uses simple glob syntax to select which tool calls trigger the hook',
|
||||
'When a tool call matches the pattern the hook runs synchronously before the call proceeds',
|
||||
'A non-zero exit code from a hook script blocks the underlying tool call entirely',
|
||||
'This blocking behavior makes hooks useful for security policy enforcement and audit logging',
|
||||
'For example a pre-bash-executor hook can scan command strings against a denylist',
|
||||
'Hooks receive structured JSON input on standard input describing the event payload',
|
||||
'The schema includes the tool name the parameters and the working directory among other fields',
|
||||
'Hooks can emit JSON output on standard output to add additional context for the model',
|
||||
'Output is appended to the conversation as a system message before the next turn begins',
|
||||
'Plugin hooks live inside the plugin directory and apply only when the plugin is enabled',
|
||||
'User hooks live in the home directory under dot claude and apply across every project',
|
||||
'Project hooks live in the project root and apply only when working in that project',
|
||||
'Conflicts between hook layers resolve in a documented precedence order favoring user settings',
|
||||
'Hooks are written as plain executable scripts in any language that the system can run',
|
||||
'Common languages include shell python and node although any executable will work fine',
|
||||
'Best practice is to keep hooks fast and deterministic so they do not slow down the agent',
|
||||
'Slow hooks add latency to every tool call which compounds across long agent turns',
|
||||
'Hook scripts should also avoid making destructive changes during their execution',
|
||||
'Read-only checks fail safely while write operations from hooks are very hard to debug',
|
||||
'Testing hooks is straightforward by invoking them directly with the same input json',
|
||||
'Capture the output and exit code and verify they match the expected values',
|
||||
'Document hook behavior in the project readme so other contributors understand the constraints',
|
||||
'Hook misconfigurations often manifest as mysterious blocked tool calls during normal use',
|
||||
'Always include a clear error message in stderr when a hook intentionally blocks a call',
|
||||
'This makes debugging easier when the user wonders why their command did not run',
|
||||
'When designing a hook you should think first about what event you actually need to intercept',
|
||||
'Pre-tool-use events fire before any tool runs and can block dangerous operations early',
|
||||
'Post-tool-use events fire after a tool returns and can log results or trigger follow-up actions',
|
||||
'Session-start events fire when the agent begins a new conversation in a fresh context window',
|
||||
'Session-end events fire when the user closes the session and are useful for cleanup tasks',
|
||||
'Stop events fire whenever the agent finishes generating a response and yields back to the user',
|
||||
'Compaction events fire when the conversation history grows too large and must be summarized',
|
||||
'Each event type passes a different payload shape so you must read the schema documentation carefully',
|
||||
'A common pattern is to write a small dispatcher hook that routes events to language-specific handlers',
|
||||
'The dispatcher pattern keeps individual handlers simple and lets you add new ones without rewriting glue code',
|
||||
'Avoid putting business logic directly in the dispatcher because it becomes a bottleneck for testing',
|
||||
'Instead keep the dispatcher pure and delegate all real work to small focused single-purpose handler scripts',
|
||||
'Hook timeouts matter because slow handlers block the agent indefinitely until they return or error out',
|
||||
'Set a strict timeout in your handler implementation rather than relying on the runtime to kill it',
|
||||
'Use exit code two for hard errors and exit code zero for normal pass-through with no policy violation',
|
||||
'Reserve exit code one for soft warnings that should appear in the conversation but not block execution',
|
||||
];
|
||||
return sentences.join('. ') + '.';
|
||||
})();
|
||||
|
||||
const wordCount = (s) => (s.match(/[\p{L}\p{N}]+/gu) || []).length;
|
||||
|
||||
// === Unit tests on pure functions ===
|
||||
|
||||
test('tokenize: lowercases and splits on word boundaries', () => {
|
||||
const tokens = tokenize('Hello, World! Foo-bar.');
|
||||
assert.deepEqual(tokens, ['hello', 'world', 'foo', 'bar']);
|
||||
});
|
||||
|
||||
test('tokenize: NFKC normalizes', () => {
|
||||
// Full-width digits normalize to ASCII
|
||||
const tokens = tokenize('café 123');
|
||||
assert.deepEqual(tokens, ['café', '123']);
|
||||
});
|
||||
|
||||
test('shingles: returns empty when input shorter than n', () => {
|
||||
assert.deepEqual(shingles(['a', 'b', 'c'], 5), []);
|
||||
});
|
||||
|
||||
test('shingles: returns sliding window of size n', () => {
|
||||
const result = shingles(['a', 'b', 'c', 'd', 'e'], 3);
|
||||
assert.deepEqual(result, ['a b c', 'b c d', 'c d e']);
|
||||
});
|
||||
|
||||
test('stripMarkdown: removes fenced code blocks', () => {
|
||||
const input = 'Before\n```js\nconst x = 1;\n```\nAfter';
|
||||
const stripped = stripMarkdown(input);
|
||||
assert.ok(!stripped.includes('const x'));
|
||||
assert.ok(stripped.includes('Before'));
|
||||
assert.ok(stripped.includes('After'));
|
||||
});
|
||||
|
||||
test('stripMarkdown: removes inline code', () => {
|
||||
const stripped = stripMarkdown('Use `npm install` to set up.');
|
||||
assert.ok(!stripped.includes('npm install'));
|
||||
});
|
||||
|
||||
test('stripMarkdown: removes heading markers but keeps text', () => {
|
||||
const stripped = stripMarkdown('# Title\nBody');
|
||||
assert.ok(!stripped.includes('#'));
|
||||
assert.ok(stripped.includes('Title'));
|
||||
});
|
||||
|
||||
test('stripMarkdown: removes emphasis markers', () => {
|
||||
const stripped = stripMarkdown('This **is bold** and *italic* and ~~strike~~');
|
||||
assert.ok(!stripped.includes('**'));
|
||||
assert.ok(!stripped.includes('~~'));
|
||||
assert.ok(stripped.includes('is bold'));
|
||||
assert.ok(stripped.includes('italic'));
|
||||
});
|
||||
|
||||
test('stripMarkdown: links keep text only', () => {
|
||||
const stripped = stripMarkdown('See [docs](https://example.com) for info.');
|
||||
assert.ok(!stripped.includes('https'));
|
||||
assert.ok(stripped.includes('docs'));
|
||||
});
|
||||
|
||||
test('stripMarkdown: removes YAML frontmatter at start', () => {
|
||||
const input = '---\nname: foo\n---\nBody text here';
|
||||
const stripped = stripMarkdown(input);
|
||||
assert.ok(!stripped.includes('name: foo'));
|
||||
assert.ok(stripped.includes('Body text here'));
|
||||
});
|
||||
|
||||
// === Overlap behavior ===
|
||||
|
||||
test('overlap: identical token streams give containment 1.0', () => {
|
||||
const tokens = tokenize(SOURCE_LONG);
|
||||
const m = overlap(tokens, tokens, 5);
|
||||
assert.equal(m.containment, 1);
|
||||
assert.ok(m.longestRun > 15);
|
||||
});
|
||||
|
||||
test('overlap: completely disjoint streams give containment 0', () => {
|
||||
const a = ['alpha', 'beta', 'gamma', 'delta', 'epsilon', 'zeta', 'eta', 'theta', 'iota', 'kappa'];
|
||||
const b = ['xray', 'yankee', 'zulu', 'whiskey', 'victor', 'uniform', 'tango', 'sierra', 'romeo', 'quebec'];
|
||||
const m = overlap(a, b, 5);
|
||||
assert.equal(m.containment, 0);
|
||||
assert.equal(m.longestRun, 0);
|
||||
});
|
||||
|
||||
// === Verdict bands ===
|
||||
|
||||
test('verdict 1: identical text → rejected (containment 1.0)', () => {
|
||||
const result = analyze(SOURCE_LONG, SOURCE_LONG);
|
||||
assert.equal(result.verdict, 'rejected');
|
||||
assert.equal(result.containment, 1);
|
||||
});
|
||||
|
||||
test('verdict 2: completely disjoint text → accepted (low containment, low run)', () => {
|
||||
// Build a draft of unrelated words ≥300 to skip too-short fallback
|
||||
const draftWords = [];
|
||||
for (let i = 0; i < 350; i++) {
|
||||
draftWords.push(`uniqueword${i}`);
|
||||
}
|
||||
const draft = draftWords.join(' ');
|
||||
const result = analyze(draft, SOURCE_LONG);
|
||||
assert.equal(result.verdict, 'accepted');
|
||||
assert.equal(result.containment, 0);
|
||||
assert.equal(result.longestRun, 0);
|
||||
});
|
||||
|
||||
test('verdict 3: partial overlap (mid-band) → needs-review', () => {
|
||||
// Construct draft where ~25% of 5-grams match source but no run is long.
|
||||
// Strategy: alternate 6-token source chunks with 2-token padding. Each
|
||||
// chunk yields exactly 2 source 5-grams (longestRun = 2). Need both
|
||||
// draft and source ≥500 tokens to keep shingleSize=5 (no fallback).
|
||||
// 65 chunks × 8 = 520 draft tokens; SOURCE_LONG is ~600 tokens.
|
||||
const sourceTokens = tokenize(SOURCE_LONG);
|
||||
const draftWords = [];
|
||||
let pad = 0;
|
||||
for (let i = 0; i < 65; i++) {
|
||||
draftWords.push(...sourceTokens.slice(i * 6, i * 6 + 6));
|
||||
draftWords.push(`padword${pad++}`, `padword${pad++}`);
|
||||
}
|
||||
const draft = draftWords.join(' ');
|
||||
const result = analyze(draft, SOURCE_LONG);
|
||||
assert.equal(result.shingleSize, 5,
|
||||
`precondition: expected shingleSize=5 (no fallback), got ${result.shingleSize}`);
|
||||
assert.equal(result.verdict, 'needs-review',
|
||||
`expected needs-review, got ${result.verdict} (containment=${result.containment.toFixed(3)}, longestRun=${result.longestRun})`);
|
||||
});
|
||||
|
||||
test('verdict 4: high overlap → rejected (containment ≥0.35)', () => {
|
||||
// Draft is 60% source + 40% padding
|
||||
const sourceTokens = tokenize(SOURCE_LONG);
|
||||
const sourcePart = sourceTokens.slice(0, 200);
|
||||
const padding = [];
|
||||
for (let i = 0; i < 130; i++) padding.push(`pad${i}`);
|
||||
const draft = sourcePart.concat(padding).join(' ');
|
||||
const result = analyze(draft, SOURCE_LONG);
|
||||
assert.equal(result.verdict, 'rejected',
|
||||
`expected rejected, got ${result.verdict} (containment=${result.containment.toFixed(3)}, longestRun=${result.longestRun})`);
|
||||
});
|
||||
|
||||
test('verdict 5: long verbatim run triggers rejection even with low containment', () => {
|
||||
// Mostly unique words (low containment) but one 25-word verbatim sentence
|
||||
// from source — longestRun ≥15 should reject.
|
||||
const verbatim = tokenize(SOURCE_LONG).slice(50, 75).join(' ');
|
||||
const padding = [];
|
||||
for (let i = 0; i < 500; i++) padding.push(`unique${i}`);
|
||||
const draft = padding.slice(0, 250).join(' ') + ' ' + verbatim + ' ' + padding.slice(250).join(' ');
|
||||
const result = analyze(draft, SOURCE_LONG);
|
||||
assert.equal(result.verdict, 'rejected',
|
||||
`expected rejected, got ${result.verdict} (containment=${result.containment.toFixed(3)}, longestRun=${result.longestRun})`);
|
||||
assert.ok(result.longestRun >= 15, `longestRun ${result.longestRun} should be ≥15`);
|
||||
});
|
||||
|
||||
test('verdict 6: fenced code block in source → stripped → not counted as match', () => {
|
||||
const draftBody = [];
|
||||
for (let i = 0; i < 350; i++) draftBody.push(`uniq${i}`);
|
||||
const draft = draftBody.join(' ');
|
||||
// Source with a fenced code block containing some of the draft's words
|
||||
const sourceWithCode = SOURCE_LONG + '\n```\n' + draftBody.slice(0, 100).join(' ') + '\n```\n';
|
||||
const result = analyze(draft, sourceWithCode);
|
||||
// The code-block words should be stripped from source, so the draft remains disjoint
|
||||
assert.equal(result.containment, 0,
|
||||
`code-block words should be stripped (got containment ${result.containment})`);
|
||||
});
|
||||
|
||||
test('verdict 7: short draft (<300 words) → needs-review with too-short reason', () => {
|
||||
const draft = 'This is a short note. It has fewer than three hundred words. Just a quick sketch.';
|
||||
const result = analyze(draft, SOURCE_LONG);
|
||||
assert.equal(result.verdict, 'needs-review');
|
||||
assert.equal(result.reason, 'too-short-to-score');
|
||||
});
|
||||
|
||||
test('verdict 8: markdown emphasis is stripped before tokenization', () => {
|
||||
// Build a draft of unique tokens then wrap parts in **bold** and *italic*
|
||||
const baseWords = [];
|
||||
for (let i = 0; i < 350; i++) baseWords.push(`tok${i}`);
|
||||
const plain = baseWords.join(' ');
|
||||
const wrapped = baseWords
|
||||
.map((w, i) => (i % 5 === 0 ? `**${w}**` : i % 7 === 0 ? `*${w}*` : w))
|
||||
.join(' ');
|
||||
const plainResult = analyze(plain, SOURCE_LONG);
|
||||
const wrappedResult = analyze(wrapped, SOURCE_LONG);
|
||||
// After stripping, both should yield the same containment / longestRun
|
||||
assert.equal(plainResult.containment, wrappedResult.containment,
|
||||
'markdown emphasis should not change containment after stripping');
|
||||
assert.equal(plainResult.longestRun, wrappedResult.longestRun,
|
||||
'markdown emphasis should not change longestRun after stripping');
|
||||
});
|
||||
|
||||
// === Integration: fixtures (Step 5 will create these; skip if missing) ===
|
||||
|
||||
const FIXTURE_DIR = 'tests/fixtures/skill-factory';
|
||||
const SCRIPT = 'scripts/ngram-overlap.mjs';
|
||||
|
||||
function runCli(draft, source) {
|
||||
const out = execFileSync('node', [SCRIPT, draft, source], { encoding: 'utf8' });
|
||||
return JSON.parse(out);
|
||||
}
|
||||
|
||||
test('integration: accepted fixture pair → verdict accepted', { skip: !existsSync(`${FIXTURE_DIR}/draft-accepted.md`) }, () => {
|
||||
const result = runCli(`${FIXTURE_DIR}/draft-accepted.md`, `${FIXTURE_DIR}/source-accepted.md`);
|
||||
assert.equal(result.verdict, 'accepted',
|
||||
`expected accepted, got ${result.verdict} (containment=${result.containment.toFixed(3)}, longestRun=${result.longestRun})`);
|
||||
});
|
||||
|
||||
test('integration: needs-review fixture pair → verdict needs-review', { skip: !existsSync(`${FIXTURE_DIR}/draft-needs-review.md`) }, () => {
|
||||
const result = runCli(`${FIXTURE_DIR}/draft-needs-review.md`, `${FIXTURE_DIR}/source-needs-review.md`);
|
||||
assert.equal(result.verdict, 'needs-review',
|
||||
`expected needs-review, got ${result.verdict} (containment=${result.containment.toFixed(3)}, longestRun=${result.longestRun})`);
|
||||
});
|
||||
|
||||
test('integration: rejected fixture pair → verdict rejected', { skip: !existsSync(`${FIXTURE_DIR}/draft-rejected.md`) }, () => {
|
||||
const result = runCli(`${FIXTURE_DIR}/draft-rejected.md`, `${FIXTURE_DIR}/source-rejected.md`);
|
||||
assert.equal(result.verdict, 'rejected',
|
||||
`expected rejected, got ${result.verdict} (containment=${result.containment.toFixed(3)}, longestRun=${result.longestRun})`);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue