From 65c9242160287b428404104378388bff67ca98ba Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 1 May 2026 05:39:47 +0200 Subject: [PATCH] =?UTF-8?q?feat(ultraplan-local):=20Spor=201=20wave=202=20?= =?UTF-8?q?=E2=80=94=205=20validators=20+=20doc-consistency,=20108=20tests?= =?UTF-8?q?=20gr=C3=B8nn=20[skip-docs]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 5 nye validator-moduler (alle m/ CLI-shim for invokering fra commands): - brief-validator.mjs — frontmatter (type, brief_version, task, slug, research_topics, research_status), state machine (research_topics > 0 + skipped requires brief_quality: partial), body sections (Intent/Goal/Success Criteria) - research-validator.mjs — type=ultraresearch-brief, confidence ∈ [0,1], dimensions ≥ 1, body sections, --dir mode for batch validering - plan-validator.mjs — wrapper over plan-schema + manifest-yaml; håndhever step-count == manifest-count, plan_version=1.7 - progress-validator.mjs — schema_version, status enum, current_step in range, step shape, checkResumeReadiness - architecture-discovery.mjs — EKSTERN KONTRAKT: drift-WARN ikke drift-FAIL; tolererer non-canonical filnavn, surfacer loose files som warnings Doc-consistency-test pinning prose vs source-of-truth: - agents/*.md count == CLAUDE.md agent-tabell rader - commands/*.md mentioned i CLAUDE.md - command frontmatter.name == filnavn - templates/plan-template.md plan_version 1.7 invariant - settings.json kun kjente scopes (ultraplan, ultraresearch) - settings.json ingen exploration eller agentTeam (vestigial guard etter Spor 0) - CLAUDE.md refererer alle 4 pipeline-commands Wave 1 + Wave 2 = 108 tester grønn. [skip-docs]: Test-infrastrukturen er ikke user-facing før Spor 1 wiring lander; README/CLAUDE.md oppdateres når commands faktisk endrer atferd (neste commit). Co-Authored-By: Claude Opus 4.7 --- .../lib/validators/architecture-discovery.mjs | 94 +++++++++++++++ .../lib/validators/brief-validator.mjs | 99 ++++++++++++++++ .../lib/validators/plan-validator.mjs | 76 ++++++++++++ .../lib/validators/progress-validator.mjs | 106 +++++++++++++++++ .../lib/validators/research-validator.mjs | 109 ++++++++++++++++++ .../tests/lib/doc-consistency.test.mjs | 102 ++++++++++++++++ .../architecture-discovery.test.mjs | 81 +++++++++++++ .../tests/validators/brief-validator.test.mjs | 94 +++++++++++++++ .../tests/validators/plan-validator.test.mjs | 99 ++++++++++++++++ .../validators/progress-validator.test.mjs | 79 +++++++++++++ .../validators/research-validator.test.mjs | 60 ++++++++++ 11 files changed, 999 insertions(+) create mode 100644 plugins/ultraplan-local/lib/validators/architecture-discovery.mjs create mode 100644 plugins/ultraplan-local/lib/validators/brief-validator.mjs create mode 100644 plugins/ultraplan-local/lib/validators/plan-validator.mjs create mode 100644 plugins/ultraplan-local/lib/validators/progress-validator.mjs create mode 100644 plugins/ultraplan-local/lib/validators/research-validator.mjs create mode 100644 plugins/ultraplan-local/tests/lib/doc-consistency.test.mjs create mode 100644 plugins/ultraplan-local/tests/validators/architecture-discovery.test.mjs create mode 100644 plugins/ultraplan-local/tests/validators/brief-validator.test.mjs create mode 100644 plugins/ultraplan-local/tests/validators/plan-validator.test.mjs create mode 100644 plugins/ultraplan-local/tests/validators/progress-validator.test.mjs create mode 100644 plugins/ultraplan-local/tests/validators/research-validator.test.mjs diff --git a/plugins/ultraplan-local/lib/validators/architecture-discovery.mjs b/plugins/ultraplan-local/lib/validators/architecture-discovery.mjs new file mode 100644 index 0000000..2a26186 --- /dev/null +++ b/plugins/ultraplan-local/lib/validators/architecture-discovery.mjs @@ -0,0 +1,94 @@ +// lib/validators/architecture-discovery.mjs +// EXTERNAL CONTRACT — drift-WARN, never drift-FAIL. +// +// The architecture/ directory is owned by the separate `ultra-cc-architect` +// plugin. ultraplan-local validates only DISCOVERY (file present at canonical +// path) and tolerates internal-format drift via warnings. +// +// Never read body content beyond first heading. Never assert frontmatter shape. + +import { existsSync, readdirSync, statSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { issue } from '../util/result.mjs'; + +const CANONICAL_OVERVIEW = 'overview.md'; +const CANONICAL_GAPS = 'gaps.md'; +const KNOWN_ALTERNATIVES = ['architecture-overview.md', 'overview.markdown', 'README.md']; + +export function discoverArchitecture(projectDir) { + const archDir = projectDir ? join(projectDir, 'architecture') : null; + const result = { + found: false, + overview: null, + gaps: null, + looseFiles: [], + warnings: [], + }; + + if (!archDir || !existsSync(archDir) || !statSync(archDir).isDirectory()) { + return result; + } + + const overviewPath = join(archDir, CANONICAL_OVERVIEW); + if (existsSync(overviewPath) && statSync(overviewPath).isFile()) { + result.found = true; + result.overview = overviewPath; + } else { + for (const alt of KNOWN_ALTERNATIVES) { + const altPath = join(archDir, alt); + if (existsSync(altPath) && statSync(altPath).isFile()) { + result.found = true; + result.overview = altPath; + result.warnings.push(issue( + 'ARCH_NON_CANONICAL_OVERVIEW', + `Architecture file at non-canonical path: ${alt}`, + `Canonical contract is architecture/overview.md. The ultra-cc-architect plugin may have drifted; this is a warning, not a blocker.`, + )); + break; + } + } + } + + const gapsPath = join(archDir, CANONICAL_GAPS); + if (existsSync(gapsPath) && statSync(gapsPath).isFile()) result.gaps = gapsPath; + + const all = readdirSync(archDir).filter(f => /\.md$/i.test(f)); + result.looseFiles = all + .filter(f => f !== CANONICAL_OVERVIEW && f !== CANONICAL_GAPS && !KNOWN_ALTERNATIVES.includes(f)) + .map(f => join(archDir, f)); + + if (result.looseFiles.length > 0) { + result.warnings.push(issue( + 'ARCH_LOOSE_FILES', + `Found ${result.looseFiles.length} unrecognized architecture file(s)`, + `Architecture contract expects overview.md (+ optional gaps.md). Loose files may indicate format drift in ultra-cc-architect.`, + )); + } + + if (result.found && result.overview) { + try { + const text = readFileSync(result.overview, 'utf-8'); + const firstHeading = text.match(/^#\s+(.+?)\s*$/m); + result.firstHeading = firstHeading ? firstHeading[1] : null; + } catch { /* ignore — only sniff */ } + } + + return result; +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const projectDir = process.argv[2]; + const wantJson = process.argv.includes('--json'); + if (!projectDir) { + process.stderr.write('Usage: architecture-discovery.mjs [--json]\n'); + process.exit(2); + } + const r = discoverArchitecture(projectDir); + if (wantJson) { + process.stdout.write(JSON.stringify(r, null, 2) + '\n'); + } else { + process.stdout.write(`architecture-discovery: ${r.found ? 'FOUND' : 'NONE'} ${r.overview || projectDir}\n`); + for (const w of r.warnings) process.stderr.write(` WARN [${w.code}] ${w.message}\n`); + } + process.exit(0); +} diff --git a/plugins/ultraplan-local/lib/validators/brief-validator.mjs b/plugins/ultraplan-local/lib/validators/brief-validator.mjs new file mode 100644 index 0000000..0fb3199 --- /dev/null +++ b/plugins/ultraplan-local/lib/validators/brief-validator.mjs @@ -0,0 +1,99 @@ +// lib/validators/brief-validator.mjs +// Validate ultrabrief frontmatter + body invariants. + +import { readFileSync, existsSync } from 'node:fs'; +import { parseDocument } from '../util/frontmatter.mjs'; +import { issue, ok, fail } from '../util/result.mjs'; + +export const BRIEF_REQUIRED_FRONTMATTER = ['type', 'brief_version', 'task', 'slug', 'research_topics', 'research_status']; +export const BRIEF_RESEARCH_STATUS_VALUES = ['pending', 'in_progress', 'complete', 'skipped']; +export const BRIEF_BODY_SECTIONS = ['Intent', 'Goal', 'Success Criteria']; + +export function validateBriefContent(text, opts = {}) { + const strict = opts.strict !== false; + const doc = parseDocument(text); + if (!doc.valid) return doc; + + const fm = doc.parsed.frontmatter || {}; + const body = doc.parsed.body || ''; + const errors = []; + const warnings = []; + + for (const k of BRIEF_REQUIRED_FRONTMATTER) { + if (!(k in fm)) { + errors.push(issue('BRIEF_MISSING_FIELD', `Required frontmatter field missing: ${k}`)); + } + } + + if (fm.type !== undefined && fm.type !== 'ultrabrief') { + errors.push(issue('BRIEF_WRONG_TYPE', `frontmatter.type must be "ultrabrief", got "${fm.type}"`)); + } + + if (fm.research_status !== undefined && !BRIEF_RESEARCH_STATUS_VALUES.includes(fm.research_status)) { + errors.push(issue( + 'BRIEF_BAD_STATUS', + `research_status "${fm.research_status}" not in [${BRIEF_RESEARCH_STATUS_VALUES.join(', ')}]`, + )); + } + + if (typeof fm.research_topics === 'number' && fm.research_topics > 0 && fm.research_status === 'skipped') { + if (fm.brief_quality !== 'partial') { + errors.push(issue( + 'BRIEF_STATE_INCOHERENT', + `research_topics=${fm.research_topics} but research_status=skipped`, + 'Either set research_status to a real progress value, or mark brief_quality: partial.', + )); + } else { + warnings.push(issue( + 'BRIEF_PARTIAL_SKIPPED', + `Brief has unresolved research topics (${fm.research_topics}) but is partial`, + )); + } + } + + for (const section of BRIEF_BODY_SECTIONS) { + const re = new RegExp(`^##\\s+${section}\\b`, 'm'); + if (!re.test(body)) { + const issueObj = issue('BRIEF_MISSING_SECTION', `Required body section missing: ## ${section}`); + if (strict) errors.push(issueObj); + else warnings.push(issueObj); + } + } + + if (typeof fm.brief_version === 'string') { + const m = fm.brief_version.match(/^(\d+)\.(\d+)$/); + if (!m) { + warnings.push(issue('BRIEF_VERSION_FORMAT', `brief_version "${fm.brief_version}" not in N.M form`)); + } + } + + return { valid: errors.length === 0, errors, warnings, parsed: { frontmatter: fm, body } }; +} + +export function validateBrief(filePath, opts = {}) { + if (!existsSync(filePath)) return fail(issue('BRIEF_NOT_FOUND', `File not found: ${filePath}`)); + let text; + try { text = readFileSync(filePath, 'utf-8'); } + catch (e) { return fail(issue('BRIEF_READ_ERROR', `Cannot read ${filePath}: ${e.message}`)); } + const r = validateBriefContent(text, opts); + return { ...r, parsed: { ...r.parsed, filePath } }; +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const args = process.argv.slice(2); + const strict = !args.includes('--soft'); + const filePath = args.find(a => !a.startsWith('--')); + if (!filePath) { + process.stderr.write('Usage: brief-validator.mjs [--soft] \n'); + process.exit(2); + } + const r = validateBrief(filePath, { strict }); + if (args.includes('--json')) { + process.stdout.write(JSON.stringify({ valid: r.valid, errors: r.errors, warnings: r.warnings }, null, 2) + '\n'); + } else { + process.stdout.write(`brief-validator: ${r.valid ? 'PASS' : 'FAIL'} ${filePath}\n`); + for (const e of r.errors) process.stderr.write(` ERROR [${e.code}] ${e.message}\n`); + for (const w of r.warnings) process.stderr.write(` WARN [${w.code}] ${w.message}\n`); + } + process.exit(r.valid ? 0 : 1); +} diff --git a/plugins/ultraplan-local/lib/validators/plan-validator.mjs b/plugins/ultraplan-local/lib/validators/plan-validator.mjs new file mode 100644 index 0000000..819ee43 --- /dev/null +++ b/plugins/ultraplan-local/lib/validators/plan-validator.mjs @@ -0,0 +1,76 @@ +// lib/validators/plan-validator.mjs +// Wraps plan-schema (heading shape) + manifest-yaml (per-step Manifest blocks). +// This is the JS equivalent of Phase 5.5 grep checks in planning-orchestrator. + +import { readFileSync, existsSync } from 'node:fs'; +import { sliceSteps, validatePlanHeadings, extractPlanVersion } from '../parsers/plan-schema.mjs'; +import { validateAllManifests } from '../parsers/manifest-yaml.mjs'; +import { issue, fail } from '../util/result.mjs'; + +export function validatePlanContent(text, opts = {}) { + const strict = opts.strict !== false; + const headRes = validatePlanHeadings(text, { strict }); + const errors = [...headRes.errors]; + const warnings = [...headRes.warnings]; + + const steps = headRes.parsed?.steps || []; + const sections = sliceSteps(text); + const manRes = validateAllManifests(sections); + errors.push(...manRes.errors); + warnings.push(...manRes.warnings); + + if (steps.length > 0 && manRes.parsed.length !== steps.length) { + errors.push(issue( + 'PLAN_MANIFEST_COUNT_MISMATCH', + `Step count (${steps.length}) does not equal manifest count (${manRes.parsed.length})`, + )); + } + + const planVersion = extractPlanVersion(text); + if (planVersion === null) { + warnings.push(issue('PLAN_NO_VERSION', 'No plan_version detected; current target is 1.7')); + } else if (planVersion !== '1.7') { + warnings.push(issue('PLAN_VERSION_MISMATCH', `plan_version=${planVersion}, current target is 1.7`)); + } + + return { + valid: errors.length === 0, + errors, + warnings, + parsed: { steps, manifests: manRes.parsed, planVersion }, + }; +} + +export function validatePlan(filePath, opts = {}) { + if (!existsSync(filePath)) return fail(issue('PLAN_NOT_FOUND', `File not found: ${filePath}`)); + let text; + try { text = readFileSync(filePath, 'utf-8'); } + catch (e) { return fail(issue('PLAN_READ_ERROR', `Cannot read ${filePath}: ${e.message}`)); } + const r = validatePlanContent(text, opts); + return { ...r, parsed: { ...r.parsed, filePath } }; +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const args = process.argv.slice(2); + const strict = !args.includes('--soft'); + const filePath = args.find(a => !a.startsWith('--')); + if (!filePath) { + process.stderr.write('Usage: plan-validator.mjs [--strict|--soft] \n'); + process.exit(2); + } + const r = validatePlan(filePath, { strict }); + if (args.includes('--json')) { + process.stdout.write(JSON.stringify({ + valid: r.valid, + errors: r.errors, + warnings: r.warnings, + steps: r.parsed?.steps?.length ?? 0, + planVersion: r.parsed?.planVersion ?? null, + }, null, 2) + '\n'); + } else { + process.stdout.write(`plan-validator: ${r.valid ? 'READY' : 'FAIL'} ${filePath} (${r.parsed?.steps?.length ?? 0} steps)\n`); + for (const e of r.errors) process.stderr.write(` ERROR [${e.code}] ${e.message}\n`); + for (const w of r.warnings) process.stderr.write(` WARN [${w.code}] ${w.message}\n`); + } + process.exit(r.valid ? 0 : 1); +} diff --git a/plugins/ultraplan-local/lib/validators/progress-validator.mjs b/plugins/ultraplan-local/lib/validators/progress-validator.mjs new file mode 100644 index 0000000..58175a4 --- /dev/null +++ b/plugins/ultraplan-local/lib/validators/progress-validator.mjs @@ -0,0 +1,106 @@ +// lib/validators/progress-validator.mjs +// Validate progress.json shape + resume-readiness. + +import { readFileSync, existsSync } from 'node:fs'; +import { issue, fail } from '../util/result.mjs'; + +export const PROGRESS_REQUIRED_TOP = ['schema_version', 'plan', 'plan_version', 'mode', 'status', 'total_steps', 'current_step', 'steps']; +export const PROGRESS_VALID_STATUSES = ['pending', 'in_progress', 'completed', 'failed', 'partial']; + +export function validateProgressContent(jsonText, opts = {}) { + let parsed; + try { parsed = JSON.parse(jsonText); } + catch (e) { + return fail(issue('PROGRESS_PARSE_ERROR', `Cannot parse JSON: ${e.message}`)); + } + + return validateProgressObject(parsed, opts); +} + +export function validateProgressObject(parsed, opts = {}) { + const errors = []; + const warnings = []; + + if (typeof parsed !== 'object' || parsed === null) { + return fail(issue('PROGRESS_NOT_OBJECT', 'Progress payload is not an object')); + } + + for (const k of PROGRESS_REQUIRED_TOP) { + if (!(k in parsed)) { + errors.push(issue('PROGRESS_MISSING_FIELD', `Required field missing: ${k}`)); + } + } + + if (parsed.schema_version !== undefined && parsed.schema_version !== '1') { + errors.push(issue('PROGRESS_SCHEMA_MISMATCH', `schema_version "${parsed.schema_version}" not supported (expected "1")`)); + } + + if (parsed.status !== undefined && !PROGRESS_VALID_STATUSES.includes(parsed.status)) { + errors.push(issue('PROGRESS_BAD_STATUS', `status "${parsed.status}" not in [${PROGRESS_VALID_STATUSES.join(', ')}]`)); + } + + if (typeof parsed.total_steps === 'number' && typeof parsed.current_step === 'number') { + if (parsed.current_step < 0 || parsed.current_step > parsed.total_steps) { + errors.push(issue('PROGRESS_STEP_RANGE', `current_step=${parsed.current_step} outside [0, ${parsed.total_steps}]`)); + } + } + + if (parsed.steps && typeof parsed.steps === 'object') { + const stepKeys = Object.keys(parsed.steps); + if (typeof parsed.total_steps === 'number' && stepKeys.length !== parsed.total_steps) { + warnings.push(issue( + 'PROGRESS_STEP_COUNT_MISMATCH', + `total_steps=${parsed.total_steps} but steps map has ${stepKeys.length} entries`, + )); + } + for (const k of stepKeys) { + const s = parsed.steps[k]; + if (s === null || typeof s !== 'object') { + errors.push(issue('PROGRESS_STEP_SHAPE', `steps["${k}"] is not an object`)); + continue; + } + if (s.status !== undefined && !['completed', 'in_progress', 'failed', 'pending', 'deferred', 'skipped'].includes(s.status)) { + warnings.push(issue('PROGRESS_STEP_BAD_STATUS', `steps["${k}"].status "${s.status}" unrecognized`)); + } + } + } + + return { valid: errors.length === 0, errors, warnings, parsed }; +} + +export function checkResumeReadiness(progressObj) { + const errors = []; + if (progressObj.status === 'completed') { + return { valid: false, errors: [issue('PROGRESS_ALREADY_DONE', 'Run is already completed; nothing to resume')], warnings: [], parsed: progressObj }; + } + if (typeof progressObj.current_step !== 'number') { + errors.push(issue('PROGRESS_NO_CURRENT', 'No current_step in progress.json')); + } + return { valid: errors.length === 0, errors, warnings: [], parsed: progressObj }; +} + +export function validateProgress(filePath, opts = {}) { + if (!existsSync(filePath)) return fail(issue('PROGRESS_NOT_FOUND', `File not found: ${filePath}`)); + let text; + try { text = readFileSync(filePath, 'utf-8'); } + catch (e) { return fail(issue('PROGRESS_READ_ERROR', `Cannot read ${filePath}: ${e.message}`)); } + return validateProgressContent(text, opts); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const args = process.argv.slice(2); + const filePath = args.find(a => !a.startsWith('--')); + if (!filePath) { + process.stderr.write('Usage: progress-validator.mjs [--quick] \n'); + process.exit(2); + } + const r = validateProgress(filePath); + if (args.includes('--json')) { + process.stdout.write(JSON.stringify({ valid: r.valid, errors: r.errors, warnings: r.warnings }, null, 2) + '\n'); + } else { + process.stdout.write(`progress-validator: ${r.valid ? 'PASS' : 'FAIL'} ${filePath}\n`); + for (const e of r.errors) process.stderr.write(` ERROR [${e.code}] ${e.message}\n`); + for (const w of r.warnings) process.stderr.write(` WARN [${w.code}] ${w.message}\n`); + } + process.exit(r.valid ? 0 : 1); +} diff --git a/plugins/ultraplan-local/lib/validators/research-validator.mjs b/plugins/ultraplan-local/lib/validators/research-validator.mjs new file mode 100644 index 0000000..1d6da95 --- /dev/null +++ b/plugins/ultraplan-local/lib/validators/research-validator.mjs @@ -0,0 +1,109 @@ +// lib/validators/research-validator.mjs +// Validate research-brief frontmatter + body invariants. + +import { readFileSync, existsSync, readdirSync, statSync } from 'node:fs'; +import { join } from 'node:path'; +import { parseDocument } from '../util/frontmatter.mjs'; +import { issue, fail } from '../util/result.mjs'; + +export const RESEARCH_REQUIRED_FRONTMATTER = ['type', 'created', 'question']; +export const RESEARCH_BODY_SECTIONS = ['Executive Summary', 'Dimensions']; + +export function validateResearchContent(text, opts = {}) { + const strict = opts.strict !== false; + const doc = parseDocument(text); + if (!doc.valid) return doc; + + const fm = doc.parsed.frontmatter || {}; + const body = doc.parsed.body || ''; + const errors = []; + const warnings = []; + + for (const k of RESEARCH_REQUIRED_FRONTMATTER) { + if (!(k in fm)) errors.push(issue('RESEARCH_MISSING_FIELD', `Required frontmatter field missing: ${k}`)); + } + + if (fm.type !== undefined && fm.type !== 'ultraresearch-brief') { + errors.push(issue('RESEARCH_WRONG_TYPE', `frontmatter.type must be "ultraresearch-brief", got "${fm.type}"`)); + } + + if (fm.confidence !== undefined) { + if (typeof fm.confidence !== 'number' || fm.confidence < 0 || fm.confidence > 1) { + errors.push(issue('RESEARCH_BAD_CONFIDENCE', `confidence must be number in [0,1], got ${fm.confidence}`)); + } + } else { + warnings.push(issue('RESEARCH_NO_CONFIDENCE', 'No confidence field — planner has no signal to weight findings')); + } + + if (fm.dimensions !== undefined && (typeof fm.dimensions !== 'number' || fm.dimensions < 1)) { + errors.push(issue('RESEARCH_BAD_DIMENSIONS', `dimensions must be positive integer, got ${fm.dimensions}`)); + } + + for (const section of RESEARCH_BODY_SECTIONS) { + const re = new RegExp(`^##\\s+${section}\\b`, 'm'); + if (!re.test(body)) { + const issueObj = issue('RESEARCH_MISSING_SECTION', `Required body section missing: ## ${section}`); + if (strict) errors.push(issueObj); + else warnings.push(issueObj); + } + } + + return { valid: errors.length === 0, errors, warnings, parsed: { frontmatter: fm, body } }; +} + +export function validateResearch(filePath, opts = {}) { + if (!existsSync(filePath)) return fail(issue('RESEARCH_NOT_FOUND', `File not found: ${filePath}`)); + let text; + try { text = readFileSync(filePath, 'utf-8'); } + catch (e) { return fail(issue('RESEARCH_READ_ERROR', `Cannot read ${filePath}: ${e.message}`)); } + const r = validateResearchContent(text, opts); + return { ...r, parsed: { ...r.parsed, filePath } }; +} + +export function validateResearchDir(dirPath, opts = {}) { + if (!existsSync(dirPath) || !statSync(dirPath).isDirectory()) { + return { valid: true, errors: [], warnings: [], parsed: { files: [] } }; + } + const files = readdirSync(dirPath).filter(f => f.endsWith('.md')).sort(); + const errors = []; + const warnings = []; + const results = []; + for (const f of files) { + const r = validateResearch(join(dirPath, f), opts); + for (const e of r.errors) errors.push(issue(e.code, `${f}: ${e.message}`, e.hint)); + for (const w of r.warnings) warnings.push(issue(w.code, `${f}: ${w.message}`, w.hint)); + results.push({ file: f, valid: r.valid }); + } + return { valid: errors.length === 0, errors, warnings, parsed: { files: results } }; +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const args = process.argv.slice(2); + const strict = !args.includes('--soft'); + const dirIdx = args.indexOf('--dir'); + if (dirIdx >= 0 && args[dirIdx + 1]) { + const r = validateResearchDir(args[dirIdx + 1], { strict }); + if (args.includes('--json')) { + process.stdout.write(JSON.stringify({ valid: r.valid, errors: r.errors, warnings: r.warnings, files: r.parsed.files }, null, 2) + '\n'); + } else { + process.stdout.write(`research-validator (dir): ${r.valid ? 'PASS' : 'FAIL'} ${args[dirIdx + 1]}\n`); + for (const e of r.errors) process.stderr.write(` ERROR [${e.code}] ${e.message}\n`); + for (const w of r.warnings) process.stderr.write(` WARN [${w.code}] ${w.message}\n`); + } + process.exit(r.valid ? 0 : 1); + } + const filePath = args.find(a => !a.startsWith('--')); + if (!filePath) { + process.stderr.write('Usage: research-validator.mjs [--soft] OR --dir \n'); + process.exit(2); + } + const r = validateResearch(filePath, { strict }); + if (args.includes('--json')) { + process.stdout.write(JSON.stringify({ valid: r.valid, errors: r.errors, warnings: r.warnings }, null, 2) + '\n'); + } else { + process.stdout.write(`research-validator: ${r.valid ? 'PASS' : 'FAIL'} ${filePath}\n`); + for (const e of r.errors) process.stderr.write(` ERROR [${e.code}] ${e.message}\n`); + for (const w of r.warnings) process.stderr.write(` WARN [${w.code}] ${w.message}\n`); + } + process.exit(r.valid ? 0 : 1); +} diff --git a/plugins/ultraplan-local/tests/lib/doc-consistency.test.mjs b/plugins/ultraplan-local/tests/lib/doc-consistency.test.mjs new file mode 100644 index 0000000..b2f9a34 --- /dev/null +++ b/plugins/ultraplan-local/tests/lib/doc-consistency.test.mjs @@ -0,0 +1,102 @@ +// tests/lib/doc-consistency.test.mjs +// Pin invariants between prose (CLAUDE.md, README.md) and source files +// (agents/*.md, commands/*.md, templates/, settings.json). +// +// When this test fails, fix the source-of-truth — do NOT rewrite the test to +// hide drift. Borrowed pattern from llm-security commit 97c5c9d. + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { readFileSync, readdirSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseDocument } from '../../lib/util/frontmatter.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = join(HERE, '..', '..'); + +function read(rel) { return readFileSync(join(ROOT, rel), 'utf-8'); } +function listMd(rel) { return readdirSync(join(ROOT, rel)).filter(f => f.endsWith('.md')); } + +test('CLAUDE.md agents table row count == agents/*.md file count', () => { + const md = read('CLAUDE.md'); + const agentFiles = listMd('agents'); + const agentTable = md.split('## Agents')[1] || ''; + const tableSection = agentTable.split('\n## ')[0]; + const dataRows = tableSection + .split('\n') + .filter(l => l.startsWith('|') && !l.match(/^\|[\s-]+\|/) && !l.match(/^\|\s*Agent\s*\|/)); + assert.equal( + dataRows.length, + agentFiles.length, + `Drift: ${agentFiles.length} agent files vs ${dataRows.length} CLAUDE.md table rows. ` + + `Sync agents/ ↔ CLAUDE.md.`, + ); +}); + +test('CLAUDE.md commands table mentions every commands/*.md file', () => { + const md = read('CLAUDE.md'); + const commandFiles = listMd('commands'); + for (const f of commandFiles) { + const cmdName = `/${f.replace(/\.md$/, '')}`; + assert.ok( + md.includes(cmdName), + `commands/${f} not mentioned in CLAUDE.md (looked for ${cmdName})`, + ); + } +}); + +test('every command frontmatter name matches its filename', () => { + for (const f of listMd('commands')) { + const text = read(`commands/${f}`); + const doc = parseDocument(text); + if (!doc.valid) continue; + const expected = f.replace(/\.md$/, ''); + if (doc.parsed.frontmatter && doc.parsed.frontmatter.name !== undefined) { + assert.equal( + doc.parsed.frontmatter.name, + expected, + `commands/${f} frontmatter.name="${doc.parsed.frontmatter.name}" should be "${expected}"`, + ); + } + } +}); + +test('templates/plan-template.md declares plan_version: 1.7', () => { + const tpl = read('templates/plan-template.md'); + assert.match(tpl, /plan_version:\s*['"]?1\.7['"]?/); +}); + +test('commands/ultraexecute-local.md still parses v1.7 plan schema', () => { + const cmd = read('commands/ultraexecute-local.md'); + const tpl = read('templates/plan-template.md'); + const tplVersion = (tpl.match(/plan_version:\s*['"]?([\d.]+)['"]?/) || [])[1]; + assert.ok(tplVersion, 'templates/plan-template.md missing plan_version'); + assert.ok( + cmd.includes(`plan_version`) || cmd.includes(`Step N:`) || cmd.includes('### Step '), + 'commands/ultraexecute-local.md should reference v1.7 plan-schema parsing', + ); +}); + +test('settings.json has only known top-level scopes after Spor 0 cleanup', () => { + const cfg = JSON.parse(read('settings.json')); + const known = ['ultraplan', 'ultraresearch']; + for (const k of Object.keys(cfg)) { + assert.ok(known.includes(k), `Unknown top-level scope in settings.json: ${k}`); + } +}); + +test('settings.json no longer carries vestigial exploration block', () => { + const cfg = JSON.parse(read('settings.json')); + assert.equal(cfg.ultraplan?.exploration, undefined, + 'exploration block was vestigial — should be deleted in v3.1.0 Spor 0'); + assert.equal(cfg.ultraplan?.agentTeam, undefined, + 'agentTeam block was vestigial — should be deleted in v3.1.0 Spor 0'); +}); + +test('CLAUDE.md mentions all four pipeline commands', () => { + const md = read('CLAUDE.md'); + for (const c of ['/ultrabrief-local', '/ultraresearch-local', '/ultraplan-local', '/ultraexecute-local']) { + assert.ok(md.includes(c), `CLAUDE.md missing reference to ${c}`); + } +}); diff --git a/plugins/ultraplan-local/tests/validators/architecture-discovery.test.mjs b/plugins/ultraplan-local/tests/validators/architecture-discovery.test.mjs new file mode 100644 index 0000000..c57ba6f --- /dev/null +++ b/plugins/ultraplan-local/tests/validators/architecture-discovery.test.mjs @@ -0,0 +1,81 @@ +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { discoverArchitecture } from '../../lib/validators/architecture-discovery.mjs'; + +function setup(structure) { + const root = mkdtempSync(join(tmpdir(), 'ultraplan-arch-')); + for (const [path, content] of Object.entries(structure)) { + const full = join(root, path); + mkdirSync(join(full, '..'), { recursive: true }); + writeFileSync(full, content); + } + return root; +} + +test('discoverArchitecture — canonical overview.md found cleanly', () => { + const root = setup({ 'architecture/overview.md': '# Overview\n' }); + try { + const r = discoverArchitecture(root); + assert.equal(r.found, true); + assert.match(r.overview, /architecture\/overview\.md$/); + assert.equal(r.warnings.length, 0); + assert.equal(r.firstHeading, 'Overview'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('discoverArchitecture — no architecture dir = not found, no warnings', () => { + const root = setup({ 'brief.md': 'b' }); + try { + const r = discoverArchitecture(root); + assert.equal(r.found, false); + assert.equal(r.warnings.length, 0); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('discoverArchitecture — non-canonical name discovered with warning (drift-WARN)', () => { + const root = setup({ 'architecture/architecture-overview.md': '# Drifted\n' }); + try { + const r = discoverArchitecture(root); + assert.equal(r.found, true); + assert.ok(r.warnings.find(w => w.code === 'ARCH_NON_CANONICAL_OVERVIEW')); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('discoverArchitecture — loose unknown files surfaced as drift warning', () => { + const root = setup({ + 'architecture/overview.md': '# OK\n', + 'architecture/random-note.md': 'x', + 'architecture/another.md': 'y', + }); + try { + const r = discoverArchitecture(root); + assert.equal(r.found, true); + assert.ok(r.warnings.find(w => w.code === 'ARCH_LOOSE_FILES')); + assert.equal(r.looseFiles.length, 2); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('discoverArchitecture — gaps.md detected when present', () => { + const root = setup({ + 'architecture/overview.md': '# OK\n', + 'architecture/gaps.md': '# Gaps\n', + }); + try { + const r = discoverArchitecture(root); + assert.match(r.gaps, /architecture\/gaps\.md$/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('discoverArchitecture — never reads body beyond first heading', () => { + const root = setup({ + 'architecture/overview.md': '# Overview\n\n## Components\n\nlots of detail that we MUST NOT validate\n', + }); + try { + const r = discoverArchitecture(root); + assert.equal(r.firstHeading, 'Overview'); + // Validator does not assert on Components section — that's the contract. + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/plugins/ultraplan-local/tests/validators/brief-validator.test.mjs b/plugins/ultraplan-local/tests/validators/brief-validator.test.mjs new file mode 100644 index 0000000..481eb2a --- /dev/null +++ b/plugins/ultraplan-local/tests/validators/brief-validator.test.mjs @@ -0,0 +1,94 @@ +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { validateBriefContent } from '../../lib/validators/brief-validator.mjs'; + +const GOOD_BRIEF = `--- +type: ultrabrief +brief_version: "2.0" +created: 2026-04-30 +task: "Add JWT auth to API" +slug: jwt-auth +project_dir: .claude/projects/2026-04-30-jwt-auth/ +research_topics: 2 +research_status: pending +auto_research: false +interview_turns: 5 +source: interview +--- + +# Task: JWT auth + +## Intent + +Why this matters. + +## Goal + +What success looks like. + +## Success Criteria + +- All tests pass. +`; + +test('validateBrief — happy path', () => { + const r = validateBriefContent(GOOD_BRIEF, { strict: true }); + assert.equal(r.valid, true, JSON.stringify(r.errors)); +}); + +test('validateBrief — wrong type rejected', () => { + const t = GOOD_BRIEF.replace('type: ultrabrief', 'type: notabrief'); + const r = validateBriefContent(t); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'BRIEF_WRONG_TYPE')); +}); + +test('validateBrief — missing required field', () => { + const t = GOOD_BRIEF.replace(/^research_topics: 2\n/m, ''); + const r = validateBriefContent(t); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'BRIEF_MISSING_FIELD' && /research_topics/.test(e.message))); +}); + +test('validateBrief — bad research_status value', () => { + const t = GOOD_BRIEF.replace('research_status: pending', 'research_status: maybe'); + const r = validateBriefContent(t); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'BRIEF_BAD_STATUS')); +}); + +test('validateBrief — state machine: research_topics > 0 + skipped without partial = error', () => { + const t = GOOD_BRIEF.replace('research_status: pending', 'research_status: skipped'); + const r = validateBriefContent(t); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'BRIEF_STATE_INCOHERENT')); +}); + +test('validateBrief — state machine: skipped + brief_quality: partial = warning only', () => { + const t = GOOD_BRIEF + .replace('research_status: pending', 'research_status: skipped') + .replace('source: interview', 'source: interview\nbrief_quality: partial'); + const r = validateBriefContent(t); + assert.equal(r.valid, true, JSON.stringify(r.errors)); + assert.ok(r.warnings.find(w => w.code === 'BRIEF_PARTIAL_SKIPPED')); +}); + +test('validateBrief — strict requires body sections', () => { + const t = GOOD_BRIEF.replace(/## Intent\n\nWhy this matters\.\n\n/, ''); + const r = validateBriefContent(t, { strict: true }); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'BRIEF_MISSING_SECTION')); +}); + +test('validateBrief — soft demotes section errors to warnings', () => { + const t = GOOD_BRIEF.replace(/## Goal\n\nWhat success looks like\.\n\n/, ''); + const r = validateBriefContent(t, { strict: false }); + assert.equal(r.valid, true); + assert.ok(r.warnings.find(w => w.code === 'BRIEF_MISSING_SECTION')); +}); + +test('validateBrief — missing frontmatter is hard error', () => { + const r = validateBriefContent('# just markdown\n\nno frontmatter\n'); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'FM_MISSING')); +}); diff --git a/plugins/ultraplan-local/tests/validators/plan-validator.test.mjs b/plugins/ultraplan-local/tests/validators/plan-validator.test.mjs new file mode 100644 index 0000000..a5569a6 --- /dev/null +++ b/plugins/ultraplan-local/tests/validators/plan-validator.test.mjs @@ -0,0 +1,99 @@ +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { validatePlanContent } from '../../lib/validators/plan-validator.mjs'; + +const VALID_PLAN = `--- +plan_version: "1.7" +--- + +# Plan + +## Implementation Plan + +### Step 1: Add foo + +- Files: a.ts +- Manifest: + \`\`\`yaml + manifest: + expected_paths: + - a.ts + min_file_count: 1 + commit_message_pattern: "^feat:" + bash_syntax_check: [] + forbidden_paths: [] + must_contain: [] + \`\`\` + +### Step 2: Add bar + +- Files: b.ts +- Manifest: + \`\`\`yaml + manifest: + expected_paths: + - b.ts + min_file_count: 1 + commit_message_pattern: "^feat:" + bash_syntax_check: [] + forbidden_paths: [] + must_contain: [] + \`\`\` +`; + +const FORBIDDEN_PLAN = `--- +plan_version: "1.7" +--- + +## Fase 1: Drift form + +content +`; + +const STEP_WITHOUT_MANIFEST = `### Step 1: oops +no manifest + +### Step 2: ok + +- Manifest: + \`\`\`yaml + manifest: + expected_paths: [foo] + min_file_count: 1 + commit_message_pattern: "^x:" + bash_syntax_check: [] + forbidden_paths: [] + must_contain: [] + \`\`\` +`; + +test('validatePlan — strict accepts canonical v1.7 plan', () => { + const r = validatePlanContent(VALID_PLAN, { strict: true }); + assert.equal(r.valid, true, JSON.stringify(r.errors)); + assert.equal(r.parsed.steps.length, 2); + assert.equal(r.parsed.planVersion, '1.7'); +}); + +test('validatePlan — forbidden Fase form blocks in strict mode', () => { + const r = validatePlanContent(FORBIDDEN_PLAN, { strict: true }); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'PLAN_FORBIDDEN_HEADING')); +}); + +test('validatePlan — manifest count mismatch caught', () => { + const r = validatePlanContent(STEP_WITHOUT_MANIFEST, { strict: true }); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => /Step 1/.test(e.message) && /MANIFEST_MISSING/.test(e.code))); +}); + +test('validatePlan — version warning when missing', () => { + const noVersion = VALID_PLAN.replace(/plan_version: "1\.7"\n/, ''); + const r = validatePlanContent(noVersion, { strict: true }); + assert.ok(r.warnings.find(w => w.code === 'PLAN_NO_VERSION')); +}); + +test('validatePlan — older version triggers warning', () => { + const old = VALID_PLAN.replace('plan_version: "1.7"', 'plan_version: "1.5"'); + const r = validatePlanContent(old, { strict: true }); + assert.ok(r.warnings.find(w => w.code === 'PLAN_VERSION_MISMATCH')); +}); diff --git a/plugins/ultraplan-local/tests/validators/progress-validator.test.mjs b/plugins/ultraplan-local/tests/validators/progress-validator.test.mjs new file mode 100644 index 0000000..4ca31b6 --- /dev/null +++ b/plugins/ultraplan-local/tests/validators/progress-validator.test.mjs @@ -0,0 +1,79 @@ +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { validateProgressObject, checkResumeReadiness } from '../../lib/validators/progress-validator.mjs'; + +function goodProgress() { + return { + schema_version: '1', + plan: '.claude/projects/x/plan.md', + plan_type: 'plan', + plan_version: '1.7', + started_at: '2026-04-18T12:00:00Z', + updated_at: '2026-04-18T13:00:00Z', + mode: 'execute', + total_steps: 2, + current_step: 1, + status: 'in_progress', + steps: { + '1': { status: 'completed', attempts: 1, error: null, completed_at: '2026-04-18T12:30:00Z', commit: 'abc123', manifest_audit: 'pass' }, + '2': { status: 'pending', attempts: 0, error: null, completed_at: null, commit: null, manifest_audit: null }, + }, + }; +} + +test('validateProgress — happy path', () => { + const r = validateProgressObject(goodProgress()); + assert.equal(r.valid, true, JSON.stringify(r.errors)); +}); + +test('validateProgress — wrong schema_version', () => { + const p = goodProgress(); + p.schema_version = '2'; + const r = validateProgressObject(p); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'PROGRESS_SCHEMA_MISMATCH')); +}); + +test('validateProgress — missing required field', () => { + const p = goodProgress(); + delete p.total_steps; + const r = validateProgressObject(p); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'PROGRESS_MISSING_FIELD' && /total_steps/.test(e.message))); +}); + +test('validateProgress — bad status', () => { + const p = goodProgress(); + p.status = 'maybe'; + const r = validateProgressObject(p); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'PROGRESS_BAD_STATUS')); +}); + +test('validateProgress — current_step out of range', () => { + const p = goodProgress(); + p.current_step = 99; + const r = validateProgressObject(p); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'PROGRESS_STEP_RANGE')); +}); + +test('validateProgress — step count mismatch is warning', () => { + const p = goodProgress(); + p.total_steps = 5; + const r = validateProgressObject(p); + assert.ok(r.warnings.find(w => w.code === 'PROGRESS_STEP_COUNT_MISMATCH')); +}); + +test('checkResumeReadiness — completed run cannot resume', () => { + const p = goodProgress(); + p.status = 'completed'; + const r = checkResumeReadiness(p); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'PROGRESS_ALREADY_DONE')); +}); + +test('checkResumeReadiness — in-progress is resumable', () => { + const r = checkResumeReadiness(goodProgress()); + assert.equal(r.valid, true); +}); diff --git a/plugins/ultraplan-local/tests/validators/research-validator.test.mjs b/plugins/ultraplan-local/tests/validators/research-validator.test.mjs new file mode 100644 index 0000000..d7c8071 --- /dev/null +++ b/plugins/ultraplan-local/tests/validators/research-validator.test.mjs @@ -0,0 +1,60 @@ +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { validateResearchContent } from '../../lib/validators/research-validator.mjs'; + +const GOOD = `--- +type: ultraresearch-brief +created: 2026-04-30 +question: "How to do X?" +confidence: 0.8 +dimensions: 3 +--- + +## Executive Summary + +3 sentences. + +## Dimensions + +### Dim A — Confidence: high +`; + +test('validateResearch — happy path', () => { + const r = validateResearchContent(GOOD); + assert.equal(r.valid, true, JSON.stringify(r.errors)); +}); + +test('validateResearch — wrong type', () => { + const t = GOOD.replace('type: ultraresearch-brief', 'type: random'); + const r = validateResearchContent(t); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'RESEARCH_WRONG_TYPE')); +}); + +test('validateResearch — confidence out of range', () => { + const t = GOOD.replace('confidence: 0.8', 'confidence: 1.5'); + const r = validateResearchContent(t); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'RESEARCH_BAD_CONFIDENCE')); +}); + +test('validateResearch — missing confidence is warning, not error', () => { + const t = GOOD.replace(/^confidence: 0\.8\n/m, ''); + const r = validateResearchContent(t); + assert.equal(r.valid, true); + assert.ok(r.warnings.find(w => w.code === 'RESEARCH_NO_CONFIDENCE')); +}); + +test('validateResearch — strict missing body section is error', () => { + const t = GOOD.replace(/## Dimensions\n\n### Dim A — Confidence: high\n/, ''); + const r = validateResearchContent(t, { strict: true }); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'RESEARCH_MISSING_SECTION')); +}); + +test('validateResearch — bad dimensions value', () => { + const t = GOOD.replace('dimensions: 3', 'dimensions: 0'); + const r = validateResearchContent(t); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'RESEARCH_BAD_DIMENSIONS')); +});