feat(ultraplan-local): M1 — profile recommendation flow in ultrabrief

Adds the profile recommendation step to /ultrabrief-local Phase 4. The
brief stays universal (same questions, same template); the new step is
purely a processing-decision layer that records which profile downstream
commands should apply.

What lands:
- agents/profile-recommender.md — new sonnet agent that scores available
  profiles against the finalized brief (keyword + NFR-signal matching,
  axis bumps, hallucination gate that forbids inventing profile names).
  Emits a fenced JSON block with ranked entries.
- templates/ultrabrief-template.md — frontmatter gains
  recommended_profile, profile_match, profile_rationale (default values
  applied when only `default` is available — true at M1).
- commands/ultrabrief-local.md — Phase 4 gains Step 4h with explicit
  branches: short-circuit when only `default` exists; AskUserQuestion
  confirmation when top score ≥ 0.7; explicit fallback message when below
  threshold; manual selection sub-question on user override. Persists the
  three frontmatter fields to brief.md after user confirmation. JSON
  parser failure falls back to `default` with `profile_match: fallback`
  rather than blocking — silent fallback is the worst outcome, but a
  *visible* fallback is acceptable.
- scripts/profile-loader.mjs — adds selectRecommendation(ranked, opts) +
  RECOMMENDATION_THRESHOLD=0.7 export. Single source of truth for the
  threshold logic so the command spec and the helper agree.
- scripts/profile-loader.test.mjs — 10 new tests for selectRecommendation
  (default-only, empty/malformed input, above/below threshold, custom
  threshold, max-by-score, missing fields). Total now 36/36.
- README.md / CLAUDE.md / marketplace landing — docs reflect M0 + M1
  shipped, M2 + M3 still pending.

In practice nothing changes for users at M1 because only `default` is
available — Step 4h takes the short-circuit path and writes
`profile_match: default-only`. M2 ships the additional profiles that
make the recommender meaningful.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-04-30 14:21:54 +02:00
commit 7e2d9e151e
8 changed files with 609 additions and 15 deletions

View file

@ -400,6 +400,98 @@ async function missingAgents(names, agentsDir) {
return missing;
}
// =====================================================================
// Recommendation helper
// =====================================================================
/**
* Recommendation threshold used by ultrabrief-local Step 4h. The
* profile-recommender agent's top-ranked profile must reach this score to
* be presented as a recommendation; below it, ultrabrief falls back to
* `default` with an explicit message.
*/
export const RECOMMENDATION_THRESHOLD = 0.7;
/**
* Decide what to do with a `profile-recommender` agent's ranked output.
* Returns `{ profile, match, rationale, source }` where:
* - `source` is `recommended` (top threshold), `fallback` (top < threshold
* or empty input), or `default-only` (only `default` available).
* - `profile` is the chosen profile name.
* - `match` is one of `exact | partial | fallback | default-only`.
* - `rationale` is a one-sentence explanation suitable for the brief
* frontmatter.
*
* Rules:
* - If `availableProfiles` only contains `default`, return `default-only`.
* - If `ranked` is empty/malformed, fall back to `default` with a fallback
* rationale.
* - Otherwise pick the highest-scoring entry; recommend it only when
* `score >= RECOMMENDATION_THRESHOLD`. Below threshold, recommend
* `default` with `match: fallback` and the top entry's rationale.
*/
export function selectRecommendation(ranked, opts = {}) {
const threshold = opts.threshold ?? RECOMMENDATION_THRESHOLD;
const available = opts.availableProfiles ?? null;
if (Array.isArray(available) && available.length === 1 && available[0] === 'default') {
return {
profile: 'default',
match: 'default-only',
rationale: 'Only the default profile is available; recommendation skipped.',
source: 'default-only',
};
}
if (!Array.isArray(ranked) || ranked.length === 0) {
return {
profile: 'default',
match: 'fallback',
rationale: 'profile-recommender returned no ranked profiles; using default.',
source: 'fallback',
};
}
// Find highest-scoring entry. Treat missing/non-numeric scores as 0.
let top = null;
for (const entry of ranked) {
if (!entry || typeof entry.name !== 'string') continue;
const score = typeof entry.score === 'number' ? entry.score : 0;
if (top === null || score > (typeof top.score === 'number' ? top.score : 0)) {
top = { ...entry, score };
}
}
if (top === null) {
return {
profile: 'default',
match: 'fallback',
rationale: 'profile-recommender output had no usable entries; using default.',
source: 'fallback',
};
}
if (top.score >= threshold) {
return {
profile: top.name,
match: typeof top.match_quality === 'string' ? top.match_quality : 'partial',
rationale: typeof top.rationale === 'string' && top.rationale.trim() !== ''
? top.rationale
: `Top-ranked profile (score ${top.score}).`,
source: 'recommended',
};
}
return {
profile: 'default',
match: 'fallback',
rationale: typeof top.rationale === 'string' && top.rationale.trim() !== ''
? `Top score ${top.score} below ${threshold}; ${top.rationale}`
: `Top score ${top.score} below recommendation threshold ${threshold}.`,
source: 'fallback',
};
}
// =====================================================================
// CLI
// =====================================================================

View file

@ -18,6 +18,8 @@ import {
validateProfile,
loadProfile,
listProfiles,
selectRecommendation,
RECOMMENDATION_THRESHOLD,
} from './profile-loader.mjs';
// =====================================================================
@ -377,3 +379,99 @@ test('listProfiles: includes default', async () => {
const names = await listProfiles();
assert.ok(names.includes('default'), `Expected default in ${names.join(', ')}`);
});
// =====================================================================
// selectRecommendation tests (M1)
// =====================================================================
test('selectRecommendation: only-default short-circuit', () => {
const result = selectRecommendation([], { availableProfiles: ['default'] });
assert.equal(result.profile, 'default');
assert.equal(result.match, 'default-only');
assert.equal(result.source, 'default-only');
});
test('selectRecommendation: empty ranked input falls back', () => {
const result = selectRecommendation([]);
assert.equal(result.profile, 'default');
assert.equal(result.match, 'fallback');
assert.equal(result.source, 'fallback');
assert.match(result.rationale, /no ranked profiles/);
});
test('selectRecommendation: malformed ranked input falls back', () => {
const result = selectRecommendation([null, { not_a_profile: true }]);
assert.equal(result.profile, 'default');
assert.equal(result.source, 'fallback');
});
test('selectRecommendation: top score above threshold returns recommendation', () => {
const ranked = [
{ name: 'security-deep', score: 0.91, match_quality: 'exact', rationale: 'OWASP + JWT in Intent.' },
{ name: 'default', score: 0.30, match_quality: 'fallback', rationale: 'No triggers.' },
];
const result = selectRecommendation(ranked);
assert.equal(result.profile, 'security-deep');
assert.equal(result.match, 'exact');
assert.equal(result.source, 'recommended');
assert.equal(result.rationale, 'OWASP + JWT in Intent.');
});
test('selectRecommendation: top score below threshold falls back to default', () => {
const ranked = [
{ name: 'feature', score: 0.55, match_quality: 'partial', rationale: 'Some keyword hits.' },
{ name: 'default', score: 0.30, match_quality: 'fallback', rationale: 'Baseline.' },
];
const result = selectRecommendation(ranked);
assert.equal(result.profile, 'default');
assert.equal(result.match, 'fallback');
assert.equal(result.source, 'fallback');
// Rationale should reference both the score and the top entry's rationale
assert.match(result.rationale, /0\.55/);
assert.match(result.rationale, /Some keyword hits/);
});
test('selectRecommendation: respects custom threshold', () => {
const ranked = [
{ name: 'feature', score: 0.55, match_quality: 'partial', rationale: 'Match.' },
];
// With low threshold the same entry IS the recommendation
const result = selectRecommendation(ranked, { threshold: 0.5 });
assert.equal(result.profile, 'feature');
assert.equal(result.source, 'recommended');
});
test('selectRecommendation: highest-score wins regardless of input order', () => {
const ranked = [
{ name: 'a', score: 0.40, match_quality: 'partial', rationale: 'Low.' },
{ name: 'b', score: 0.95, match_quality: 'exact', rationale: 'High.' },
{ name: 'c', score: 0.72, match_quality: 'partial', rationale: 'Mid.' },
];
const result = selectRecommendation(ranked);
assert.equal(result.profile, 'b');
assert.equal(result.source, 'recommended');
});
test('selectRecommendation: missing score treated as 0', () => {
const ranked = [
{ name: 'a', match_quality: 'fallback', rationale: 'No score.' },
];
const result = selectRecommendation(ranked);
// Top entry has effective score 0 → falls back
assert.equal(result.profile, 'default');
assert.equal(result.source, 'fallback');
});
test('selectRecommendation: missing rationale gets a synthetic one', () => {
const ranked = [
{ name: 'security-deep', score: 0.85, match_quality: 'exact' },
];
const result = selectRecommendation(ranked);
assert.equal(result.profile, 'security-deep');
assert.match(result.rationale, /Top-ranked/);
});
test('RECOMMENDATION_THRESHOLD: matches plan default', () => {
// Sanity check that the export agrees with the documented threshold.
assert.equal(RECOMMENDATION_THRESHOLD, 0.7);
});