feat(linkedin-studio): RE-R2a — item→store capture bridge + publishedAt persistence (schema v1→v2, lossless migrate) [skip-docs]
Closes the research-engine capture loop RE-R1 deferred:
- itemToInput(item, capturedAt): pure envelope→TrendInput bridge in item.ts —
injects capturedAt, carries publishedAt verbatim; no id, no re-validate
- publishedAt persisted: TrendRecord/TrendInput gain it; addTrend conditional-spread,
first-sight kept on re-capture (no back-fill). SCHEMA_VERSION 1→2 with a lossless
forward migrate-on-load: Math.max(onDisk, current) + numeric-typeof coercion
(string/NaN/absent → current; non-array trends coercion preserved verbatim)
- `capture` CLI: stdin raw item|batch → normalize → bridge → addTrend → saveStore once;
tally {added,duplicates,merged,errors} from AddResult; content-invalid → errors[],
exit 2 only on bad stdin; --json summary
- wiring: trend-spotter.md Step 4.5 N×`add` → one normalizing `capture` batch; README
add/capture framing corrected; test-runner Section 16h (capture wiring, unconditional)
+ floors bumped (trends 62→79, ASSERT 87→90)
TDD: 17 new tests (12 genuinely-RED logic-RED + 5 regression guards), tsc clean,
gate 105/0/0. No version bump (additive, v0.5.2 dev).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VmHCQjJHUyWwxGAVVjNLgp
This commit is contained in:
parent
b4e500fad4
commit
7a158030b6
10 changed files with 465 additions and 31 deletions
|
|
@ -40,8 +40,10 @@
|
|||
# reconcileRecentPosts by literal name, with a non-vacuity self-test) in Section 16f;
|
||||
# the trends-scorer wiring guard (RE-R1: scripts/trends/src/score.ts encodes both mode
|
||||
# weight-sets AND agents/trend-spotter.md references the scorer CLI 'src/cli.ts score',
|
||||
# with a non-vacuity self-test) in Section 16g; the assertion-count anti-erosion floor
|
||||
# (SC6) in Section 18. All are live below (Sections 8–18).
|
||||
# with a non-vacuity self-test) in Section 16g; the trends-capture wiring guard (RE-R2a:
|
||||
# scripts/trends/src/cli.ts dispatches `capture` AND agents/trend-spotter.md references the
|
||||
# capture CLI 'src/cli.ts capture', with a non-vacuity self-test) in Section 16h; the
|
||||
# assertion-count anti-erosion floor (SC6) in Section 18. All are live below (Sections 8–18).
|
||||
#
|
||||
# Usage: bash scripts/test-runner.sh
|
||||
# bash 3.2-safe: plain arrays only, no `declare -A`, no `mapfile`/`readarray`.
|
||||
|
|
@ -692,7 +694,7 @@ if [ -x "$TR_DIR/node_modules/.bin/tsx" ]; then
|
|||
TR_OUT=$( set +e; (cd "$TR_DIR" && npm test) 2>&1; echo "TR_EXIT:$?" )
|
||||
TR_EXIT=$(echo "$TR_OUT" | grep -oE 'TR_EXIT:[0-9]+' | grep -oE '[0-9]+' | head -1)
|
||||
TR_TESTS=$(echo "$TR_OUT" | grep -oE 'tests [0-9]+' | grep -oE '[0-9]+' | tail -1)
|
||||
TRENDS_TESTS_FLOOR=62 # store 24 + RE-R1: item 18 + score 16 + cli 4 (item-schema + triage-scorer)
|
||||
TRENDS_TESTS_FLOOR=79 # store 24 + RE-R1: item 18 + score 16 + cli 4 + RE-R2a: store +9 + item +4 + cli +4 (capture bridge + publishedAt)
|
||||
if [ "$TR_EXIT" = "0" ] && [ -n "$TR_TESTS" ] && [ "$TR_TESTS" -ge "$TRENDS_TESTS_FLOOR" ]; then
|
||||
pass "trends-store suite green: $TR_TESTS tests pass (floor $TRENDS_TESTS_FLOOR)"
|
||||
else
|
||||
|
|
@ -1069,6 +1071,50 @@ fi
|
|||
|
||||
echo ""
|
||||
|
||||
# --- Section 16h: Trends Capture Wiring (research-engine RE-R2a) ---
|
||||
echo "--- Trends Capture Wiring ---"
|
||||
|
||||
# RE-R2a closes the capture loop: the item->store bridge (itemToInput) + a `capture` CLI that
|
||||
# normalizes a raw batch from stdin and folds it into the store (persisting publishedAt). Two
|
||||
# literals must hold, grepped EXACT (grep -F), deps-absent-safe (pure grep, no tsx):
|
||||
# (1) cli.ts dispatches the `capture` subcommand (the handler exists), by the literal
|
||||
# 'command === "capture"' (the capture path is real, not merely documented);
|
||||
# (2) agents/trend-spotter.md re-points Step 4.5 to the capture CLI by the literal
|
||||
# 'src/cli.ts capture' (one normalizing batch call, replacing the N x `add` block).
|
||||
# Non-vacuity self-test mirrors Sections 16c-16g: the wiring predicate must accept a probe
|
||||
# carrying the capture-pointer literal and reject one without it. Labelled 16h but placed after
|
||||
# Section 17 / before Section 18 (anti-erosion must run last so it sees every prior check).
|
||||
# UNCONDITIONAL (no tsx) -> counts toward ASSERT_BASELINE_FLOOR.
|
||||
CAPTURE_HANDLER_LIT='command === "capture"'
|
||||
CAPTURE_WIRE_LIT='src/cli.ts capture'
|
||||
|
||||
H16_SELFTEST_OK=1
|
||||
if ! echo 'pipe the raw batch to src/cli.ts capture for the store fold' | grep -qF "$CAPTURE_WIRE_LIT"; then
|
||||
H16_SELFTEST_OK=0; echo " non-vacuity FAIL: a wired capture-pointer probe was not detected"
|
||||
fi
|
||||
if echo 'the agent folds each trend into the store itself' | grep -qF "$CAPTURE_WIRE_LIT"; then
|
||||
H16_SELFTEST_OK=0; echo " false-positive FAIL: an unwired probe matched the capture pointer"
|
||||
fi
|
||||
if [ "$H16_SELFTEST_OK" -eq 1 ]; then
|
||||
pass "trends-capture self-test: capture-pointer predicate detects wiring, rejects the under-wired form"
|
||||
else
|
||||
fail "trends-capture self-test failed — the capture-wiring lint is vacuous or over-eager"
|
||||
fi
|
||||
|
||||
if grep -qF "$CAPTURE_HANDLER_LIT" scripts/trends/src/cli.ts; then
|
||||
pass "cli.ts dispatches the capture subcommand ('$CAPTURE_HANDLER_LIT')"
|
||||
else
|
||||
fail "cli.ts has no capture handler — add a '$CAPTURE_HANDLER_LIT' branch (RE-R2a capture loop)"
|
||||
fi
|
||||
|
||||
if grep -qF "$CAPTURE_WIRE_LIT" agents/trend-spotter.md; then
|
||||
pass "trend-spotter.md references the capture CLI ('$CAPTURE_WIRE_LIT') as the Step 4.5 store-fold owner"
|
||||
else
|
||||
fail "trend-spotter.md does not reference the capture CLI — re-point Step 4.5 to a '$CAPTURE_WIRE_LIT' batch call (RE-R2a wiring)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# --- Section 18: Assertion-Count Anti-Erosion (SC6) ---
|
||||
# The lint self-modifies its own checks, so a green run could mask a silently dropped
|
||||
# assertion. Pin the total pass()+fail() invocations as a monotonic floor; the count
|
||||
|
|
@ -1080,12 +1126,14 @@ echo ""
|
|||
# checks (ops-reader self-test + strategy-advisor ops-wiring grep) = 82; +2 for SB-S3e's
|
||||
# two UNCONDITIONAL Section-16f checks (reconcile self-test + brain-CLI reconcile grep) = 84;
|
||||
# +3 for RE-R1's three UNCONDITIONAL Section-16g checks (trends-scorer self-test + score.ts
|
||||
# both-modes weight-set grep + trend-spotter scorer-pointer grep) = 87.
|
||||
# both-modes weight-set grep + trend-spotter scorer-pointer grep) = 87; +3 for RE-R2a's three
|
||||
# UNCONDITIONAL Section-16h checks (trends-capture self-test + cli.ts capture-handler grep +
|
||||
# trend-spotter capture-pointer grep) = 90.
|
||||
# NB: the floor tracks the deps-absent MINIMUM (conditional TS suites warn-skip and drop
|
||||
# the count), so it is bumped only by UNCONDITIONAL new checks — NOT pinned to the
|
||||
# deps-present TOTAL_CHECKS (that would zero the warn-skip margin and false-fail a fresh
|
||||
# clone). Runs last so TOTAL_CHECKS sees every prior check.
|
||||
ASSERT_BASELINE_FLOOR=87
|
||||
ASSERT_BASELINE_FLOOR=90
|
||||
TOTAL_CHECKS=$((PASS + FAIL))
|
||||
if [ "$TOTAL_CHECKS" -ge "$ASSERT_BASELINE_FLOOR" ]; then
|
||||
pass "assertion-count anti-erosion: $TOTAL_CHECKS checks >= baseline floor $ASSERT_BASELINE_FLOOR"
|
||||
|
|
|
|||
|
|
@ -35,7 +35,8 @@ interface TrendRecord {
|
|||
title: string; // headline, verbatim
|
||||
url: string; // source URL, verbatim
|
||||
source: string; // "tavily" | "websearch" | "manual" | <mcp-name>
|
||||
capturedAt: string; // ISO-8601 date
|
||||
capturedAt: string; // ISO-8601 date — when WE captured it
|
||||
publishedAt?: string;// optional source publish date (ISO-8601); distinct from capturedAt, first-sight, never back-filled
|
||||
topics: string[]; // query tags; unioned across re-captures
|
||||
summary?: string; // optional, verbatim
|
||||
}
|
||||
|
|
@ -47,7 +48,16 @@ slice without breaking the shape.
|
|||
## CLI
|
||||
|
||||
```bash
|
||||
# Capture a freshly-polled trend (dedupes on title+url; unions topics on re-capture)
|
||||
# Capture freshly-polled trends — the NORMALIZING BATCH path (the research agent's path):
|
||||
# raw items on stdin → validate+normalize each → dedupe on title+url → union topics on
|
||||
# re-capture → persist the source's publishedAt. Content-invalid items are reported in the
|
||||
# summary errors[], never fail the run; the summary is {added, duplicates, merged, errors}.
|
||||
echo '[{"source":"tavily","title":"Agentic workflows hit production",
|
||||
"url":"https://example.com/agentic","topics":["agents","engineering"],
|
||||
"publishedAt":"2026-06-20","summary":"Teams ship multi-step agents past the demo stage."}]' \
|
||||
| node --import tsx src/cli.ts capture [--store <path>] [--json]
|
||||
|
||||
# Add a SINGLE trend MANUALLY — raw flags, no normalization, publish-date-free:
|
||||
node --import tsx src/cli.ts add \
|
||||
--title "Agentic workflows hit production" \
|
||||
--url "https://example.com/agentic" \
|
||||
|
|
@ -61,7 +71,8 @@ node --import tsx src/cli.ts query --topics "agents,engineering" [--json]
|
|||
node --import tsx src/cli.ts list [--since 2026-06-01] [--limit 10] [--json]
|
||||
```
|
||||
|
||||
Re-running `add` with the same title+url never appends a duplicate.
|
||||
Both `capture` and `add` dedupe on normalized title+url — re-capturing the same trend
|
||||
never appends a duplicate, it only unions any new topics in.
|
||||
|
||||
## Tests
|
||||
|
||||
|
|
|
|||
|
|
@ -9,15 +9,20 @@
|
|||
* node --import tsx src/cli.ts status [--store <path>] [--json]
|
||||
* echo '<raw item|batch>' | node --import tsx src/cli.ts normalize
|
||||
* echo '<scored candidates>' | node --import tsx src/cli.ts score [--mode kortform|long-form] [--threshold N]
|
||||
* echo '<raw item|batch>' | node --import tsx src/cli.ts capture [--store <path>] [--json]
|
||||
*
|
||||
* The capture agent (research-engine) calls `add` to fold a freshly-polled trend
|
||||
* into the store, and `query`/`list` to reason over accumulated history. The
|
||||
* The capture agent (research-engine) folds freshly-polled trends into the store via
|
||||
* `capture` (the normalizing batch path: stdin → normalizeItem(s) → itemToInput →
|
||||
* addTrend), and reasons over accumulated history via `query`/`list`. `add` is the
|
||||
* MANUAL single-trend path (raw flags, no normalization, publish-date-free). The
|
||||
* polling + relevance-scoring itself lives upstream; this is the deterministic store.
|
||||
*
|
||||
* `normalize` + `score` (RE-R1) are the deterministic research-engine seam: both read
|
||||
* their JSON PAYLOAD FROM STDIN (so they do not overload `--json`, which stays an
|
||||
* output toggle) and print JSON to stdout. `normalize` validates raw items into the
|
||||
* canonical envelope; `score` triages scored candidates (composite/band/threshold).
|
||||
* `normalize` + `score` (RE-R1) and `capture` (RE-R2a) are the deterministic
|
||||
* research-engine seam: all read their JSON PAYLOAD FROM STDIN (so they do not overload
|
||||
* `--json`, which stays an output toggle). `normalize` validates raw items into the
|
||||
* canonical envelope; `score` triages scored candidates (composite/band/threshold);
|
||||
* `capture` normalizes + folds each valid item into the store (persisting `publishedAt`),
|
||||
* reporting content-invalid items in the summary `errors[]`, never via the exit code.
|
||||
*
|
||||
* Exit code: 0 on success, 2 on usage error (incl. unparseable stdin / bad flag).
|
||||
*/
|
||||
|
|
@ -33,7 +38,7 @@ import {
|
|||
queryByTopic,
|
||||
saveStore,
|
||||
} from "./store.js";
|
||||
import { normalizeItem, normalizeItems } from "./item.js";
|
||||
import { normalizeItem, normalizeItems, itemToInput } from "./item.js";
|
||||
import { triage } from "./score.js";
|
||||
import type { ScoreMode } from "./score.js";
|
||||
|
||||
|
|
@ -72,7 +77,8 @@ function usage(msg: string): never {
|
|||
" list [--since <YYYY-MM-DD>] [--limit <n>] [--store <path>] [--json]\n" +
|
||||
" status [--store <path>] [--json]\n" +
|
||||
" normalize < raw-item-or-batch.json\n" +
|
||||
" score [--mode kortform|long-form] [--threshold N] < scored-candidates.json",
|
||||
" score [--mode kortform|long-form] [--threshold N] < scored-candidates.json\n" +
|
||||
" capture [--store <path>] [--json] < raw-item-or-batch.json",
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
|
|
@ -228,6 +234,34 @@ function main(): void {
|
|||
return;
|
||||
}
|
||||
|
||||
if (command === "capture") {
|
||||
const payload = readStdinJson(); // exits 2 on empty/unparseable stdin
|
||||
const raw = Array.isArray(payload) ? payload : [payload];
|
||||
const { items, errors } = normalizeItems(raw);
|
||||
const store = loadStore(storePath);
|
||||
// Tally derived from AddResult {added, merged} (no `duplicates` field): a fold is
|
||||
// `added` (new), else `merged` (existing gained topics), else a plain `duplicate`.
|
||||
let added = 0;
|
||||
let merged = 0;
|
||||
let duplicates = 0;
|
||||
for (const item of items) {
|
||||
const res = addTrend(store, itemToInput(item, today()));
|
||||
if (res.added) added++;
|
||||
else if (res.merged) merged++;
|
||||
else duplicates++;
|
||||
}
|
||||
saveStore(storePath, store);
|
||||
if (asJson) {
|
||||
console.log(JSON.stringify({ added, duplicates, merged, errors }, null, 2));
|
||||
return;
|
||||
}
|
||||
console.log(
|
||||
`Captured into ${storePath}: ${added} added, ${merged} merged, ` +
|
||||
`${duplicates} duplicate, ${errors.length} invalid (${store.trends.length} total)`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
usage(command ? `unknown command: ${command}` : "no command given");
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
*/
|
||||
|
||||
import { normalizeField } from "./store.js";
|
||||
import type { TrendInput } from "./store.js";
|
||||
|
||||
export interface TrendItem {
|
||||
/** Capture origin: a research-MCP name ("tavily"), "websearch", or "manual". Stored VERBATIM. */
|
||||
|
|
@ -117,6 +118,26 @@ export function normalizeItem(raw: unknown): NormalizeResult {
|
|||
return { ok: true, item };
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a validated envelope to a store input (the item→store bridge RE-R1 deferred).
|
||||
* Pure: injects `capturedAt` (the store's "when WE saw it", supplied by the caller —
|
||||
* never derived here) and carries the rest verbatim. Does NOT re-validate (the item is
|
||||
* already validated by normalizeItem) and does NOT derive an `id` (the store owns id via
|
||||
* addTrend→trendId). `publishedAt`/`summary` are carried only when present (key omitted
|
||||
* otherwise), mirroring the store's conditional-spread idiom.
|
||||
*/
|
||||
export function itemToInput(item: TrendItem, capturedAt: string): TrendInput {
|
||||
return {
|
||||
source: item.source,
|
||||
title: item.title,
|
||||
url: item.url,
|
||||
capturedAt,
|
||||
topics: [...item.topics],
|
||||
...(item.publishedAt !== undefined ? { publishedAt: item.publishedAt } : {}),
|
||||
...(item.summary !== undefined ? { summary: item.summary } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Partition a raw batch into normalized items + per-index errors (never throws). */
|
||||
export function normalizeItems(raw: unknown[]): { items: TrendItem[]; errors: ItemError[] } {
|
||||
const items: TrendItem[] = [];
|
||||
|
|
|
|||
|
|
@ -28,6 +28,8 @@ export interface TrendInput {
|
|||
url: string;
|
||||
source: string;
|
||||
capturedAt: string;
|
||||
/** The source's own publish date (ISO-8601), if known. Distinct from capturedAt; persisted first-sight, never back-filled. */
|
||||
publishedAt?: string;
|
||||
topics: string[];
|
||||
summary?: string;
|
||||
}
|
||||
|
|
@ -74,8 +76,15 @@ export function emptyStore(): TrendStore {
|
|||
export function loadStore(path: string): TrendStore {
|
||||
if (!existsSync(path)) return emptyStore();
|
||||
const parsed = JSON.parse(readFileSync(path, "utf8")) as Partial<TrendStore>;
|
||||
// Forward migrate-on-load: stamp to the current version, never downgrade. v1→v2 is
|
||||
// purely additive-optional (an old record is already a valid v2 record that simply
|
||||
// lacks the optional publishedAt), so the migration is the version stamp alone —
|
||||
// records pass through untouched (lossless + idempotent for any well-formed store).
|
||||
// A string / NaN / absent version coerces to the current version (never crashes); the
|
||||
// non-array `trends` coercion below is unchanged and out of the losslessness claim.
|
||||
const onDisk = typeof parsed.schemaVersion === "number" ? parsed.schemaVersion : SCHEMA_VERSION;
|
||||
return {
|
||||
schemaVersion: parsed.schemaVersion ?? SCHEMA_VERSION,
|
||||
schemaVersion: Math.max(onDisk, SCHEMA_VERSION),
|
||||
trends: Array.isArray(parsed.trends) ? parsed.trends : [],
|
||||
};
|
||||
}
|
||||
|
|
@ -122,6 +131,7 @@ export function addTrend(store: TrendStore, input: TrendInput): AddResult {
|
|||
url: input.url,
|
||||
source: input.source,
|
||||
capturedAt: input.capturedAt,
|
||||
...(input.publishedAt !== undefined ? { publishedAt: input.publishedAt } : {}),
|
||||
topics: [...input.topics],
|
||||
...(input.summary !== undefined ? { summary: input.summary } : {}),
|
||||
};
|
||||
|
|
|
|||
|
|
@ -34,6 +34,13 @@ export interface TrendRecord {
|
|||
source: string;
|
||||
/** ISO-8601 date the trend was captured. Supplied by the caller (CLI edge). */
|
||||
capturedAt: string;
|
||||
/**
|
||||
* The SOURCE's own publish date (ISO-8601), if the captured item carried one —
|
||||
* distinct from `capturedAt` (when WE saw it): this is when the source published.
|
||||
* First-sight provenance: kept, never overwritten on re-capture. Forward-compat
|
||||
* for B4 freshness ranking. Absent when the source gave no date (key omitted).
|
||||
*/
|
||||
publishedAt?: string;
|
||||
/** Topic tags for query-by-topic. Unioned across re-captures of the same trend. */
|
||||
topics: string[];
|
||||
/** Optional short summary of the trend, stored VERBATIM. */
|
||||
|
|
@ -52,4 +59,4 @@ export interface TrendQueryHit {
|
|||
topicOverlap: number;
|
||||
}
|
||||
|
||||
export const SCHEMA_VERSION = 1;
|
||||
export const SCHEMA_VERSION = 2;
|
||||
|
|
|
|||
|
|
@ -2,6 +2,9 @@ import { describe, test } from "node:test";
|
|||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { mkdtempSync, rmSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
// Resolve the package root (scripts/trends) so the subprocess `src/cli.ts` path + the
|
||||
// `tsx` loader resolve regardless of the runner's cwd.
|
||||
|
|
@ -67,4 +70,92 @@ describe("trends CLI — normalize/score subcommands (RE-R1 / Step 4)", () => {
|
|||
assert.equal(status, 2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("capture (stdin raw item|batch -> folds into the store) (RE-R2a / Step 4)", () => {
|
||||
const tmpStore = () => join(mkdtempSync(join(tmpdir(), "trends-capture-")), "trends.json");
|
||||
|
||||
test("happy path: a valid item piped in -> folded into the store, added:1, publishedAt persisted", () => {
|
||||
const store = tmpStore();
|
||||
try {
|
||||
const batch = JSON.stringify([
|
||||
{
|
||||
source: "tavily",
|
||||
title: "Captured",
|
||||
url: "https://example.com/c",
|
||||
topics: ["ai"],
|
||||
publishedAt: "2026-06-20",
|
||||
},
|
||||
]);
|
||||
const { status, stdout } = run(["capture", "--store", store, "--json"], batch);
|
||||
assert.equal(status, 0);
|
||||
const summary = JSON.parse(stdout);
|
||||
assert.equal(summary.added, 1);
|
||||
assert.equal(summary.errors.length, 0);
|
||||
assert.equal(
|
||||
summary.added + summary.merged + summary.duplicates + summary.errors.length,
|
||||
1,
|
||||
"tally must sum to the input size",
|
||||
);
|
||||
const persisted = JSON.parse(readFileSync(store, "utf8"));
|
||||
assert.equal(persisted.schemaVersion, 2);
|
||||
assert.equal(persisted.trends.length, 1);
|
||||
assert.equal(persisted.trends[0].publishedAt, "2026-06-20");
|
||||
assert.match(persisted.trends[0].capturedAt, /^\d{4}-\d{2}-\d{2}$/);
|
||||
assert.notEqual(
|
||||
persisted.trends[0].capturedAt,
|
||||
persisted.trends[0].publishedAt,
|
||||
"capturedAt (when WE saw it) must be distinct from publishedAt (source date)",
|
||||
);
|
||||
} finally {
|
||||
rmSync(join(store, ".."), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("a batch with one content-invalid item -> valid added, invalid in errors[], exit 0", () => {
|
||||
const store = tmpStore();
|
||||
try {
|
||||
const batch = JSON.stringify([
|
||||
{ source: "tavily", title: "Valid", url: "https://example.com/v", topics: ["x"] },
|
||||
{ title: "no source or url" },
|
||||
]);
|
||||
const { status, stdout } = run(["capture", "--store", store, "--json"], batch);
|
||||
assert.equal(status, 0);
|
||||
const summary = JSON.parse(stdout);
|
||||
assert.equal(summary.added, 1);
|
||||
assert.equal(summary.errors.length, 1);
|
||||
assert.equal(
|
||||
summary.added + summary.merged + summary.duplicates + summary.errors.length,
|
||||
2,
|
||||
);
|
||||
} finally {
|
||||
rmSync(join(store, ".."), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("re-capturing the same trend with a new topic -> merged:1, tally still sums", () => {
|
||||
const store = tmpStore();
|
||||
try {
|
||||
const item = (topics: string[]) =>
|
||||
JSON.stringify([{ source: "tavily", title: "Dup", url: "https://example.com/d", topics }]);
|
||||
run(["capture", "--store", store, "--json"], item(["a"]));
|
||||
const { status, stdout } = run(["capture", "--store", store, "--json"], item(["a", "b"]));
|
||||
assert.equal(status, 0);
|
||||
const summary = JSON.parse(stdout);
|
||||
assert.equal(summary.added, 0);
|
||||
assert.equal(summary.merged, 1);
|
||||
assert.equal(summary.duplicates, 0);
|
||||
assert.equal(
|
||||
summary.added + summary.merged + summary.duplicates + summary.errors.length,
|
||||
1,
|
||||
);
|
||||
} finally {
|
||||
rmSync(join(store, ".."), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("bad invocation: empty stdin -> exit 2", () => {
|
||||
const { status } = run(["capture"], "");
|
||||
assert.equal(status, 2);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { describe, test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { normalizeItem, normalizeItems } from "../src/item.js";
|
||||
import { normalizeItem, normalizeItems, itemToInput } from "../src/item.js";
|
||||
import { normalizeField } from "../src/store.js";
|
||||
|
||||
describe("trends item normalizer (RE-R1 / B1)", () => {
|
||||
|
|
@ -179,4 +179,49 @@ describe("trends item normalizer (RE-R1 / B1)", () => {
|
|||
assert.deepEqual(errors, []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("itemToInput — bridge to the store input (RE-R2a / B-bridge)", () => {
|
||||
// Build a realistic envelope via the normalizer (the actual upstream path).
|
||||
const mkItem = (extra: Record<string, unknown> = {}) => {
|
||||
const r = normalizeItem({
|
||||
source: "tavily",
|
||||
title: "OpenAI ships a reasoning model",
|
||||
url: "https://example.com/Article",
|
||||
topics: ["ai", "reasoning"],
|
||||
summary: "A short summary.",
|
||||
...extra,
|
||||
});
|
||||
assert.equal(r.ok, true);
|
||||
if (!r.ok) throw new Error("fixture item failed to normalize");
|
||||
return r.item;
|
||||
};
|
||||
|
||||
test("injects capturedAt and carries source/title/url/topics/summary/publishedAt verbatim", () => {
|
||||
const input = itemToInput(mkItem({ publishedAt: "2026-06-20" }), "2026-06-24");
|
||||
assert.equal(input.capturedAt, "2026-06-24");
|
||||
assert.equal(input.source, "tavily");
|
||||
assert.equal(input.title, "OpenAI ships a reasoning model");
|
||||
assert.equal(input.url, "https://example.com/Article");
|
||||
assert.deepEqual(input.topics, ["ai", "reasoning"]);
|
||||
assert.equal(input.summary, "A short summary.");
|
||||
assert.equal(input.publishedAt, "2026-06-20");
|
||||
});
|
||||
|
||||
test("derives NO id (the store owns id via addTrend->trendId)", () => {
|
||||
const input = itemToInput(mkItem(), "2026-06-24") as Record<string, unknown>;
|
||||
assert.equal("id" in input, false);
|
||||
});
|
||||
|
||||
test("an item without publishedAt -> input omits the key (not undefined-valued)", () => {
|
||||
const input = itemToInput(mkItem(), "2026-06-24"); // mkItem carries no publishedAt
|
||||
assert.equal("publishedAt" in input, false);
|
||||
});
|
||||
|
||||
test("field-confusion guard: capturedAt (injected) is distinct from publishedAt (carried)", () => {
|
||||
const input = itemToInput(mkItem({ publishedAt: "2026-06-20" }), "2026-06-24");
|
||||
assert.notEqual(input.capturedAt, input.publishedAt);
|
||||
assert.equal(input.publishedAt, "2026-06-20");
|
||||
assert.equal(input.capturedAt, "2026-06-24");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { describe, test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync, existsSync } from "node:fs";
|
||||
import { mkdtempSync, rmSync, existsSync, writeFileSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
|
|
@ -206,6 +206,75 @@ describe("trends store", () => {
|
|||
assert.equal(res2.added, false);
|
||||
assert.equal(res2.merged, false, "no new tags → merged:false");
|
||||
});
|
||||
|
||||
// ── RE-R2a: publishedAt persistence ──
|
||||
test("RED: persists publishedAt on a new record when present", () => {
|
||||
const res = addTrend(emptyStore(), {
|
||||
title: "Dated trend",
|
||||
url: "https://example.com/d",
|
||||
source: "tavily",
|
||||
capturedAt: "2026-06-24",
|
||||
topics: ["ai"],
|
||||
publishedAt: "2026-06-20",
|
||||
});
|
||||
assert.equal(res.store.trends[0].publishedAt, "2026-06-20");
|
||||
});
|
||||
|
||||
test("regression guard: omits publishedAt when absent (no undefined-valued key)", () => {
|
||||
const res = addTrend(emptyStore(), {
|
||||
title: "Undated trend",
|
||||
url: "https://example.com/u",
|
||||
source: "tavily",
|
||||
capturedAt: "2026-06-24",
|
||||
topics: ["ai"],
|
||||
});
|
||||
assert.equal("publishedAt" in res.store.trends[0], false);
|
||||
});
|
||||
|
||||
test("RED: re-capture keeps the first sighting's publishedAt (no overwrite), unions topics", () => {
|
||||
let store = emptyStore();
|
||||
store = addTrend(store, {
|
||||
title: "Same dated trend",
|
||||
url: "https://example.com/sd",
|
||||
source: "tavily",
|
||||
capturedAt: "2026-06-01",
|
||||
topics: ["a"],
|
||||
publishedAt: "2026-05-30",
|
||||
}).store;
|
||||
const res2 = addTrend(store, {
|
||||
title: "Same dated trend",
|
||||
url: "https://example.com/sd",
|
||||
source: "gemini",
|
||||
capturedAt: "2026-06-20",
|
||||
topics: ["a", "b"],
|
||||
publishedAt: "2026-06-15",
|
||||
});
|
||||
assert.equal(res2.added, false);
|
||||
assert.equal(res2.merged, true);
|
||||
assert.equal(res2.store.trends[0].publishedAt, "2026-05-30", "first-sight publishedAt kept");
|
||||
assert.deepEqual([...res2.store.trends[0].topics].sort(), ["a", "b"]);
|
||||
});
|
||||
|
||||
test("regression guard: no back-fill — re-capture carrying publishedAt onto a record that lacked one does NOT add it", () => {
|
||||
let store = emptyStore();
|
||||
store = addTrend(store, {
|
||||
title: "Undated first sight",
|
||||
url: "https://example.com/uf",
|
||||
source: "tavily",
|
||||
capturedAt: "2026-06-01",
|
||||
topics: ["a"],
|
||||
}).store; // no publishedAt on first sight
|
||||
const res2 = addTrend(store, {
|
||||
title: "Undated first sight",
|
||||
url: "https://example.com/uf",
|
||||
source: "tavily",
|
||||
capturedAt: "2026-06-20",
|
||||
topics: ["a", "b"],
|
||||
publishedAt: "2026-06-15",
|
||||
});
|
||||
assert.equal(res2.merged, true, "topic union still reported");
|
||||
assert.equal("publishedAt" in res2.store.trends[0], false, "no back-fill of first-sight provenance");
|
||||
});
|
||||
});
|
||||
|
||||
describe("queryByTopic", () => {
|
||||
|
|
@ -330,4 +399,98 @@ describe("trends store", () => {
|
|||
assert.equal(newestCaptureDate(store), "2026-06-01");
|
||||
});
|
||||
});
|
||||
|
||||
describe("schema migration (RE-R2a / publishedAt v1→v2)", () => {
|
||||
const withFixture = (contents: string, fn: (path: string) => void) => {
|
||||
const dir = tmp();
|
||||
const path = join(dir, "trends.json");
|
||||
try {
|
||||
writeFileSync(path, contents, "utf8");
|
||||
fn(path);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
};
|
||||
|
||||
// ── genuinely RED: old loadStore returns the on-disk/garbage version unchanged ──
|
||||
test("RED: a v1 store loads stamped as v2, records intact, no publishedAt invented", () => {
|
||||
const v1 = JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
trends: [
|
||||
{
|
||||
id: "abc123",
|
||||
title: "Old trend",
|
||||
url: "https://example.com/o",
|
||||
source: "tavily",
|
||||
capturedAt: "2026-05-01",
|
||||
topics: ["ai"],
|
||||
},
|
||||
],
|
||||
});
|
||||
withFixture(v1, (path) => {
|
||||
const s = loadStore(path);
|
||||
assert.equal(s.schemaVersion, 2, "v1 store must migrate to v2");
|
||||
assert.equal(s.trends.length, 1);
|
||||
assert.equal(s.trends[0].title, "Old trend");
|
||||
assert.equal(s.trends[0].capturedAt, "2026-05-01");
|
||||
assert.deepEqual(s.trends[0].topics, ["ai"]);
|
||||
assert.equal("publishedAt" in s.trends[0], false, "migration must not invent publishedAt");
|
||||
});
|
||||
});
|
||||
|
||||
test("RED: round-trip loadStore→saveStore writes schemaVersion:2 to disk", () => {
|
||||
withFixture(JSON.stringify({ schemaVersion: 1, trends: [] }), (path) => {
|
||||
saveStore(path, loadStore(path));
|
||||
const onDisk = JSON.parse(readFileSync(path, "utf8"));
|
||||
assert.equal(onDisk.schemaVersion, 2);
|
||||
});
|
||||
});
|
||||
|
||||
test("RED: a non-numeric schemaVersion is coerced to v2 (old code passes the string through)", () => {
|
||||
withFixture(JSON.stringify({ schemaVersion: "weird", trends: [] }), (path) => {
|
||||
assert.equal(loadStore(path).schemaVersion, 2);
|
||||
});
|
||||
});
|
||||
|
||||
// ── GREEN-only regression guards: old code already returns the current version ──
|
||||
test("regression guard: a v2 store loads as v2, idempotent (records + publishedAt intact)", () => {
|
||||
const v2 = JSON.stringify({
|
||||
schemaVersion: 2,
|
||||
trends: [
|
||||
{
|
||||
id: "x",
|
||||
title: "T",
|
||||
url: "https://example.com/t",
|
||||
source: "tavily",
|
||||
capturedAt: "2026-06-01",
|
||||
topics: ["ai"],
|
||||
publishedAt: "2026-05-30",
|
||||
},
|
||||
],
|
||||
});
|
||||
withFixture(v2, (path) => {
|
||||
const s = loadStore(path);
|
||||
assert.equal(s.schemaVersion, 2);
|
||||
assert.equal(s.trends[0].publishedAt, "2026-05-30");
|
||||
});
|
||||
});
|
||||
|
||||
test("regression guard: a store with no schemaVersion is stamped to the current version", () => {
|
||||
const noVer = JSON.stringify({
|
||||
trends: [
|
||||
{
|
||||
id: "x",
|
||||
title: "T",
|
||||
url: "https://example.com/t",
|
||||
source: "tavily",
|
||||
capturedAt: "2026-06-01",
|
||||
topics: ["ai"],
|
||||
},
|
||||
],
|
||||
});
|
||||
withFixture(noVer, (path) => {
|
||||
assert.equal(loadStore(path).schemaVersion, SCHEMA_VERSION);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue