Kjell Tore Guttormsen
5d1ff620be
Read-only audit av 389 ref-filer: 0 ekte orphans (mappe-referanse), 6 ekte datoløse, median 481 linjer (183 >500), 34 header-varianter, 21% uten kilde-URL. Spor D-scorer 91-96 (struktur sunn) — innholdskorrekthet er den umålte aksen. Beslutningsnotat for retning genereres separat.
109 lines
4.4 KiB
Python
109 lines
4.4 KiB
Python
#!/usr/bin/env python3
|
|
"""ref-file-audit.py — read-only structural/usage audit of the reference KB.
|
|
|
|
Reports, with verified ground truth, on the ~389 files under
|
|
skills/<skill>/references/**/*.md across six dimensions:
|
|
|
|
1. Inventory — count per skill.
|
|
2. Size — line distribution (median/mean/max + buckets + biggest).
|
|
3. Date hygiene — day-precise (YYYY-MM-DD) vs month-only (YYYY-MM) vs truly
|
|
dateless. Month-only is a deliberate convention
|
|
(report-changes.mjs parses YYYY-MM as -01), NOT a defect.
|
|
4. Reachability — named by basename in a SKILL.md/agent, reachable via FOLDER
|
|
reference (progressive disclosure), or a true orphan.
|
|
Folder-awareness matters: K5 named-ratio target is only 0.2,
|
|
so most files are reached by folder, not by name.
|
|
5. Source URI — does the file cite any learn/docs.microsoft.com URL.
|
|
6. Header keys — variant count (prose-header fragmentation; 0 use YAML).
|
|
|
|
Pairs with the Spor D scorer (score-skill.mjs), which covers SKILL.md authoring
|
|
quality + structural ref hygiene but NOT substantive content correctness.
|
|
|
|
Usage: python3 scripts/kb-eval/ref-file-audit.py
|
|
Zero dependencies, no network, makes no changes.
|
|
"""
|
|
import os, re, glob, collections, statistics
|
|
|
|
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
|
ref_files = sorted(glob.glob(os.path.join(ROOT, "skills/*/references/**/*.md"), recursive=True))
|
|
|
|
# Concatenate every routing hub (SKILL.md + agents) to test reachability.
|
|
hub_text = ""
|
|
for p in (glob.glob(os.path.join(ROOT, "skills/*/SKILL.md"))
|
|
+ glob.glob(os.path.join(ROOT, "agents/*.md"))):
|
|
with open(p, encoding="utf-8") as f:
|
|
hub_text += f.read() + "\n"
|
|
|
|
DAY_RE = re.compile(r"20\d\d-\d\d-\d\d")
|
|
MONTH_RE = re.compile(r"\*\*(?:Last updated|Sist oppdatert|Dato|Oppdatert):\*\*\s*20\d\d-\d\d(?!-)")
|
|
DATE_KEY = re.compile(r"\*\*(?:Last updated|Sist oppdatert|Dato|Oppdatert):\*\*")
|
|
URL_RE = re.compile(r"https?://(?:learn|docs)\.microsoft\.com[^\s)\"']*")
|
|
HDR_KEY_RE = re.compile(r"^\*\*([^:*]+):\*\*", re.M)
|
|
|
|
per_skill = collections.Counter()
|
|
day_prec, month_only, truly_dateless = [], [], []
|
|
named, folder_only, true_orphan = [], [], []
|
|
no_source, sizes = [], []
|
|
header_keys = collections.Counter()
|
|
|
|
for p in ref_files:
|
|
rel = os.path.relpath(p, ROOT)
|
|
per_skill[rel.split("/")[1]] += 1
|
|
with open(p, encoding="utf-8") as f:
|
|
txt = f.read()
|
|
head = "\n".join(txt.splitlines()[:10])
|
|
sizes.append((txt.rstrip().count("\n") + 1, rel))
|
|
|
|
if DAY_RE.search(head) and DATE_KEY.search(head):
|
|
day_prec.append(rel)
|
|
elif MONTH_RE.search(head):
|
|
month_only.append(rel)
|
|
else:
|
|
truly_dateless.append(rel)
|
|
|
|
if not URL_RE.search(txt):
|
|
no_source.append(rel)
|
|
|
|
for k in HDR_KEY_RE.findall(head):
|
|
header_keys[k.strip()] += 1
|
|
|
|
base = os.path.basename(p)
|
|
folder = os.path.basename(os.path.dirname(p))
|
|
if base in hub_text:
|
|
named.append(rel)
|
|
elif ("references/" + folder) in hub_text or ("/" + folder + "/") in hub_text:
|
|
folder_only.append(rel)
|
|
else:
|
|
true_orphan.append(rel)
|
|
|
|
ln = [s for s, _ in sizes]
|
|
print(f"TOTAL: {len(ref_files)} reference files")
|
|
print(" per skill:", dict(per_skill))
|
|
print()
|
|
print("[SIZE] lines")
|
|
print(f" min={min(ln)} median={int(statistics.median(ln))} mean={int(statistics.mean(ln))} max={max(ln)}")
|
|
b = collections.Counter()
|
|
for n in ln:
|
|
b["0-100" if n <= 100 else "101-300" if n <= 300 else "301-500" if n <= 500 else "500+"] += 1
|
|
for k in ["0-100", "101-300", "301-500", "500+"]:
|
|
print(f" {k:8s}: {b[k]}")
|
|
print(" biggest 10:")
|
|
for n, r in sorted(sizes, reverse=True)[:10]:
|
|
print(f" {n:5d} {r}")
|
|
print()
|
|
print("[DATE] header precision")
|
|
print(f" day-precise (YYYY-MM-DD): {len(day_prec)}")
|
|
print(f" month-only (YYYY-MM): {len(month_only)} (deliberate convention)")
|
|
print(f" TRULY DATELESS: {len(truly_dateless)}")
|
|
for r in truly_dateless:
|
|
print(" -", r)
|
|
print()
|
|
print(f"[REACHABILITY] named={len(named)} via-folder={len(folder_only)} true-orphans={len(true_orphan)}")
|
|
for r in true_orphan:
|
|
print(" orphan:", r)
|
|
print()
|
|
print(f"[SOURCE] no learn/docs.microsoft.com URL: {len(no_source)} ({100*len(no_source)//len(ref_files)}%)")
|
|
print()
|
|
print(f"[HEADER KEYS] {len(header_keys)} distinct variants (0 files use YAML frontmatter)")
|
|
for k, c in header_keys.most_common(12):
|
|
print(f" {c:4d} **{k}:**")
|