"""Shape census of the pinned OKF corpus: which surface form each key is written in."""
import os, re, sys, collections
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
from llm_ingestion_guard import okf

ROOT = os.path.join(os.path.dirname(__file__), "corpus", "okf", "bundles")

def docs():
    out = []
    for dirpath, _d, fns in os.walk(ROOT):
        for fn in sorted(fns):
            if fn.endswith(".md") and fn not in ("index.md", "log.md"):
                out.append(os.path.join(dirpath, fn))
    return sorted(out)

def split_fm(text):
    lines = text.split("\n")
    if not lines or lines[0].strip() != "---":
        return None, None
    for i in range(1, len(lines)):
        if lines[i].strip() == "---":
            return lines[1:i], lines[i + 1:]
    return None, None

def key_shape(fm, key):
    for i, raw in enumerate(fm):
        m = re.match(r"^(%s):(.*)$" % re.escape(key), raw)
        if not m:
            continue
        val = m.group(2).strip()
        follow = fm[i + 1] if i + 1 < len(fm) else ""
        if val.startswith("["):
            return "flow-sequence"
        if val.startswith("{"):
            return "flow-mapping"
        if val:
            if follow[:1] in (" ", "\t") and not follow.strip().startswith("- "):
                return "scalar+continuation"
            return "single-line-scalar"
        if follow.strip().startswith("- "):
            return "block-seq-indented" if follow[:1] in (" ", "\t") else "block-seq-flush"
        if follow[:1] in (" ", "\t"):
            return "block-mapping"
        return "empty"
    return "ABSENT"

def main():
    paths = docs()
    print("denominator: %d" % len(paths))
    for key in ("tags", "description", "generated", "verified", "sources"):
        c = collections.Counter()
        for p in paths:
            fm, _ = split_fm(open(p, encoding="utf-8").read())
            c[key_shape(fm, key)] += 1
        print("\n%s:" % key)
        for shape, n in c.most_common():
            print("  %-22s %d/%d" % (shape, n, len(paths)))

main()