feat(m1): add strict frontmatter parser with documented grammar

scripts/jobbsok_lib/frontmatter.py

@@ -0,0 +1,286 @@
+"""A strict, dependency-free frontmatter reader for kandidat.md and sak.md.
+
+No YAML runtime is allowed here (risk H7), so this module hand-rolls a parser
+that sits directly on the hard-filter path: `lonn.gulv_nok` and
+`geografi.maks_reisetid_min` are read from here and compared numerically by
+scoring. A hand-rolled parser on that path is only safe if it is loud, so the
+one rule this module never breaks is that anything outside the grammar below
+raises :class:`FrontmatterError` carrying the offending line number. It never
+guesses, never applies a default, and never skips a line it did not understand.
+
+Grammar
+-------
+
+The whole accepted grammar, and nothing beyond it::
+
+    document   := [ "---" NL entries "---" NL ] body
+    entry      := comment | blank | mapping | block
+    comment    := "#" ...          -- only when "#" is the FIRST column
+    mapping    := KEY ":" SP scalar
+    block      := KEY ":" NL ( child | item )+
+    child      := "  " KEY ":" SP scalar        -- one level of nesting, no more
+    item       := "  - " scalar                 -- a flat list, scalars only
+    KEY        := [A-Za-z_][A-Za-z0-9_]*
+    scalar     := integer | '"' text '"' | "'" text "'" | unquoted-text
+
+Indentation is exactly zero or two spaces; a tab in the indentation is an
+error rather than a width the reader has to assume. A `#` anywhere but the
+first column is data -- `notat: rolle nr # 3` keeps its hash, because trimming
+it would silently rewrite what the operator wrote. Inside double quotes only
+``\\"`` and ``\\\\`` are escapes; single quotes take the text verbatim.
+A duplicate key is refused instead of resolving last-wins: two salary floors
+in one profile must stop the run, not pick one.
+
+A document with no opening fence has no frontmatter -- that is an empty
+mapping and an untouched body, not an error. Rejecting a kandidat.md that
+lacks frontmatter is the section 5.1 contract check's job, not the reader's.
+
+Preservation
+------------
+
+:func:`render` writes values back exactly as they came in: no Unicode
+normalisation, no escaping of non-ASCII, no reordering. `paths.slug`
+normalises deliberately; this module must not, because the file on disk is
+the operator's own text and a round trip through here has to return the same
+bytes.
+"""
+
+import re
+
+from . import paths
+
+#: Keys are conservative identifiers: the frontmatter is a machine contract,
+#: not free text, and a key with a space in it is a typo worth stopping for.
+KEY_RE = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*):(.*)$")
+
+INTEGER_RE = re.compile(r"^-?\d+$")
+
+FENCE = "---"
+
+INDENT = "  "
+
+
+class FrontmatterError(Exception):
+    """A line outside the grammar, reported with the line it was on.
+
+    ``line`` is 1-based and counts from the start of the document, so it
+    matches what an editor shows.
+    """
+
+    def __init__(self, line, message):
+        self.line = line
+        super().__init__("line %d: %s" % (line, message))
+
+
+def parse(text):
+    """Parse ``text`` into ``(metadata, body)``.
+
+    ``metadata`` preserves key order. ``body`` is everything after the closing
+    fence, byte for byte. A document without an opening fence returns
+    ``({}, text)``.
+    """
+    if not text.startswith(FENCE + "\n") and text.rstrip("\n") != FENCE:
+        return {}, text
+
+    lines = text.split("\n")
+    closing = _find_closing_fence(lines)
+    body = "\n".join(lines[closing + 1:])
+
+    return _parse_entries(lines[1:closing], first_lineno=2), body
+
+
+def render(metadata, body=""):
+    """Render ``metadata`` and ``body`` back into a document.
+
+    The inverse of :func:`parse` for any document written in the canonical
+    form this function emits. Values are written as they are held: Norwegian
+    characters go out as themselves, never escaped and never normalised.
+    """
+    out = [FENCE]
+    for key, value in metadata.items():
+        if isinstance(value, dict):
+            out.append("%s:" % key)
+            out.extend("%s%s: %s" % (INDENT, k, _scalar_out(v)) for k, v in value.items())
+        elif isinstance(value, list):
+            out.append("%s:" % key)
+            out.extend("%s- %s" % (INDENT, _scalar_out(item)) for item in value)
+        else:
+            out.append("%s: %s" % (key, _scalar_out(value)))
+    out.append(FENCE)
+    return "\n".join(out) + "\n" + body
+
+
+def read(root, *parts):
+    """Read and parse a file resolved under ``root`` via :func:`paths.safe_join`.
+
+    Going through ``safe_join`` rather than ``os.path.join`` means a caller
+    that hands in a traversing path gets a refusal here too, not only in the
+    scripts that remembered to check.
+    """
+    target = paths.safe_join(root, *parts)
+    with open(target, "r", encoding="utf-8") as handle:
+        return parse(handle.read())
+
+
+def _find_closing_fence(lines):
+    for index in range(1, len(lines)):
+        if lines[index] == FENCE:
+            return index
+    raise FrontmatterError(
+        1, "frontmatter fence opened here was never closed with %r" % FENCE
+    )
+
+
+def _parse_entries(lines, first_lineno):
+    metadata = {}
+    open_key = None
+    open_lineno = None
+    block = None  # None until the first child decides mapping or list
+
+    for offset, line in enumerate(lines):
+        lineno = first_lineno + offset
+
+        if line.strip() == "":
+            continue
+        if line.startswith("#"):
+            continue
+
+        indent = _indent_width(line, lineno)
+        content = line[indent:]
+
+        if indent == 0:
+            if open_key is not None and block is None:
+                raise _empty_block(open_key, open_lineno)
+            open_key, open_lineno, block = _top_level(metadata, content, lineno)
+        else:
+            if open_key is None:
+                raise FrontmatterError(
+                    lineno, "indented line has no parent key above it"
+                )
+            block = _child(metadata, open_key, block, content, lineno)
+
+    if open_key is not None and block is None:
+        raise _empty_block(open_key, open_lineno)
+    return metadata
+
+
+def _top_level(metadata, content, lineno):
+    if content.startswith("- "):
+        raise FrontmatterError(
+            lineno, "list item at column 0 has no key to belong to"
+        )
+    match = KEY_RE.match(content)
+    if match is None:
+        raise FrontmatterError(
+            lineno, "expected %r, got %r" % ("key: value", content)
+        )
+    key, rest = match.group(1), match.group(2)
+    if key in metadata:
+        raise FrontmatterError(lineno, "duplicate key %r" % key)
+
+    if rest.strip() == "":
+        return key, lineno, None
+    metadata[key] = _scalar_in(rest.strip(), lineno)
+    return None, None, None
+
+
+def _child(metadata, open_key, block, content, lineno):
+    if content.startswith("- "):
+        if block == "mapping":
+            raise FrontmatterError(
+                lineno, "list item under %r, which already holds a mapping" % open_key
+            )
+        metadata.setdefault(open_key, [])
+        metadata[open_key].append(_scalar_in(content[2:].strip(), lineno))
+        return "list"
+
+    match = KEY_RE.match(content)
+    if match is None:
+        raise FrontmatterError(
+            lineno, "expected %r or %r, got %r" % ("  key: value", "  - item", content)
+        )
+    if block == "list":
+        raise FrontmatterError(
+            lineno, "mapping entry under %r, which already holds a list" % open_key
+        )
+    key, rest = match.group(1), match.group(2)
+    if rest.strip() == "":
+        raise FrontmatterError(
+            lineno, "%r nests deeper than one level, which the grammar does not allow" % key
+        )
+    nested = metadata.setdefault(open_key, {})
+    if key in nested:
+        raise FrontmatterError(lineno, "duplicate key %r under %r" % (key, open_key))
+    nested[key] = _scalar_in(rest.strip(), lineno)
+    return "mapping"
+
+
+def _indent_width(line, lineno):
+    leading = line[: len(line) - len(line.lstrip(" \t"))]
+    if "\t" in leading:
+        raise FrontmatterError(
+            lineno, "tab in the indentation; use exactly two spaces per level"
+        )
+    if len(leading) not in (0, 2):
+        raise FrontmatterError(
+            lineno,
+            "indentation is %d spaces; the grammar allows 0 or 2" % len(leading),
+        )
+    return len(leading)
+
+
+def _empty_block(key, lineno):
+    return FrontmatterError(
+        lineno, "%r has no value and no indented block beneath it" % key
+    )
+
+
+def _scalar_in(raw, lineno):
+    if raw.startswith('"'):
+        if len(raw) < 2 or not raw.endswith('"'):
+            raise FrontmatterError(lineno, "unterminated double-quoted value")
+        return _unescape(raw[1:-1])
+    if raw.startswith("'"):
+        if len(raw) < 2 or not raw.endswith("'"):
+            raise FrontmatterError(lineno, "unterminated single-quoted value")
+        return raw[1:-1]
+    if INTEGER_RE.match(raw):
+        return int(raw)
+    return raw
+
+
+def _unescape(text):
+    out = []
+    index = 0
+    while index < len(text):
+        char = text[index]
+        if char == "\\" and index + 1 < len(text) and text[index + 1] in '"\\':
+            out.append(text[index + 1])
+            index += 2
+            continue
+        out.append(char)
+        index += 1
+    return "".join(out)
+
+
+def _scalar_out(value):
+    if isinstance(value, bool):
+        # Booleans are not in the grammar; letting one through here would
+        # write a value the reader cannot read back.
+        raise ValueError("the grammar has no boolean; write the word you mean")
+    if isinstance(value, int):
+        return str(value)
+    text = str(value)
+    if _needs_quoting(text):
+        return '"%s"' % text.replace("\\", "\\\\").replace('"', '\\"')
+    return text
+
+
+def _needs_quoting(text):
+    if text == "" or text != text.strip():
+        return True
+    if text[0] in "#\"'":
+        return True
+    # An unquoted digit string would come back as an int and stop being the
+    # string it was written as.
+    return bool(INTEGER_RE.match(text))

tests/test_frontmatter.py

@@ -0,0 +1,127 @@
+"""The strict frontmatter grammar for kandidat.md and sak.md (plan Step 4).
+
+Risk H7: this is a hand-rolled parser sitting on the hard-filter path. A YAML
+runtime is not allowed, so the grammar has to be small enough to hold in the
+head and loud enough that nothing it does not understand slips past as a
+default. Every test below is therefore about a refusal or about preservation:
+what the parser accepts, and what it must never quietly swallow.
+
+The preservation half matters as much as the refusal half. `geografi.base` is
+a Norwegian place name; if a round trip through the parser normalises it from
+NFC to NFD or escapes it to an ASCII sequence, the file on disk stops being
+the file the operator wrote. paths.slug normalises on purpose -- this module
+must not.
+
+Style note: this file follows tests/test_paths.py.
+"""
+
+import pytest
+
+from jobbsok_lib import frontmatter
+
+
+ROUND_TRIP = """---
+oppdatert: 2026-09-15
+geografi:
+  base: Tromsø
+  maks_reisetid_min: 45
+absolutte_nei:
+  - Ærøy-prosjektet
+  - nattskift på Målselv
+---
+Kjerne: rådgiver i Ålesund.
+"""
+
+
+def test_nested_mapping_and_scalar_types():
+    meta, body = frontmatter.parse(
+        "---\n"
+        "oppdatert: 2026-09-15\n"
+        "geografi:\n"
+        "  base: Bergen\n"
+        "  maks_reisetid_min: 45\n"
+        "  vurderer_flytting: nei\n"
+        "---\n"
+        "brodtekst\n"
+    )
+    assert meta["oppdatert"] == "2026-09-15"
+    assert meta["geografi"] == {
+        "base": "Bergen",
+        "maks_reisetid_min": 45,
+        "vurderer_flytting": "nei",
+    }
+    # An integer arrives as an int, not as the string that spelled it: the
+    # scoring path compares it numerically.
+    assert isinstance(meta["geografi"]["maks_reisetid_min"], int)
+    assert body == "brodtekst\n"
+
+
+def test_flat_list_of_quoted_and_unquoted_items():
+    meta, _ = frontmatter.parse(
+        "---\n"
+        "absolutte_nei:\n"
+        "  - salg\n"
+        '  - "turnus, med helg"\n'
+        "  - 3\n"
+        "---\n"
+    )
+    assert meta["absolutte_nei"] == ["salg", "turnus, med helg", 3]
+
+
+def test_empty_document_has_no_frontmatter_and_is_not_an_error():
+    # Absent frontmatter is not this module's error to raise -- Step 8's
+    # contract check is where a kandidat.md without frontmatter is rejected.
+    # Here it is simply an empty mapping and an untouched body.
+    assert frontmatter.parse("") == ({}, "")
+    assert frontmatter.parse("bare brodtekst\n") == ({}, "bare brodtekst\n")
+
+
+def test_hash_is_a_comment_only_at_line_start():
+    meta, _ = frontmatter.parse(
+        "---\n"
+        "# denne linjen er en kommentar\n"
+        "notat: rolle nr # 3, ikke nr 4\n"
+        "---\n"
+    )
+    assert "#" not in meta
+    # The hash inside the value is data. Trimming it would silently rewrite
+    # what the operator wrote, which is the failure mode this test pins.
+    assert meta["notat"] == "rolle nr # 3, ikke nr 4"
+
+
+def test_unterminated_fence_raises_with_the_line_number():
+    with pytest.raises(frontmatter.FrontmatterError) as caught:
+        frontmatter.parse("---\noppdatert: 2026-09-15\ngeografi:\n  base: Bodø\n")
+    assert caught.value.line == 1
+    assert "---" in str(caught.value)
+
+
+def test_tab_indentation_is_refused():
+    with pytest.raises(frontmatter.FrontmatterError) as caught:
+        frontmatter.parse("---\ngeografi:\n\tbase: Bodø\n---\n")
+    assert caught.value.line == 3
+    assert "tab" in str(caught.value).lower()
+
+
+def test_duplicate_key_is_refused_rather_than_last_wins():
+    # Last-wins is the dangerous default: a profile with two salary floors
+    # would silently score against whichever came last.
+    with pytest.raises(frontmatter.FrontmatterError) as caught:
+        frontmatter.parse("---\nlonn:\n  gulv_nok: 750000\n  gulv_nok: 650000\n---\n")
+    assert caught.value.line == 4
+
+
+def test_norwegian_characters_survive_a_round_trip_byte_identically():
+    meta, body = frontmatter.parse(ROUND_TRIP)
+    assert meta["geografi"]["base"] == "Tromsø"
+    assert meta["absolutte_nei"][1] == "nattskift på Målselv"
+    rendered = frontmatter.render(meta, body)
+    assert rendered == ROUND_TRIP
+    assert rendered.encode("utf-8") == ROUND_TRIP.encode("utf-8")
+
+
+def test_a_line_outside_the_grammar_names_its_line_number():
+    with pytest.raises(frontmatter.FrontmatterError) as caught:
+        frontmatter.parse("---\noppdatert: 2026-09-15\ndette er ikke en nokkel\n---\n")
+    assert caught.value.line == 3
+    assert "3" in str(caught.value)