feat(m1): add strict frontmatter parser with documented grammar
This commit is contained in:
parent
8328d487a9
commit
a0874a5e17
2 changed files with 413 additions and 0 deletions
286
scripts/jobbsok_lib/frontmatter.py
Normal file
286
scripts/jobbsok_lib/frontmatter.py
Normal file
|
|
@ -0,0 +1,286 @@
|
|||
"""A strict, dependency-free frontmatter reader for kandidat.md and sak.md.
|
||||
|
||||
No YAML runtime is allowed here (risk H7), so this module hand-rolls a parser
|
||||
that sits directly on the hard-filter path: `lonn.gulv_nok` and
|
||||
`geografi.maks_reisetid_min` are read from here and compared numerically by
|
||||
scoring. A hand-rolled parser on that path is only safe if it is loud, so the
|
||||
one rule this module never breaks is that anything outside the grammar below
|
||||
raises :class:`FrontmatterError` carrying the offending line number. It never
|
||||
guesses, never applies a default, and never skips a line it did not understand.
|
||||
|
||||
Grammar
|
||||
-------
|
||||
|
||||
The whole accepted grammar, and nothing beyond it::
|
||||
|
||||
document := [ "---" NL entries "---" NL ] body
|
||||
entry := comment | blank | mapping | block
|
||||
comment := "#" ... -- only when "#" is the FIRST column
|
||||
mapping := KEY ":" SP scalar
|
||||
block := KEY ":" NL ( child | item )+
|
||||
child := " " KEY ":" SP scalar -- one level of nesting, no more
|
||||
item := " - " scalar -- a flat list, scalars only
|
||||
KEY := [A-Za-z_][A-Za-z0-9_]*
|
||||
scalar := integer | '"' text '"' | "'" text "'" | unquoted-text
|
||||
|
||||
Indentation is exactly zero or two spaces; a tab in the indentation is an
|
||||
error rather than a width the reader has to assume. A `#` anywhere but the
|
||||
first column is data -- `notat: rolle nr # 3` keeps its hash, because trimming
|
||||
it would silently rewrite what the operator wrote. Inside double quotes only
|
||||
``\\"`` and ``\\\\`` are escapes; single quotes take the text verbatim.
|
||||
A duplicate key is refused instead of resolving last-wins: two salary floors
|
||||
in one profile must stop the run, not pick one.
|
||||
|
||||
A document with no opening fence has no frontmatter -- that is an empty
|
||||
mapping and an untouched body, not an error. Rejecting a kandidat.md that
|
||||
lacks frontmatter is the section 5.1 contract check's job, not the reader's.
|
||||
|
||||
Preservation
|
||||
------------
|
||||
|
||||
:func:`render` writes values back exactly as they came in: no Unicode
|
||||
normalisation, no escaping of non-ASCII, no reordering. `paths.slug`
|
||||
normalises deliberately; this module must not, because the file on disk is
|
||||
the operator's own text and a round trip through here has to return the same
|
||||
bytes.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from . import paths
|
||||
|
||||
#: Keys are conservative identifiers: the frontmatter is a machine contract,
|
||||
#: not free text, and a key with a space in it is a typo worth stopping for.
|
||||
KEY_RE = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*):(.*)$")
|
||||
|
||||
INTEGER_RE = re.compile(r"^-?\d+$")
|
||||
|
||||
FENCE = "---"
|
||||
|
||||
INDENT = " "
|
||||
|
||||
|
||||
class FrontmatterError(Exception):
|
||||
"""A line outside the grammar, reported with the line it was on.
|
||||
|
||||
``line`` is 1-based and counts from the start of the document, so it
|
||||
matches what an editor shows.
|
||||
"""
|
||||
|
||||
def __init__(self, line, message):
|
||||
self.line = line
|
||||
super().__init__("line %d: %s" % (line, message))
|
||||
|
||||
|
||||
def parse(text):
|
||||
"""Parse ``text`` into ``(metadata, body)``.
|
||||
|
||||
``metadata`` preserves key order. ``body`` is everything after the closing
|
||||
fence, byte for byte. A document without an opening fence returns
|
||||
``({}, text)``.
|
||||
"""
|
||||
if not text.startswith(FENCE + "\n") and text.rstrip("\n") != FENCE:
|
||||
return {}, text
|
||||
|
||||
lines = text.split("\n")
|
||||
closing = _find_closing_fence(lines)
|
||||
body = "\n".join(lines[closing + 1:])
|
||||
|
||||
return _parse_entries(lines[1:closing], first_lineno=2), body
|
||||
|
||||
|
||||
def render(metadata, body=""):
|
||||
"""Render ``metadata`` and ``body`` back into a document.
|
||||
|
||||
The inverse of :func:`parse` for any document written in the canonical
|
||||
form this function emits. Values are written as they are held: Norwegian
|
||||
characters go out as themselves, never escaped and never normalised.
|
||||
"""
|
||||
out = [FENCE]
|
||||
for key, value in metadata.items():
|
||||
if isinstance(value, dict):
|
||||
out.append("%s:" % key)
|
||||
out.extend("%s%s: %s" % (INDENT, k, _scalar_out(v)) for k, v in value.items())
|
||||
elif isinstance(value, list):
|
||||
out.append("%s:" % key)
|
||||
out.extend("%s- %s" % (INDENT, _scalar_out(item)) for item in value)
|
||||
else:
|
||||
out.append("%s: %s" % (key, _scalar_out(value)))
|
||||
out.append(FENCE)
|
||||
return "\n".join(out) + "\n" + body
|
||||
|
||||
|
||||
def read(root, *parts):
|
||||
"""Read and parse a file resolved under ``root`` via :func:`paths.safe_join`.
|
||||
|
||||
Going through ``safe_join`` rather than ``os.path.join`` means a caller
|
||||
that hands in a traversing path gets a refusal here too, not only in the
|
||||
scripts that remembered to check.
|
||||
"""
|
||||
target = paths.safe_join(root, *parts)
|
||||
with open(target, "r", encoding="utf-8") as handle:
|
||||
return parse(handle.read())
|
||||
|
||||
|
||||
def _find_closing_fence(lines):
|
||||
for index in range(1, len(lines)):
|
||||
if lines[index] == FENCE:
|
||||
return index
|
||||
raise FrontmatterError(
|
||||
1, "frontmatter fence opened here was never closed with %r" % FENCE
|
||||
)
|
||||
|
||||
|
||||
def _parse_entries(lines, first_lineno):
|
||||
metadata = {}
|
||||
open_key = None
|
||||
open_lineno = None
|
||||
block = None # None until the first child decides mapping or list
|
||||
|
||||
for offset, line in enumerate(lines):
|
||||
lineno = first_lineno + offset
|
||||
|
||||
if line.strip() == "":
|
||||
continue
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
|
||||
indent = _indent_width(line, lineno)
|
||||
content = line[indent:]
|
||||
|
||||
if indent == 0:
|
||||
if open_key is not None and block is None:
|
||||
raise _empty_block(open_key, open_lineno)
|
||||
open_key, open_lineno, block = _top_level(metadata, content, lineno)
|
||||
else:
|
||||
if open_key is None:
|
||||
raise FrontmatterError(
|
||||
lineno, "indented line has no parent key above it"
|
||||
)
|
||||
block = _child(metadata, open_key, block, content, lineno)
|
||||
|
||||
if open_key is not None and block is None:
|
||||
raise _empty_block(open_key, open_lineno)
|
||||
return metadata
|
||||
|
||||
|
||||
def _top_level(metadata, content, lineno):
|
||||
if content.startswith("- "):
|
||||
raise FrontmatterError(
|
||||
lineno, "list item at column 0 has no key to belong to"
|
||||
)
|
||||
match = KEY_RE.match(content)
|
||||
if match is None:
|
||||
raise FrontmatterError(
|
||||
lineno, "expected %r, got %r" % ("key: value", content)
|
||||
)
|
||||
key, rest = match.group(1), match.group(2)
|
||||
if key in metadata:
|
||||
raise FrontmatterError(lineno, "duplicate key %r" % key)
|
||||
|
||||
if rest.strip() == "":
|
||||
return key, lineno, None
|
||||
metadata[key] = _scalar_in(rest.strip(), lineno)
|
||||
return None, None, None
|
||||
|
||||
|
||||
def _child(metadata, open_key, block, content, lineno):
|
||||
if content.startswith("- "):
|
||||
if block == "mapping":
|
||||
raise FrontmatterError(
|
||||
lineno, "list item under %r, which already holds a mapping" % open_key
|
||||
)
|
||||
metadata.setdefault(open_key, [])
|
||||
metadata[open_key].append(_scalar_in(content[2:].strip(), lineno))
|
||||
return "list"
|
||||
|
||||
match = KEY_RE.match(content)
|
||||
if match is None:
|
||||
raise FrontmatterError(
|
||||
lineno, "expected %r or %r, got %r" % (" key: value", " - item", content)
|
||||
)
|
||||
if block == "list":
|
||||
raise FrontmatterError(
|
||||
lineno, "mapping entry under %r, which already holds a list" % open_key
|
||||
)
|
||||
key, rest = match.group(1), match.group(2)
|
||||
if rest.strip() == "":
|
||||
raise FrontmatterError(
|
||||
lineno, "%r nests deeper than one level, which the grammar does not allow" % key
|
||||
)
|
||||
nested = metadata.setdefault(open_key, {})
|
||||
if key in nested:
|
||||
raise FrontmatterError(lineno, "duplicate key %r under %r" % (key, open_key))
|
||||
nested[key] = _scalar_in(rest.strip(), lineno)
|
||||
return "mapping"
|
||||
|
||||
|
||||
def _indent_width(line, lineno):
|
||||
leading = line[: len(line) - len(line.lstrip(" \t"))]
|
||||
if "\t" in leading:
|
||||
raise FrontmatterError(
|
||||
lineno, "tab in the indentation; use exactly two spaces per level"
|
||||
)
|
||||
if len(leading) not in (0, 2):
|
||||
raise FrontmatterError(
|
||||
lineno,
|
||||
"indentation is %d spaces; the grammar allows 0 or 2" % len(leading),
|
||||
)
|
||||
return len(leading)
|
||||
|
||||
|
||||
def _empty_block(key, lineno):
|
||||
return FrontmatterError(
|
||||
lineno, "%r has no value and no indented block beneath it" % key
|
||||
)
|
||||
|
||||
|
||||
def _scalar_in(raw, lineno):
|
||||
if raw.startswith('"'):
|
||||
if len(raw) < 2 or not raw.endswith('"'):
|
||||
raise FrontmatterError(lineno, "unterminated double-quoted value")
|
||||
return _unescape(raw[1:-1])
|
||||
if raw.startswith("'"):
|
||||
if len(raw) < 2 or not raw.endswith("'"):
|
||||
raise FrontmatterError(lineno, "unterminated single-quoted value")
|
||||
return raw[1:-1]
|
||||
if INTEGER_RE.match(raw):
|
||||
return int(raw)
|
||||
return raw
|
||||
|
||||
|
||||
def _unescape(text):
|
||||
out = []
|
||||
index = 0
|
||||
while index < len(text):
|
||||
char = text[index]
|
||||
if char == "\\" and index + 1 < len(text) and text[index + 1] in '"\\':
|
||||
out.append(text[index + 1])
|
||||
index += 2
|
||||
continue
|
||||
out.append(char)
|
||||
index += 1
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _scalar_out(value):
|
||||
if isinstance(value, bool):
|
||||
# Booleans are not in the grammar; letting one through here would
|
||||
# write a value the reader cannot read back.
|
||||
raise ValueError("the grammar has no boolean; write the word you mean")
|
||||
if isinstance(value, int):
|
||||
return str(value)
|
||||
text = str(value)
|
||||
if _needs_quoting(text):
|
||||
return '"%s"' % text.replace("\\", "\\\\").replace('"', '\\"')
|
||||
return text
|
||||
|
||||
|
||||
def _needs_quoting(text):
|
||||
if text == "" or text != text.strip():
|
||||
return True
|
||||
if text[0] in "#\"'":
|
||||
return True
|
||||
# An unquoted digit string would come back as an int and stop being the
|
||||
# string it was written as.
|
||||
return bool(INTEGER_RE.match(text))
|
||||
127
tests/test_frontmatter.py
Normal file
127
tests/test_frontmatter.py
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
"""The strict frontmatter grammar for kandidat.md and sak.md (plan Step 4).
|
||||
|
||||
Risk H7: this is a hand-rolled parser sitting on the hard-filter path. A YAML
|
||||
runtime is not allowed, so the grammar has to be small enough to hold in the
|
||||
head and loud enough that nothing it does not understand slips past as a
|
||||
default. Every test below is therefore about a refusal or about preservation:
|
||||
what the parser accepts, and what it must never quietly swallow.
|
||||
|
||||
The preservation half matters as much as the refusal half. `geografi.base` is
|
||||
a Norwegian place name; if a round trip through the parser normalises it from
|
||||
NFC to NFD or escapes it to an ASCII sequence, the file on disk stops being
|
||||
the file the operator wrote. paths.slug normalises on purpose -- this module
|
||||
must not.
|
||||
|
||||
Style note: this file follows tests/test_paths.py.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from jobbsok_lib import frontmatter
|
||||
|
||||
|
||||
ROUND_TRIP = """---
|
||||
oppdatert: 2026-09-15
|
||||
geografi:
|
||||
base: Tromsø
|
||||
maks_reisetid_min: 45
|
||||
absolutte_nei:
|
||||
- Ærøy-prosjektet
|
||||
- nattskift på Målselv
|
||||
---
|
||||
Kjerne: rådgiver i Ålesund.
|
||||
"""
|
||||
|
||||
|
||||
def test_nested_mapping_and_scalar_types():
|
||||
meta, body = frontmatter.parse(
|
||||
"---\n"
|
||||
"oppdatert: 2026-09-15\n"
|
||||
"geografi:\n"
|
||||
" base: Bergen\n"
|
||||
" maks_reisetid_min: 45\n"
|
||||
" vurderer_flytting: nei\n"
|
||||
"---\n"
|
||||
"brodtekst\n"
|
||||
)
|
||||
assert meta["oppdatert"] == "2026-09-15"
|
||||
assert meta["geografi"] == {
|
||||
"base": "Bergen",
|
||||
"maks_reisetid_min": 45,
|
||||
"vurderer_flytting": "nei",
|
||||
}
|
||||
# An integer arrives as an int, not as the string that spelled it: the
|
||||
# scoring path compares it numerically.
|
||||
assert isinstance(meta["geografi"]["maks_reisetid_min"], int)
|
||||
assert body == "brodtekst\n"
|
||||
|
||||
|
||||
def test_flat_list_of_quoted_and_unquoted_items():
|
||||
meta, _ = frontmatter.parse(
|
||||
"---\n"
|
||||
"absolutte_nei:\n"
|
||||
" - salg\n"
|
||||
' - "turnus, med helg"\n'
|
||||
" - 3\n"
|
||||
"---\n"
|
||||
)
|
||||
assert meta["absolutte_nei"] == ["salg", "turnus, med helg", 3]
|
||||
|
||||
|
||||
def test_empty_document_has_no_frontmatter_and_is_not_an_error():
|
||||
# Absent frontmatter is not this module's error to raise -- Step 8's
|
||||
# contract check is where a kandidat.md without frontmatter is rejected.
|
||||
# Here it is simply an empty mapping and an untouched body.
|
||||
assert frontmatter.parse("") == ({}, "")
|
||||
assert frontmatter.parse("bare brodtekst\n") == ({}, "bare brodtekst\n")
|
||||
|
||||
|
||||
def test_hash_is_a_comment_only_at_line_start():
|
||||
meta, _ = frontmatter.parse(
|
||||
"---\n"
|
||||
"# denne linjen er en kommentar\n"
|
||||
"notat: rolle nr # 3, ikke nr 4\n"
|
||||
"---\n"
|
||||
)
|
||||
assert "#" not in meta
|
||||
# The hash inside the value is data. Trimming it would silently rewrite
|
||||
# what the operator wrote, which is the failure mode this test pins.
|
||||
assert meta["notat"] == "rolle nr # 3, ikke nr 4"
|
||||
|
||||
|
||||
def test_unterminated_fence_raises_with_the_line_number():
|
||||
with pytest.raises(frontmatter.FrontmatterError) as caught:
|
||||
frontmatter.parse("---\noppdatert: 2026-09-15\ngeografi:\n base: Bodø\n")
|
||||
assert caught.value.line == 1
|
||||
assert "---" in str(caught.value)
|
||||
|
||||
|
||||
def test_tab_indentation_is_refused():
|
||||
with pytest.raises(frontmatter.FrontmatterError) as caught:
|
||||
frontmatter.parse("---\ngeografi:\n\tbase: Bodø\n---\n")
|
||||
assert caught.value.line == 3
|
||||
assert "tab" in str(caught.value).lower()
|
||||
|
||||
|
||||
def test_duplicate_key_is_refused_rather_than_last_wins():
|
||||
# Last-wins is the dangerous default: a profile with two salary floors
|
||||
# would silently score against whichever came last.
|
||||
with pytest.raises(frontmatter.FrontmatterError) as caught:
|
||||
frontmatter.parse("---\nlonn:\n gulv_nok: 750000\n gulv_nok: 650000\n---\n")
|
||||
assert caught.value.line == 4
|
||||
|
||||
|
||||
def test_norwegian_characters_survive_a_round_trip_byte_identically():
|
||||
meta, body = frontmatter.parse(ROUND_TRIP)
|
||||
assert meta["geografi"]["base"] == "Tromsø"
|
||||
assert meta["absolutte_nei"][1] == "nattskift på Målselv"
|
||||
rendered = frontmatter.render(meta, body)
|
||||
assert rendered == ROUND_TRIP
|
||||
assert rendered.encode("utf-8") == ROUND_TRIP.encode("utf-8")
|
||||
|
||||
|
||||
def test_a_line_outside_the_grammar_names_its_line_number():
|
||||
with pytest.raises(frontmatter.FrontmatterError) as caught:
|
||||
frontmatter.parse("---\noppdatert: 2026-09-15\ndette er ikke en nokkel\n---\n")
|
||||
assert caught.value.line == 3
|
||||
assert "3" in str(caught.value)
|
||||
Loading…
Add table
Add a link
Reference in a new issue