"""The append-only store for logg.jsonl, beslutninger.jsonl and varianter.jsonl. Append-only is the whole design, and it has a consequence people usually discover too late: the schema is irreversible. There is no migration that can go back and add a field to lines already written, so `type` and `skjema` are on the very first line this module ever writes (risk H10). A reader that meets a line without them is meeting a file this module did not produce. Two writers, one file (risk H4). The CLI and the host MCP server can both be appending while the operator watches, so every record goes out as a single ``os.write`` on a descriptor opened ``O_APPEND``. That is what makes the append indivisible; building the line first and writing it in one call is not a style choice here, it is the guarantee. Anything that split the record into two writes -- a buffered file object, a separate newline write -- would let a second writer land between them. Torn writes versus corruption (risk M5r). A process killed mid-append leaves a partial last line with no newline after it. That is recoverable and expected, so :func:`read_lines` skips it. A malformed line anywhere else is corruption, not a torn write, and it raises with its line number rather than being silently dropped -- the difference between "the last append did not finish" and "a record in the middle of your decision log is gone" is the difference this module refuses to blur. Timestamps carry an explicit UTC offset, always. Norwegian local time repeats 02:30 on the last Sunday of October, so two records an hour apart share a wall clock and are told apart only by the offset. A naive timestamp is rejected on write, because by read time there is no way to recover which hour was meant. """ import datetime import json import os from . import paths #: The closed set of record kinds. `beslutning` is a sourcing decision, #: `korrigering` restates one that was wrong, `utfall` records how a case #: ended. Anything else is a caller bug, not a new kind. TYPES = ("beslutning", "korrigering", "utfall") #: Schema version stamped on every line. Bumping this is a fork of the format, #: never an in-place migration -- the file behind it cannot be rewritten. SKJEMA = 1 #: Keys whose values are timestamps, wherever they appear. `ts` is the case #: log (build-brief 5.3), `dato` the decision log (5.4). TIMESTAMP_KEYS = ("ts", "dato") #: Fields belonging to the correction record itself, not to the decision it #: corrects. Folding must not smear these onto the target. _CORRECTION_OWN_FIELDS = ("type", "id", "korrigerer", "skjema") class JsonlError(Exception): """A refusal from this module, with the line number when there is one.""" def __init__(self, message, line=None): self.line = line super().__init__("line %d: %s" % (line, message) if line else message) def append_line(path, obj, root=None): """Append ``obj`` as exactly one line, in a single ``O_APPEND`` write. Validation happens before the file is opened, so a rejected record leaves no trace at all -- not an empty file, not a partial line. ``root``, when given, resolves ``path`` under it with :func:`paths.safe_join`, so a caller-supplied path that escapes the workspace is refused here rather than in whichever script remembered to check. """ record = _validated(obj) target = paths.safe_join(root, path) if root is not None else path payload = (json.dumps(record, ensure_ascii=False) + "\n").encode("utf-8") handle = os.open(target, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600) try: written = os.write(handle, payload) finally: os.close(handle) if written != len(payload): # Retrying the remainder would append a fragment behind whatever # another writer put there in the meantime. Fail instead. raise JsonlError( "short write: %d of %d bytes reached %r" % (written, len(payload), target) ) return record def read_lines(path): """Read every record, skipping only a torn trailing line. A missing file reads as no records -- an append-only log that has never been appended to is empty, not broken. """ if not os.path.exists(path): return [] with open(path, "r", encoding="utf-8") as handle: data = handle.read() if data == "": return [] lines = data.split("\n") # A file ending in a newline leaves an empty final element; a file that # does not was cut off mid-append, and only THAT last line may be skipped. if lines and lines[-1] == "": lines.pop() torn_index = None else: torn_index = len(lines) records = [] for index, line in enumerate(lines, start=1): if line.strip() == "": continue try: record = json.loads(line) except ValueError as exc: if index == torn_index: continue raise JsonlError("not valid JSON: %s" % exc, line=index) if not isinstance(record, dict): if index == torn_index: continue raise JsonlError("expected an object, got %s" % type(record).__name__, line=index) records.append(record) return records def fold(lines): """Return the decision stream with corrections applied and outcomes removed. A `korrigering` overwrites the fields it names on the `beslutning` it points at, and contributes no row of its own -- the log keeps both, the stream shows the corrected decision. An `utfall` is not a decision and is excluded; build-brief 5.7 reads outcomes on its own terms. The input is never mutated: the caller's records came off disk and stay as they were read. """ stream = [] by_id = {} for record in lines: kind = record.get("type") if kind == "beslutning": copy = dict(record) stream.append(copy) if "id" in copy: by_id[copy["id"]] = copy elif kind == "korrigering": target_id = record.get("korrigerer") if target_id not in by_id: raise JsonlError( "correction %r names %r, which is not a decision above it" % (record.get("id"), target_id) ) by_id[target_id].update( {k: v for k, v in record.items() if k not in _CORRECTION_OWN_FIELDS} ) elif kind == "utfall": continue else: raise JsonlError("unknown record type %r" % (kind,)) return stream def instant(record): """The record's timestamp as an aware :class:`datetime.datetime`.""" for key in TIMESTAMP_KEYS: if key in record: return _aware(record[key], key) raise JsonlError("record has no timestamp; expected one of %r" % (TIMESTAMP_KEYS,)) def by_time(lines): """Sort records by the instant they name, not by the string that spells it.""" return sorted(lines, key=instant) def _validated(obj): if not isinstance(obj, dict): raise JsonlError("a record must be an object, got %s" % type(obj).__name__) kind = obj.get("type") if kind not in TYPES: raise JsonlError("type %r is not one of %r" % (kind, TYPES)) record = dict(obj) record.setdefault("skjema", SKJEMA) for key in TIMESTAMP_KEYS: if key in record: _aware(record[key], key) return record def _aware(value, key): try: moment = datetime.datetime.fromisoformat(value) except (TypeError, ValueError): raise JsonlError("%s=%r is not an ISO-8601 timestamp" % (key, value)) if moment.tzinfo is None or moment.utcoffset() is None: raise JsonlError( "%s=%r has no UTC offset; a naive timestamp cannot be ordered " "across the October hour that happens twice" % (key, value) ) return moment