Appendix B, appendix C and the reference list are generated from the frozen parameter file, the pre-registration and the source bank. Citations link to their item in the list. The two-page summary states only what the summary in the report states. Draft PDFs of the report and the summary are in pdf/. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
668 lines
26 KiB
Python
668 lines
26 KiB
Python
# ruff: noqa: INP001
|
|
r"""Write the reference list of the report: the entries of the source bank it cites.
|
|
|
|
The report cites entries in the source bank (``kilder/*.md``) with ``\kilde{K03-020}``,
|
|
``\kilde{K03-044, K03-045}`` or ``\kilde{K27-007 til K27-011}``. This script collects
|
|
the cited ids from three places:
|
|
|
|
(a) every ``\kilde{...}`` in the hand-written ``rapport/*.tex`` (LaTeX comments are
|
|
skipped);
|
|
(b) every ``poster`` list in ``modell/parametere.json`` (the parameter appendix cites
|
|
them);
|
|
(c) every ``\kilde{...}`` in ``rapport/generert/vedlegg-*.tex``, when such files exist.
|
|
|
|
For each cited id it looks up the entry (``### K03-020 Title`` and its bullet fields)
|
|
and writes ``rapport/generert/referanser.tex``: a short lead-in with the counts, then
|
|
one item per entry, sorted by id in natural order, with the id as label (the look of
|
|
``\kilde``), a hyperlink target named ``kilde:<id>``, the entry's title and its
|
|
``Kilde`` field as the entry gives it. Nothing is paraphrased: the text is the
|
|
entry's own, escaped for LaTeX; URLs become ``\url{...}``, Markdown italics
|
|
``\emph{...}`` and code spans ``\fil{...}``.
|
|
|
|
The rule of the project: no reference is written in unless the entry's ``Kontroll``
|
|
line confirms it. The line is classified from what it says (see ``klasse``). An
|
|
entry whose class is not in ``TILLATT`` stops the script, unless its id stands in
|
|
``UNNTAK`` with the operator's decision.
|
|
|
|
Standard library only. Same input, same bytes. The script stops with an error, and
|
|
writes nothing, when a cited id has no entry, when a citation or an entry does not
|
|
have the expected form, when a cited entry is not confirmed, or when the text holds
|
|
a character or a Markdown construct the script does not know.
|
|
|
|
Run from anywhere: ``python3 rapport/lag_referanser.py``. With ``--sjekk`` it writes
|
|
nothing and prints everything it found: the ids per origin, every problem at once,
|
|
the classes of the Kontroll lines with the entries outside the plain classes and
|
|
where they are cited, and the ids the report mentions outside ``\kilde{}``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import sys
|
|
from collections import Counter
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
ROT = Path(__file__).resolve().parent.parent
|
|
KILDER = ROT / "kilder"
|
|
RAPPORT = ROT / "rapport"
|
|
GENERERT = RAPPORT / "generert"
|
|
PARAMETERE = ROT / "modell" / "parametere.json"
|
|
UT = GENERERT / "referanser.tex"
|
|
|
|
# Cited entries whose Kontroll class is not in TILLATT, let through by the operator's
|
|
# decision. Key: the id. Value: the date and the reason, as the operator gave them.
|
|
# Empty: no exception has been decided.
|
|
UNNTAK: dict[str, str] = {}
|
|
|
|
BEKREFTET = "bekreftet"
|
|
FORBEHOLD = "bekreftet med forbehold"
|
|
RETTET = "rettet"
|
|
LAGT_TIL = "lagt til av kontrolløren"
|
|
IKKE_BEKREFTET = "ikke bekreftet"
|
|
MANGLER = "mangler"
|
|
ANNET = "annen ordlyd"
|
|
KLASSER = (BEKREFTET, FORBEHOLD, RETTET, LAGT_TIL, IKKE_BEKREFTET, MANGLER, ANNET)
|
|
TILLATT = frozenset({BEKREFTET, FORBEHOLD, RETTET, LAGT_TIL})
|
|
|
|
ID = r"K\d{2}[a-z]?-[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*"
|
|
ER_ID = re.compile(ID)
|
|
I_TEKST = re.compile(r"(?<![A-Za-z0-9-])" + ID + r"(?![A-Za-z0-9])")
|
|
OMRAADE = re.compile(r"^K(\d{2})([a-z]?)-")
|
|
INTERVALL = re.compile(r"^(" + ID + r")\s+til\s+(" + ID + r")$")
|
|
NUMMERERT = re.compile(r"^(K\d{2}[a-z]?-)(\d+)$")
|
|
OVERSKRIFT = re.compile(r"^### (Ikke verifisert: )?(" + ID + r") (\S.*)$")
|
|
ID_OVERSKRIFT = re.compile(r"^#+ (?:Ikke verifisert: )?" + ID + r"\b")
|
|
FELT = re.compile(r"^- \*\*([^*]+?):\*\*(?: (.*))?$")
|
|
# A citation is \kilde{ with no space: a definition such as \RenewDocumentCommand \kilde { m }
|
|
# is not one.
|
|
KILDE_START = re.compile(r"\\kilde\{")
|
|
|
|
# A code span, a URL (it ends at white space, a guillemet, a backtick or an ellipsis) or
|
|
# Markdown italics (the star opens before a word and closes after one, as in CommonMark).
|
|
DEL = re.compile(
|
|
r"`(?P<kode>[^`]*)`"
|
|
r"|(?P<url>https?://[^\s\u00ab\u00bb`\u2026]+)"
|
|
r"|(?<![\w*])\*(?=[^\s*])(?P<kursiv>[^*]+?)(?<=[^\s*])\*(?![\w*])"
|
|
)
|
|
TRYGG_KODE = re.compile(r"^[A-Za-z0-9._/-]+$")
|
|
# Printable ASCII but the characters \url cannot take (" is a shorthand in babel norsk).
|
|
TRYGG_URL = re.compile(r"^[!#-\[\]_a-z|~]+$")
|
|
SLUTT_URL = ".,;:!?"
|
|
|
|
# LaTeX for the ASCII characters that need it in running text.
|
|
ASCII_TEX = {
|
|
"\\": r"\textbackslash{}",
|
|
"{": r"\{",
|
|
"}": r"\}",
|
|
"$": r"\$",
|
|
"&": r"\&",
|
|
"%": r"\%",
|
|
"#": r"\#",
|
|
"_": r"\_",
|
|
"^": r"\textasciicircum{}",
|
|
"~": r"\textasciitilde{}",
|
|
"<": r"\textless{}",
|
|
">": r"\textgreater{}",
|
|
"|": r"\textbar{}",
|
|
'"': r"\textquotedbl{}",
|
|
}
|
|
|
|
# Every non-ASCII character the entries hold, and how it is written. A letter the
|
|
# T1 encoding has is written as itself (inputenc reads it); the rest as a command.
|
|
# A character that is not here stops the script.
|
|
UNICODE_TEX = {
|
|
"\u00e5": "\u00e5", # a with ring
|
|
"\u00c5": "\u00c5",
|
|
"\u00f8": "\u00f8", # o with stroke
|
|
"\u00d8": "\u00d8",
|
|
"\u00e6": "\u00e6", # ae
|
|
"\u00c6": "\u00c6",
|
|
"\u00e4": "\u00e4", # a with diaeresis
|
|
"\u00c4": "\u00c4",
|
|
"\u00f6": "\u00f6", # o with diaeresis
|
|
"\u00d6": "\u00d6",
|
|
"\u00fc": "\u00fc", # u with diaeresis
|
|
"\u00e9": "\u00e9", # e with acute
|
|
"\u00c9": "\u00c9",
|
|
"\u00e8": "\u00e8", # e with grave
|
|
"\u00ea": "\u00ea", # e with circumflex
|
|
"\u00e7": "\u00e7", # c with cedilla
|
|
"\u00e0": "\u00e0", # a with grave
|
|
"\u00e1": "\u00e1", # a with acute
|
|
"\u00fb": "\u00fb", # u with circumflex
|
|
"\u00da": "\u00da", # U with acute
|
|
"\u00ed": "\u00ed", # i with acute
|
|
"\u00f3": "\u00f3", # o with acute
|
|
"\u00ab": r"\guillemotleft{}",
|
|
"\u00bb": r"\guillemotright{}",
|
|
"\u2013": r"\textendash{}",
|
|
"\u2014": r"\textemdash{}",
|
|
"\u2026": r"\textellipsis{}",
|
|
"\u2018": r"\textquoteleft{}",
|
|
"\u2019": r"\textquoteright{}",
|
|
"\u201c": r"\textquotedblleft{}",
|
|
"\u201d": r"\textquotedblright{}",
|
|
"\u00a7": r"\S{}",
|
|
"\u00b6": r"\P{}",
|
|
"\u2212": r"\textminus{}",
|
|
"\u2192": r"\textrightarrow{}",
|
|
"\u20ac": r"\texteuro{}",
|
|
"\u00a3": r"\pounds{}",
|
|
"\u00b3": r"\textthreesuperior{}",
|
|
"\u00b2": r"\texttwosuperior{}",
|
|
"\u00d7": r"\texttimes{}",
|
|
"\u00b0": r"\textdegree{}",
|
|
"\u00b1": r"\textpm{}",
|
|
"\u2265": r"\ensuremath{\geq}",
|
|
"\u2264": r"\ensuremath{\leq}",
|
|
"\u2248": r"\ensuremath{\approx}",
|
|
"\u2082": r"\textsubscript{2}",
|
|
"\u00a0": "~", # no-break space
|
|
}
|
|
|
|
LISTE = (
|
|
r"\begin{itemize}[label={},leftmargin=1.6em,itemindent=-1.6em,labelwidth=0pt,"
|
|
r"labelsep=0pt,align=left,itemsep=0.6pt plus 0.4pt,parsep=0pt,topsep=4pt]"
|
|
)
|
|
# URLs may break after any letter or digit too (with a penalty), so that a long
|
|
# path never runs into the margin. Set inside the group of the list only.
|
|
BRUDD = "".join(
|
|
rf"\do\{c}" for c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-"
|
|
)
|
|
|
|
|
|
class ReferanseError(Exception):
|
|
"""The input does not have the expected form."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Post:
|
|
"""One entry of the source bank, as far as the reference list needs it."""
|
|
|
|
id: str
|
|
tittel: str
|
|
kilde: str
|
|
kontroll: str | None
|
|
merket: bool # the heading says "Ikke verifisert:"
|
|
sted: str # file and line of the heading
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The source bank
|
|
|
|
|
|
def les_poster(navn: str, tekst: str, feil: list[str]) -> list[Post]:
|
|
"""Return the entries of one file of the source bank; append problems to ``feil``."""
|
|
linjer = tekst.splitlines()
|
|
poster: list[Post] = []
|
|
i = 0
|
|
while i < len(linjer):
|
|
m = OVERSKRIFT.match(linjer[i])
|
|
if not m:
|
|
if ID_OVERSKRIFT.match(linjer[i]):
|
|
feil.append(
|
|
f"{navn}:{i + 1}: heading with an id that is not an entry: {linjer[i]!r}"
|
|
)
|
|
i += 1
|
|
continue
|
|
sted = f"kilder/{navn}:{i + 1}"
|
|
felt: list[tuple[str, list[str]]] = []
|
|
i += 1
|
|
while i < len(linjer):
|
|
linje = linjer[i]
|
|
if linje.startswith("#"):
|
|
break
|
|
f = FELT.match(linje)
|
|
if f:
|
|
felt.append((f.group(1), [f.group(2) or ""]))
|
|
elif not linje.strip() or linje.startswith((" ", "\t")):
|
|
if felt:
|
|
felt[-1][1].append(linje.strip())
|
|
elif linje.strip():
|
|
feil.append(f"{sted}: indented line before the first field")
|
|
else:
|
|
break # text after the entry (a section heading in bold, a table)
|
|
i += 1
|
|
poster.append(lag_post(m, felt, sted, feil))
|
|
return poster
|
|
|
|
|
|
def lag_post(
|
|
m: re.Match[str], felt: list[tuple[str, list[str]]], sted: str, feil: list[str]
|
|
) -> Post:
|
|
"""Put one entry together from its heading and fields."""
|
|
kilder = [deler for navn, deler in felt if navn == "Kilde"]
|
|
kontroller = [deler for navn, deler in felt if navn == "Kontroll"]
|
|
kilde = ""
|
|
if len(kilder) != 1:
|
|
feil.append(f"{sted}: {m.group(2)} has {len(kilder)} Kilde fields, not one")
|
|
else:
|
|
deler = kilder[0]
|
|
while deler and not deler[-1]:
|
|
deler = deler[:-1]
|
|
if any(d.startswith(("- ", "* ", "|")) for d in deler[1:]):
|
|
feil.append(f"{sted}: {m.group(2)}: a list or table inside the Kilde field")
|
|
kilde = " ".join(d for d in deler if d)
|
|
if not kilde:
|
|
feil.append(f"{sted}: {m.group(2)}: empty Kilde field")
|
|
kontroll: str | None = None
|
|
if len(kontroller) > 1:
|
|
feil.append(f"{sted}: {m.group(2)} has {len(kontroller)} Kontroll fields")
|
|
elif kontroller:
|
|
kontroll = " ".join(d for d in kontroller[0] if d)
|
|
return Post(m.group(2), m.group(3).strip(), kilde, kontroll, bool(m.group(1)), sted)
|
|
|
|
|
|
def les_banken(feil: list[str]) -> dict[str, Post]:
|
|
"""Return every entry of the source bank by id."""
|
|
poster: dict[str, Post] = {}
|
|
filer = sorted(p for p in KILDER.glob("*.md") if p.name != "README.md")
|
|
if not filer:
|
|
msg = f"no files in {KILDER}"
|
|
raise ReferanseError(msg)
|
|
for fil in filer:
|
|
for post in les_poster(fil.name, fil.read_text(encoding="utf-8"), feil):
|
|
if post.id in poster:
|
|
feil.append(f"{post.sted}: {post.id} also stands at {poster[post.id].sted}")
|
|
continue
|
|
poster[post.id] = post
|
|
return poster
|
|
|
|
|
|
def klasse(post: Post) -> str:
|
|
"""Classify the entry's Kontroll line from what it says.
|
|
|
|
The line's first words decide: «ikke bekreftet»; «bekreftet som …» or
|
|
«bekreftet for …» (confirmed as a news item, as a rendering, for one edition
|
|
or some years: «bekreftet med forbehold»); «bekreftet» (also «bekreftet ved ny
|
|
kontroll»); «rettet»; «ny post …» or «lagt til …» (added by the controller,
|
|
who read the primary source). No line is «mangler»; anything else, or a
|
|
heading that still says «Ikke verifisert:», is «annen ordlyd».
|
|
"""
|
|
if post.kontroll is None:
|
|
return MANGLER
|
|
t = post.kontroll.strip()
|
|
if re.match(r"ikke bekreftet\b", t):
|
|
klasse_ = IKKE_BEKREFTET
|
|
elif re.match(r"bekreftet (som|for)\b", t):
|
|
klasse_ = FORBEHOLD
|
|
elif re.match(r"bekreftet\b", t):
|
|
klasse_ = BEKREFTET
|
|
elif re.match(r"rettet\b", t):
|
|
klasse_ = RETTET
|
|
elif re.match(r"(ny post|lagt til)\b", t):
|
|
klasse_ = LAGT_TIL
|
|
else:
|
|
klasse_ = ANNET
|
|
if post.merket and klasse_ in TILLATT:
|
|
return ANNET # the heading still says "Ikke verifisert:"
|
|
return klasse_
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Where the report cites
|
|
|
|
|
|
def uten_kommentarer(tekst: str) -> str:
|
|
"""Remove LaTeX comments, keeping the line breaks."""
|
|
return "\n".join(re.sub(r"(?<!\\)%.*", "", linje) for linje in tekst.split("\n"))
|
|
|
|
|
|
def utvid(del_: str) -> list[str] | None:
|
|
"""Return the ids of one part of a citation (an id, or 'A til B'), or None."""
|
|
if ER_ID.fullmatch(del_):
|
|
return [del_]
|
|
m = INTERVALL.match(del_)
|
|
if not m:
|
|
return None
|
|
a, b = NUMMERERT.match(m.group(1)), NUMMERERT.match(m.group(2))
|
|
if not a or not b or a.group(1) != b.group(1) or len(a.group(2)) != len(b.group(2)):
|
|
return None
|
|
fra, til = int(a.group(2)), int(b.group(2))
|
|
if fra >= til:
|
|
return None
|
|
bredde = len(a.group(2))
|
|
return [f"{a.group(1)}{n:0{bredde}d}" for n in range(fra, til + 1)]
|
|
|
|
|
|
def les_henvisninger(navn: str, tekst: str, feil: list[str]) -> list[tuple[str, str]]:
|
|
r"""Return (id, file:line) for every id inside a \kilde{...} of a .tex file."""
|
|
ren = uten_kommentarer(tekst)
|
|
funnet: list[tuple[str, str]] = []
|
|
for m in KILDE_START.finditer(ren):
|
|
linje = ren.count("\n", 0, m.start()) + 1
|
|
sted = f"{navn}:{linje}"
|
|
slutt = ren.find("}", m.end())
|
|
innhold = ren[m.end() : slutt] if slutt >= 0 else ""
|
|
if slutt < 0 or "{" in innhold or "\\" in innhold:
|
|
feil.append(f"{sted}: a \\kilde{{...}} that is not a plain list of ids")
|
|
continue
|
|
for del_ in innhold.split(","):
|
|
del_ = " ".join(del_.split()) # noqa: PLW2901
|
|
ider = utvid(del_)
|
|
if ider is None:
|
|
feil.append(f"{sted}: not an id or a range of ids: {del_!r}")
|
|
continue
|
|
funnet += [(i, sted) for i in ider]
|
|
return funnet
|
|
|
|
|
|
def les_parameterposter(data: object, sti: str = "") -> list[tuple[str, str]]:
|
|
"""Return (id, place) for every id in a ``poster`` list of the parameter file."""
|
|
funnet: list[tuple[str, str]] = []
|
|
if isinstance(data, dict):
|
|
navn = data.get("id")
|
|
sted = f"modell/parametere.json: {navn}" if isinstance(navn, str) else sti
|
|
for nokkel, verdi in data.items():
|
|
if nokkel == "poster":
|
|
if not isinstance(verdi, list) or not all(isinstance(v, str) for v in verdi):
|
|
msg = f"{sted or 'modell/parametere.json'}: poster is not a list of ids"
|
|
raise ReferanseError(msg)
|
|
funnet += [(v, sted or f"modell/parametere.json: {sti}") for v in verdi]
|
|
else:
|
|
funnet += les_parameterposter(verdi, f"{sti}/{nokkel}")
|
|
elif isinstance(data, list):
|
|
for n, verdi in enumerate(data):
|
|
funnet += les_parameterposter(verdi, f"{sti}[{n}]")
|
|
return funnet
|
|
|
|
|
|
def tex_filer() -> tuple[list[Path], list[Path]]:
|
|
"""Return the hand-written .tex files of the report and the generated appendix files."""
|
|
hand = sorted(RAPPORT.glob("*.tex"))
|
|
generert = sorted(GENERERT.glob("vedlegg-*.tex"))
|
|
return hand, generert
|
|
|
|
|
|
def les_alle_henvisninger(feil: list[str]) -> dict[str, list[tuple[str, str]]]:
|
|
"""Return the citations per origin: 'a' hand-written, 'b' parameter file, 'c' generated."""
|
|
hand, generert = tex_filer()
|
|
a: list[tuple[str, str]] = []
|
|
for fil in hand:
|
|
a += les_henvisninger(f"rapport/{fil.name}", fil.read_text(encoding="utf-8"), feil)
|
|
c: list[tuple[str, str]] = []
|
|
for fil in generert:
|
|
c += les_henvisninger(f"rapport/generert/{fil.name}", fil.read_text(encoding="utf-8"), feil)
|
|
b = les_parameterposter(json.loads(PARAMETERE.read_text(encoding="utf-8")))
|
|
for i, sted in b:
|
|
if not ER_ID.fullmatch(i):
|
|
feil.append(f"{sted}: not an id: {i!r}")
|
|
return {"a": a, "b": [(i, s) for i, s in b if ER_ID.fullmatch(i)], "c": c}
|
|
|
|
|
|
def utenfor_kilde() -> list[tuple[str, str]]:
|
|
r"""Return (id, file:line) for ids the hand-written report text holds outside \kilde{}."""
|
|
funnet: list[tuple[str, str]] = []
|
|
hand, _ = tex_filer()
|
|
for fil in hand:
|
|
ren = uten_kommentarer(fil.read_text(encoding="utf-8"))
|
|
ren = re.sub(r"\\kilde\{[^{}]*\}", lambda m: "\n" * m.group(0).count("\n"), ren)
|
|
for m in I_TEKST.finditer(ren):
|
|
linje = ren.count("\n", 0, m.start()) + 1
|
|
funnet.append((m.group(0), f"rapport/{fil.name}:{linje}"))
|
|
return funnet
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# LaTeX
|
|
|
|
|
|
def sorteringsnokkel(post_id: str) -> tuple[tuple[int, int | str], ...]:
|
|
"""Natural order: area, then the rest with numbers as numbers (001 before K01)."""
|
|
deler = re.split(r"(\d+)", post_id)
|
|
return tuple((0, int(d)) if d.isdigit() else (1, d) for d in deler if d)
|
|
|
|
|
|
def etter_ord(tekst: str, k: int) -> bool:
|
|
"""True when the character at k follows a word and no word follows it."""
|
|
return k > 0 and tekst[k - 1].isalnum() and not (k + 1 < len(tekst) and tekst[k + 1].isalnum())
|
|
|
|
|
|
def tex_tegn(tekst: str, sted: str) -> str:
|
|
"""Escape plain running text (no URLs, code spans or italics) for LaTeX."""
|
|
ut: list[str] = []
|
|
for k, tegn in enumerate(tekst):
|
|
if tegn in ASCII_TEX:
|
|
ut.append(ASCII_TEX[tegn])
|
|
elif tegn == "*" and etter_ord(tekst, k):
|
|
ut.append("*") # a mark after a word, like «1y*»: not Markdown
|
|
elif tegn in "*`":
|
|
msg = f"{sted}: a lone {tegn!r} (unpaired Markdown) in {tekst!r}"
|
|
raise ReferanseError(msg)
|
|
elif ord(tegn) < 128: # noqa: PLR2004
|
|
if not tegn.isprintable():
|
|
msg = f"{sted}: control character {tegn!r}"
|
|
raise ReferanseError(msg)
|
|
ut.append(tegn)
|
|
elif tegn in UNICODE_TEX:
|
|
ut.append(UNICODE_TEX[tegn])
|
|
else:
|
|
msg = f"{sted}: unknown character {tegn!r} (U+{ord(tegn):04X}); add it to UNICODE_TEX"
|
|
raise ReferanseError(msg)
|
|
tex = "".join(ut)
|
|
# Keep the T1 ligatures out: -- (dash), '' (quote), ,, (low quote).
|
|
for par in ("--", "''", ",,"):
|
|
while par in tex:
|
|
tex = tex.replace(par, par[0] + "{}" + par[1])
|
|
return tex
|
|
|
|
|
|
def tex_url(url: str, sted: str) -> str:
|
|
r"""A URL as \url{...}, with % and # escaped (hyperref removes the backslash).
|
|
|
|
A URL with a letter outside ASCII (``§36-4``, ``inngår``, a template such as
|
|
``ÅÅÅÅ-MM-DD``) cannot go through \url under pdflatex. It is written as
|
|
\href with the letters percent-encoded as UTF-8 in the link, and shown as it
|
|
stands in the entry, in the typewriter font of \url, with break points.
|
|
"""
|
|
if TRYGG_URL.match(url):
|
|
return r"\url{" + url.replace("%", r"\%").replace("#", r"\#") + "}"
|
|
ascii_ = "".join(c if ord(c) < 128 else quote(c, safe="") for c in url) # noqa: PLR2004
|
|
if not TRYGG_URL.match(ascii_):
|
|
msg = f"{sted}: URL with a character \\url cannot take: {url!r}"
|
|
raise ReferanseError(msg)
|
|
lenke = ascii_.replace("%", r"\%").replace("#", r"\#")
|
|
vist = "".join(tex_tegn(c, sted) + (r"\allowbreak{}" if c in "/.-=&?" else "") for c in url)
|
|
return r"\href{" + lenke + r"}{\texttt{" + vist + "}}"
|
|
|
|
|
|
def klipp_url(url: str) -> str:
|
|
"""Strip the punctuation that ends the sentence, not the URL."""
|
|
while True:
|
|
siste = url[-1]
|
|
aapen = {")": "(", "]": "["}.get(siste)
|
|
if siste in SLUTT_URL or (aapen and url.count(aapen) < url.count(siste)):
|
|
url = url[:-1]
|
|
else:
|
|
return url
|
|
|
|
|
|
def tex_tekst(tekst: str, sted: str, *, kursiv: bool = True) -> str:
|
|
"""Escape a field with Markdown code spans, URLs and italics for LaTeX."""
|
|
ut: list[str] = []
|
|
pos = 0
|
|
while True:
|
|
m = DEL.search(tekst, pos)
|
|
if not m:
|
|
ut.append(tex_tegn(tekst[pos:], sted))
|
|
return "".join(ut)
|
|
ut.append(tex_tegn(tekst[pos : m.start()], sted))
|
|
if m.group("kode") is not None:
|
|
kode = m.group("kode")
|
|
if not TRYGG_KODE.match(kode):
|
|
msg = f"{sted}: code span with characters \\fil cannot take: {kode!r}"
|
|
raise ReferanseError(msg)
|
|
ut.append(r"\fil{" + kode + "}")
|
|
pos = m.end()
|
|
elif m.group("url") is not None:
|
|
url = klipp_url(m.group("url"))
|
|
ut.append(tex_url(url, sted))
|
|
pos = m.start() + len(url)
|
|
else:
|
|
if not kursiv:
|
|
msg = f"{sted}: italics inside italics in {tekst!r}"
|
|
raise ReferanseError(msg)
|
|
ut.append(r"\emph{" + tex_tekst(m.group("kursiv"), sted, kursiv=False) + "}")
|
|
pos = m.end()
|
|
|
|
|
|
def tex_post(post: Post) -> str:
|
|
"""One item of the list: target, label, title, the Kilde field."""
|
|
tittel = tex_tekst(post.tittel, post.sted)
|
|
skille = " " if post.tittel.rstrip("*")[-1:] in ".?!" else ". "
|
|
return (
|
|
rf"\item \hypertarget{{kilde:{post.id}}}{{}}{{\color{{accent}}\textsf{{[{post.id}]}}}}"
|
|
rf"\enskip {tittel}{skille}{tex_tekst(post.kilde, post.sted)}"
|
|
)
|
|
|
|
|
|
def omraade(post_id: str) -> str:
|
|
"""The area of an id: its two digits (and letter, if any)."""
|
|
m = OMRAADE.match(post_id)
|
|
if not m:
|
|
msg = f"no area in {post_id!r}"
|
|
raise ReferanseError(msg)
|
|
return m.group(1) + m.group(2)
|
|
|
|
|
|
def lag(poster: list[Post], i_tekst: int, i_vedlegg: int, alle_omraader: int) -> str:
|
|
"""Return the LaTeX of the reference list."""
|
|
omraader = len({omraade(p.id) for p in poster})
|
|
innledning = (
|
|
rf"\noindent Lista har de {len(poster)} postene i kildebanken som rapporten viser til,"
|
|
rf" fra {omraader} av de {alle_omraader} områdene i \fil{{kilder/}}: {i_tekst} i teksten"
|
|
rf" og {i_vedlegg} i parametertabellen i \cref{{app:parametere}} (noen begge steder)."
|
|
r" Hver post står med tittel og med kildefeltet slik posten gir det: kilde, sted i kilden"
|
|
r" og lenke. Posten i \fil{kilder/} har i tillegg påstanden, tallene, et ordrett sitat og"
|
|
r" kontrollen mot primærkilden. Lista er laget av \fil{rapport/lag_referanser.py} og"
|
|
r" rettes ikke for hånd."
|
|
)
|
|
ut = [
|
|
"% Laget av rapport/lag_referanser.py fra kilder/*.md, rapport/*.tex og",
|
|
"% modell/parametere.json. Rettes ikke for hånd: kjør skriptet på nytt.",
|
|
innledning,
|
|
"",
|
|
r"\begingroup",
|
|
r"\fontsize{9}{10.5}\selectfont",
|
|
r"\raggedright",
|
|
r"\expandafter\def\expandafter\UrlBreaks\expandafter{\UrlBreaks" + BRUDD + "}",
|
|
LISTE,
|
|
]
|
|
ut += [tex_post(p) for p in poster]
|
|
ut += [r"\end{itemize}", r"\endgroup", ""]
|
|
return "\n".join(ut)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Running
|
|
|
|
|
|
@dataclass
|
|
class Funn:
|
|
"""Everything the script found, before anything is written."""
|
|
|
|
bank: dict[str, Post]
|
|
henvisninger: dict[str, list[tuple[str, str]]]
|
|
feil: list[str]
|
|
|
|
def steder(self) -> dict[str, list[str]]:
|
|
"""Where each cited id is cited, in the order found."""
|
|
steder: dict[str, list[str]] = {}
|
|
for opphav in ("a", "b", "c"):
|
|
for i, sted in self.henvisninger[opphav]:
|
|
steder.setdefault(i, []).append(sted)
|
|
return steder
|
|
|
|
|
|
def finn() -> Funn:
|
|
"""Read the source bank and the citations; collect every problem."""
|
|
feil: list[str] = []
|
|
bank = les_banken(feil)
|
|
henvisninger = les_alle_henvisninger(feil)
|
|
funn = Funn(bank, henvisninger, feil)
|
|
alle = funn.steder()
|
|
for i in sorted(alle, key=sorteringsnokkel):
|
|
hvor = ", ".join(alle[i])
|
|
if i not in bank:
|
|
feil.append(f"{i} has no entry in kilder/ (cited at {hvor})")
|
|
continue
|
|
k = klasse(bank[i])
|
|
if k not in TILLATT and i not in UNNTAK:
|
|
feil.append(
|
|
f"{i} ({bank[i].sted}) is not confirmed: Kontroll is '{k}' (cited at {hvor})"
|
|
)
|
|
try:
|
|
tex_post(bank[i])
|
|
except ReferanseError as e:
|
|
feil.append(str(e))
|
|
feil.extend(f"{i} stands in UNNTAK but is not cited" for i in sorted(set(UNNTAK) - set(alle)))
|
|
return funn
|
|
|
|
|
|
def sjekk(funn: Funn) -> int:
|
|
"""Print what was found; write nothing."""
|
|
s = sys.stdout.write
|
|
steder = funn.steder()
|
|
for opphav, navn in (("a", "rapport/*.tex"), ("b", "parametere.json"), ("c", "generert")):
|
|
s(f"ids {opphav} ({navn}): {len({i for i, _ in funn.henvisninger[opphav]})}\n")
|
|
s(f"ids in all: {len(steder)}\n")
|
|
s(f"entries in the bank: {len(funn.bank)}\n\n")
|
|
teller: Counter[str] = Counter()
|
|
for i in steder:
|
|
if i in funn.bank:
|
|
teller[klasse(funn.bank[i])] += 1
|
|
s("Kontroll classes of the cited entries:\n")
|
|
for k in KLASSER:
|
|
s(f" {k}: {teller[k]}\n")
|
|
s("\nCited entries outside the classes bekreftet, rettet and lagt til:\n")
|
|
for i in sorted(steder, key=sorteringsnokkel):
|
|
if i in funn.bank and klasse(funn.bank[i]) not in {BEKREFTET, RETTET, LAGT_TIL}:
|
|
post = funn.bank[i]
|
|
s(f" {i} [{klasse(post)}] {post.sted}\n Kontroll: {(post.kontroll or '')[:200]}\n")
|
|
s(f" cited at: {', '.join(steder[i])}\n")
|
|
s("\nIds mentioned outside \\kilde{} in rapport/*.tex:\n")
|
|
for i, sted in utenfor_kilde():
|
|
s(f" {i} at {sted}\n")
|
|
s(f"\nProblems ({len(funn.feil)}):\n")
|
|
for f in funn.feil:
|
|
s(f" {f}\n")
|
|
return 1 if funn.feil else 0
|
|
|
|
|
|
def main(argv: list[str]) -> int:
|
|
"""Write the reference list; return 0, or 1 with a message on error."""
|
|
if argv not in ([], ["--sjekk"]):
|
|
sys.stderr.write("bruk: lag_referanser.py [--sjekk]\n")
|
|
return 2
|
|
try:
|
|
funn = finn()
|
|
except (ReferanseError, OSError, json.JSONDecodeError) as e:
|
|
sys.stderr.write(f"lag_referanser: {e}\n")
|
|
return 1
|
|
if argv == ["--sjekk"]:
|
|
return sjekk(funn)
|
|
if funn.feil:
|
|
sys.stderr.write("lag_referanser: the reference list is not written:\n")
|
|
sys.stderr.writelines(f" {f}\n" for f in funn.feil)
|
|
return 1
|
|
steder = funn.steder()
|
|
poster = [funn.bank[i] for i in sorted(steder, key=sorteringsnokkel)]
|
|
i_tekst = len({i for i, _ in funn.henvisninger["a"]})
|
|
i_vedlegg = len({i for i, _ in funn.henvisninger["b"] + funn.henvisninger["c"]})
|
|
alle_omraader = len({omraade(i) for i in funn.bank})
|
|
try:
|
|
tekst = lag(poster, i_tekst, i_vedlegg, alle_omraader)
|
|
except ReferanseError as e:
|
|
sys.stderr.write(f"lag_referanser: {e}\n")
|
|
return 1
|
|
GENERERT.mkdir(exist_ok=True)
|
|
UT.write_text(tekst, encoding="utf-8", newline="\n")
|
|
sys.stdout.write(f"lag_referanser: skrev {UT.relative_to(ROT)} ({len(poster)} poster)\n")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|