diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..246a60f --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,108 @@ +name: Build DCHP specification + +on: + push: + paths: + - 'draft/**' + - 'tools/**' + - 'Makefile' + - '.github/workflows/build.yml' + pull_request: + paths: + - 'draft/**' + - 'tools/**' + - 'Makefile' + - '.github/workflows/build.yml' + workflow_dispatch: + +permissions: + contents: read + +env: + DOC: digital-credentials-harmonized-presentation + +jobs: + build: + name: Build HTML Editor's Copy and ISO Word document + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # HTML Editor's Copy (markdown2rfc / mmark), same tooling as other OpenID + # DCP specifications; published to GitHub Pages below. + - name: Render HTML Editor's Copy + run: make html + + # Pin the interpreter: the mmark->pandoc converter needs tomllib (3.11+). + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Run tests + run: make test + + # ISO-styled Word document (pandoc), for sharing the draft with ISO. + # Delivered as a workflow artifact only; not published to the site. + # Pinned .deb release so the .docx rendering is reproducible (and faster + # than `apt-get update && install`, which pulls whatever the runner has). + - name: Install pandoc + env: + PANDOC_VERSION: '3.8.3' + run: | + curl -fsSL -o /tmp/pandoc.deb \ + "https://github.com/jgm/pandoc/releases/download/${PANDOC_VERSION}/pandoc-${PANDOC_VERSION}-1-amd64.deb" + sudo dpkg -i /tmp/pandoc.deb + + - name: Render ISO Word document + run: make docx + + - name: Assemble HTML for GitHub Pages + run: | + mkdir -p _site + cp build/*.html _site/ + # Redirect the bare Pages URL to the Editor's Copy (avoids a root 404). + printf '\n\n' \ + "${{ env.DOC }}" > _site/index.html + + - name: Upload site artifact + uses: actions/upload-artifact@v4 + with: + name: site + path: _site + + - name: Upload ISO Word document artifact + uses: actions/upload-artifact@v4 + with: + name: iso-word-document + path: build/${{ env.DOC }}.docx + + publish-to-pages: + name: Publish Editor's Copy to GitHub Pages + if: github.ref == 'refs/heads/main' + needs: build + runs-on: ubuntu-latest + # Serialise Pages deployments so two quick merges to main can't race (one + # failing with "a deployment is already in progress", or publishing stale). + concurrency: + group: "pages" + cancel-in-progress: false + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Download site artifact + uses: actions/download-artifact@v4 + with: + name: site + path: _site + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: _site + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 15d664c..9c3c563 100644 --- a/.gitignore +++ b/.gitignore @@ -6,11 +6,16 @@ *.redxml *.upload /*-[0-9][0-9].xml +/draft/*.xml /.refcache /versioned/ archive.json report.xml +# Build outputs — HTML draft, ISO Word document and all intermediates. +# Source scripts live in tools/; the ISO template lives in tools/template/. +/build/ + # Build tooling /.gems/ /.targets.mk @@ -20,6 +25,10 @@ Gemfile.lock package-lock.json /lib/ +# Python bytecode caches (tools/ and tools/tests/) +__pycache__/ +*.py[cod] + # Editor / OS files *~ *.swp diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e77e127 --- /dev/null +++ b/Makefile @@ -0,0 +1,81 @@ +# Build the DCHP specification. +# +# make html - render the HTML Editor's Copy with markdown2rfc (Docker) +# make docx - render the ISO-styled Word document with pandoc +# make all - build both +# make test - run the test suite +# make clean - remove the build/ directory +# +# Source scripts live in tools/; everything generated goes to build/ (which is +# git-ignored). `make html` needs Docker; `make docx` needs pandoc and a Python +# with tomllib (3.11+, for the converter). + +SRCDIR := draft +DOC := digital-credentials-harmonized-presentation +SRC := $(SRCDIR)/$(DOC).md + +TOOLS := tools +BUILD := build + +# The mmark->pandoc converter needs tomllib (Python 3.11+). Auto-pick the first +# available interpreter that has it, so `make docx` works even where the default +# python3 is older; override explicitly with `make docx PYTHON=/path/to/python`. +PYTHON ?= $(shell for p in python3 python3.13 python3.12 python3.11; do \ + command -v $$p >/dev/null 2>&1 && $$p -c 'import tomllib' >/dev/null 2>&1 \ + && { echo $$p; exit 0; }; \ + done) + +# Pinned by digest so the HTML rendering is reproducible: an untagged/:latest +# image could silently change the output between identical commits. Regenerate +# the digest with `docker inspect --format '{{index .RepoDigests 0}}' `. +MD2RFC_IMAGE := danielfett/markdown2rfc@sha256:7b4412559d6ba5db45a14174a28da5b240512e7c2a886a5e4adb44e5e67f34ca + +# Reference document with the ISO styles/layout, committed to the repo (derived +# once from the ISO template by tools/make-iso-reference.py; not regenerated per +# build). The ISO template itself is not committed — see that script's docstring. +REFDOC := $(TOOLS)/template/iso-reference.docx + +HTML_OUT := $(BUILD)/$(DOC)-editors-copy.html + +.PHONY: all html docx test clean need-python + +all: html docx + +# The converter and the tests need $(PYTHON); fail with guidance if none found. +need-python: + @test -n "$(strip $(PYTHON))" || { \ + echo "error: no Python with tomllib (3.11+) found;"; \ + echo " install one or run: make PYTHON=/path/to/python3.11+"; \ + exit 1; } + +## HTML Editor's Copy (markdown2rfc / mmark) -> build/ +html: $(SRC) + # Clean stale intermediates first so the copy below is unambiguous even if a + # previous run was interrupted. + rm -f $(SRCDIR)/$(DOC)*.html $(SRCDIR)/$(DOC)*.xml $(SRCDIR)/$(DOC)*.txt + docker run --rm -v "$(CURDIR)/$(SRCDIR):/data" $(MD2RFC_IMAGE) $(DOC).md + mkdir -p $(BUILD) + # Cleaned above, so this glob now matches exactly the fresh output whatever + # markdown2rfc names it (it may add a draft-version suffix). + cp $(SRCDIR)/$(DOC)*.html $(HTML_OUT) + rm -f $(SRCDIR)/$(DOC)*.html $(SRCDIR)/$(DOC)*.xml $(SRCDIR)/$(DOC)*.txt + @echo "HTML Editor's Copy -> $(HTML_OUT)" + +## ISO-styled Word document (pandoc) -> build/ +docx: need-python $(SRC) $(REFDOC) $(TOOLS)/mmark-to-pandoc.py $(TOOLS)/iso-styles.lua + mkdir -p $(BUILD) + $(PYTHON) $(TOOLS)/mmark-to-pandoc.py < $(SRC) > $(BUILD)/$(DOC).pandoc.md + pandoc $(BUILD)/$(DOC).pandoc.md \ + --reference-doc=$(REFDOC) \ + --lua-filter=$(TOOLS)/iso-styles.lua \ + -o $(BUILD)/$(DOC).docx + rm -f $(BUILD)/$(DOC).pandoc.md + @echo "ISO Word document -> $(BUILD)/$(DOC).docx" + +## Test suite (every tools/tests/test_*.py, so new tests run without editing this) +test: need-python + @for t in $(TOOLS)/tests/test_*.py; do $(PYTHON) $$t || exit 1; done + +clean: + rm -rf $(BUILD) + rm -f $(SRCDIR)/$(DOC)*.html $(SRCDIR)/$(DOC)*.xml $(SRCDIR)/$(DOC)*.txt diff --git a/README.md b/README.md index 589f1c3..2f31e1a 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,11 @@ for more information. ## Specifications -The working group is at an early stage and no drafts have been published yet. -Links to the Editor's Copy and Working Group Drafts will be added here as they -become available. +The Editor's Copy is built automatically from [draft/](draft/) on every change +to the `main` branch. It reflects the latest in-progress edits and is not an +approved Working Group Draft: + +- [Digital Credentials Harmonized Presentation — Editor's Copy](https://openid.github.io/dchp/digital-credentials-harmonized-presentation-editors-copy.html) ## Contributing diff --git a/draft/digital-credentials-harmonized-presentation.md b/draft/digital-credentials-harmonized-presentation.md new file mode 100644 index 0000000..c3f04f2 --- /dev/null +++ b/draft/digital-credentials-harmonized-presentation.md @@ -0,0 +1,120 @@ +%%% +title = "Digital Credentials Harmonized Presentation - Editor's Copy" +abbrev = "dchp" +ipr = "none" +workgroup = "Digital Credentials Harmonized Presentation" +keyword = ["digital credentials", "mdoc", "sd-jwt vc", "presentation", "iso"] + +# NOTE: this [seriesInfo] block is IETF Internet-Draft scaffolding needed by the +# markdown2rfc (mmark/xml2rfc) HTML toolchain; it is NOT a claim that this is an +# IETF document. mmark 2.2.31 hard-codes and does not +# propagate the stream, so a non-IETF stream (e.g. "independent") makes xml2rfc +# fail with a stream/submissionType mismatch. We therefore leave "stream" unset +# rather than assert a false "IETF" stream; mmark only warns ("Empty 'stream'") +# and the HTML still builds. The real publication reference for this OpenID/ISO +# joint document is a WG/SDO decision -- see issue 13. +[seriesInfo] +name = "Internet-Draft" +value = "digital-credentials-harmonized-presentation" +status = "standard" + +[[author]] +initials = "TBD" +surname = "Editor" +fullname = "TBD Editor" +organization = "OpenID Foundation" + [author.address] + email = "openid-specs-dchp@lists.openid.net" + +%%% + +.# Abstract + +This document specifies a harmonized protocol for the presentation of digital +credentials, bringing together the credential presentation approaches of +ISO/IEC 18013-5 and OpenID for Verifiable Presentations across multiple +credential formats, including ISO mdoc and IETF SD-JWT VC. + +.# Foreword + +This specification has been jointly developed by members of ISO/IEC JTC 1/SC 17 +WG 10 and members of the OpenID Foundation Digital Credentials Protocols (DCP) +Working Group, under the OpenID Foundation's Digital Credentials Harmonized +Presentation (DCHP) Working Group. + +This is an Editor's Copy. It is a work in progress and is subject to change +at any time. + +.# Introduction + +ISO/IEC 18013-5 (Device Request / Device Response) and OpenID for Verifiable +Presentations (Authorization Request / Authorization Response) take different +approaches to credential presentation. This document defines a harmonized +digital credentials request protocol that brings these together, supporting both +in-person and online presentation and the coexistence of multiple credential +formats, including ISO mdoc and IETF SD-JWT VC. + +The objective is to enable interoperability among the parties involved in the +presentation of digital credentials while allowing existing deployments to +continue to operate. + +{mainmatter} + +# Scope + +To be completed. + +# Normative references + +The following documents are referred to in the text in such a way that some or +all of their content constitutes requirements of this document. + +To be completed. + +# Terms and definitions + +For the purposes of this document, the following terms and definitions apply. + +ISO and IEC maintain terminology databases for use in standardization at the +following addresses: + +* ISO Online browsing platform: available at +* IEC Electropedia: available at + +To be completed. + +# Symbols and abbreviated terms + +To be completed. + +# Conventions + +In this document, the following verbal forms are used: + +* "shall" indicates a requirement; +* "should" indicates a recommendation; +* "may" indicates a permission; +* "can" indicates a possibility or a capability. + +These verbal forms are used in accordance with ISO/IEC Directives, Part 2, +Clause 7 (see ). + +{backmatter} + +# Bibliography + +[1] ISO/IEC Directives, Part 2, *Principles and rules for the structure and +drafting of ISO and IEC documents* + +# Acknowledgements {#Acknowledgements} + +We would like to thank TBD for their +valuable feedback and contributions to this specification. + +# Document History + + [[ To be removed from the final specification ]] + + -00 + + * initial working draft diff --git a/tools/iso-styles.lua b/tools/iso-styles.lua new file mode 100644 index 0000000..b992f06 --- /dev/null +++ b/tools/iso-styles.lua @@ -0,0 +1,82 @@ +--[[ + ISO front/back-matter formatting for the pandoc -> Word build. + + 1. Title block. mmark-to-pandoc.py carries the title across as two pandoc + metadata fields (`title` and an optional `subtitle` holding the document + status); here we render them as a centred title plus a status subtitle at + the very top. Without this, pandoc + the ISO reference document produce a + Word document with no title at all. + + 2. Foreword / Introduction / Bibliography. These are ISO headings that must NOT + be auto-numbered, so they use the template's ForewordTitle / IntroTitle / + BiblioTitle paragraph styles instead of Heading1 (so the first real + Heading1, Scope, still numbers as clause 1). Some docx viewers apply a + custom style's paragraph properties but not its run properties, so the text + falls back to plain body formatting. To render correctly in every viewer + (not just Microsoft Word) we also set bold + size (14 pt) directly on the + run, in addition to referencing the ISO style. +]]-- + +local title_style = { + ["Foreword"] = "ForewordTitle", + ["Introduction"] = "IntroTitle", + ["Bibliography"] = "BiblioTitle", +} +local HEADING_SIZE = "28" -- half-points (14 pt) + +local function xml_escape(s) + return (s:gsub("&", "&"):gsub("<", "<"):gsub(">", ">")) +end + +-- Unnumbered ISO heading: ISO paragraph style + explicit bold/size on the run. +local function heading_block(text, style) + local xml = table.concat({ + '', + '', + '', + '', xml_escape(text), '', + }) + return pandoc.RawBlock("openxml", xml) +end + +-- A centred paragraph (used for the title and status subtitle). +local function centred_block(text, size, bold, after) + local xml = table.concat({ + '', + '', + '', bold and "" or "", + '', + '', xml_escape(text), '', + }) + return pandoc.RawBlock("openxml", xml) +end + +function Header(el) + if el.level == 1 then + local text = pandoc.utils.stringify(el) + local style = title_style[text] + if style then + return heading_block(text, style) + end + end + return nil +end + +function Pandoc(doc) + local meta_title = doc.meta.title + if meta_title then + local main = pandoc.utils.stringify(meta_title) + local sub = doc.meta.subtitle and pandoc.utils.stringify(doc.meta.subtitle) or nil + doc.meta.title = nil -- suppress pandoc's own (unstyled) title block + doc.meta.subtitle = nil + local head = { centred_block(main, "36", true, "120") } -- 18 pt bold + if sub and sub ~= "" then + head[#head + 1] = centred_block(sub, "26", false, "360") -- 13 pt + end + for _, b in ipairs(doc.blocks) do + head[#head + 1] = b + end + doc.blocks = head + end + return doc +end diff --git a/tools/make-iso-reference.py b/tools/make-iso-reference.py new file mode 100644 index 0000000..9749528 --- /dev/null +++ b/tools/make-iso-reference.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""Derive a clean pandoc reference document from the ISO Word template. + +Pandoc's ``--reference-doc`` needs a ``.docx`` whose *styles* (Heading1..6, +ForewordTitle, TermNum, Terms, Definition, ANNEX, BiblioTitle, Note, ...) and +page/section setup are used to render the generated document. We reuse the ISO +Word template's styles and layout so the exported document looks like an ISO +deliverable, but we deliberately do NOT carry over any ISO copyright / IPR +boilerplate or branding: the document is not an ISO deliverable yet. + +This script therefore: + 1. flips the main-document content type from *template* (.dotx) to *document* + (.docx) so the result is a normal Word document; + 2. replaces the document body — the ISO title page with its document-number + placeholders, WD/CD review warning, and the full ISO copyright/IPR + notice — with a single empty paragraph, keeping only the body-level + (page geometry, running footers) that pandoc reads; + 3. neutralises the ISO copyright notice and document-number placeholders that + live in the running headers/footers; + 4. drops the ISO logo images, the embedded OLE object, and the ISO document + metadata (docProps and customXml parts) so no ISO branding, classification, + or template-author identity travels with the exported file; and + 5. un-hides the styles (removes ) so every docx viewer applies + the ISO heading styles instead of falling back to plain body text. + +This script is NOT run on every build. Its output is committed as +``tools/template/iso-reference.docx`` and used directly by ``make docx``; run +this script by hand to regenerate that file whenever the ISO template (or this +script) changes. The ISO template itself is deliberately NOT committed — it is +ISO-copyrighted and carries personal and classification metadata (docProps) — +so obtain it from ISO/IEC JTC 1/SC 17 WG 10 and place it at +``tools/template/Word_template_for_ISO_standards.dotx`` (or pass its path) to +regenerate. ``tools/tests/test_iso_reference.py`` checks the committed file matches +what this script produces whenever that template is present locally, and +always checks that no ISO metadata/boilerplate ships in any committed Word +file. +Note: pandoc derives a single-section layout from the reference document, so the +committed ``iso-reference.docx`` may still benefit from a one-time manual pass in +Word to finalise section setup, page-numbering restarts, and margins. + +Everything else (styles, numbering, theme, fonts, headers/footers) is copied +through so the document keeps the ISO look-and-feel. + +Usage: + python3 tools/make-iso-reference.py [INPUT.dotx] [OUTPUT.docx] + +Defaults: tools/template/Word_template_for_ISO_standards.dotx -> tools/template/iso-reference.docx +""" +from __future__ import annotations + +import io +import re +import sys +import zipfile +from pathlib import Path + +# --- Neutralise ISO branding in running headers/footers ---------------------- +# The ISO template ships placeholder branding in the running headers/footers: +# footers: "© ISO #### – All rights reserved" (with NBSPs and an en-dash) +# headers: "ISO #####-#:####(X)" (document-number placeholder) +# We replace the text of any run () that carries this ISO branding with a +# neutral, non-IPR marker, leaving page-number fields (separate runs) intact. +# Matching by content (not exact whitespace) keeps this robust to the template's +# non-breaking spaces and dash characters. +DRAFT_LABEL = "Editor's Copy" +_RUN_TEXT = re.compile(r"(]*>)([^<]*)()", re.S) + + +def _neutralise_runs(xml: str) -> str: + def repl(m: "re.Match[str]") -> str: + inner = m.group(2) + if "####" in inner or "All rights reserved" in inner or "© ISO" in inner: + return m.group(1) + DRAFT_LABEL + m.group(3) + return m.group(0) + + return _RUN_TEXT.sub(repl, xml) + + +# --- Main-document content-type override: template -> document ---------------- +CT_TEMPLATE = ( + "application/vnd.openxmlformats-officedocument." + "wordprocessingml.template.main+xml" +) +CT_DOCUMENT = ( + "application/vnd.openxmlformats-officedocument." + "wordprocessingml.document.main+xml" +) + +# --- Parts to drop entirely (ISO logos, OLE object, ISO document metadata) --- +# The template's only images (word/media) are the ISO logo/branding and the +# template's embedded OLE object (word/embeddings/oleObject1.bin), all referenced +# solely by the title page (which pandoc discards). customXml holds ISO metadata +# bindings; docProps holds the ISO company/classification metadata (app.xml, +# custom.xml) and the template author's personal metadata (core.xml carries +# dc:creator / cp:lastModifiedBy). None are needed to style the document, so we +# drop them all — Word and pandoc are fine without document properties. +# [trash]/ is zip-housekeeping junk left in the .dotx by whatever tool last +# edited the ISO template; it is not part of the OPC package. +DROP_PREFIXES = ("word/media/", "word/embeddings/", "customXml/", "docProps/", "[trash]/") + +# Substrings that identify a relationship/content-type entry pointing at a +# dropped part, so we can keep [Content_Types].xml and the *.rels files +# internally consistent after the drop. Derived from DROP_PREFIXES because +# the two must move in lockstep; relationship targets are relative to the +# referencing part, so "word/media/" appears as "media/" ("[trash]/" never +# appears in a relationship target, matching it is harmless). +_DANGLING = tuple(p.removeprefix("word/") for p in DROP_PREFIXES) +_XML_ELEMENT = re.compile(r"<(Relationship|Override|Default)\b[^>]*/>") + + +def _strip_dangling(xml: str) -> str: + def repl(m: "re.Match[str]") -> str: + el = m.group(0) + if any(ref in el for ref in _DANGLING): + return "" + return el + + return _XML_ELEMENT.sub(repl, xml) + + +# The template's document body is the ISO title page: document-number +# placeholders, the WD/CD review warning, the full ISO copyright/IPR notice, +# and the logo drawings / OLE object that referenced the dropped media parts. +# Pandoc ignores the reference document's body — it reads only the styles and +# the final body-level — so replace the whole body with one empty +# paragraph plus that . Nothing ISO-owned survives in the committed +# file when it is opened standalone, and no dangling image references are left +# behind. (Both .* are greedy, so the match keeps the *last* sectPr: the +# body-level one.) +_BODY = re.compile(r"().*()\s*()", re.S) + + +def _empty_body(xml: str) -> str: + return _BODY.sub(r"\1\2\3", xml) + + +# Belt-and-braces for the header/footer parts we keep: _strip_dangling removes +# media/OLE relationships from *all* .rels files, so also strip any +# // elements there (none in the current template) +# rather than leave dangling references that render as broken-image +# placeholders. +_DRAWING_OR_OBJECT = re.compile( + r"", re.S +) + + +def _strip_media_elements(xml: str) -> str: + return _DRAWING_OR_OBJECT.sub("", xml) + + +# The ISO template hides most of its styles from the Word gallery +# ( / ). Several docx viewers (Preview/Quick +# Look, Pages, Google Docs, some LibreOffice paths) skip the *formatting* of +# semi-hidden styles and fall back to Normal, so the unnumbered headings +# (Foreword/Introduction, which use the ForewordTitle/IntroTitle styles) render +# as plain body text. We remove those flags so every viewer applies the ISO +# styles — and so editors can see them in the Word styles pane. +_HIDE_FLAGS = re.compile(r"]*/>") + + +def _activate_styles(xml: str) -> str: + return _HIDE_FLAGS.sub("", xml) + + +def transform(name: str, data: bytes) -> bytes: + if name == "[Content_Types].xml": + text = data.decode("utf-8").replace(CT_TEMPLATE, CT_DOCUMENT) + return _strip_dangling(text).encode("utf-8") + if name.endswith(".rels"): + return _strip_dangling(data.decode("utf-8")).encode("utf-8") + if name == "word/document.xml": + return _empty_body(data.decode("utf-8")).encode("utf-8") + if name == "word/styles.xml": + return _activate_styles(data.decode("utf-8")).encode("utf-8") + if name.startswith(("word/header", "word/footer")) and name.endswith(".xml"): + text = _strip_media_elements(data.decode("utf-8")) + return _neutralise_runs(text).encode("utf-8") + return data + + +def build(src: Path) -> bytes: + """Return the transformed reference document as .docx bytes.""" + buf = io.BytesIO() + with zipfile.ZipFile(src) as zin, zipfile.ZipFile( + buf, "w", zipfile.ZIP_DEFLATED + ) as zout: + for item in zin.infolist(): + if item.filename.startswith(DROP_PREFIXES): + continue + data = transform(item.filename, zin.read(item.filename)) + zout.writestr(item, data) + return buf.getvalue() + + +def main() -> int: + root = Path(__file__).resolve().parent.parent + src = Path(sys.argv[1]) if len(sys.argv) > 1 else root / "tools" / "template" / "Word_template_for_ISO_standards.dotx" + dst = Path(sys.argv[2]) if len(sys.argv) > 2 else root / "tools" / "template" / "iso-reference.docx" + + if not src.is_file(): + print(f"error: template not found: {src}", file=sys.stderr) + return 1 + + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_bytes(build(src)) + + try: + shown = dst.relative_to(root) + except ValueError: + shown = dst + print(f"wrote {shown}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/mmark-to-pandoc.py b/tools/mmark-to-pandoc.py new file mode 100644 index 0000000..83faa6c --- /dev/null +++ b/tools/mmark-to-pandoc.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Strip mmark-only syntax so the spec source can be fed to pandoc. + +The canonical spec source is authored in mmark (so ``markdown2rfc`` can render +the HTML editor's draft). mmark adds a few constructs that pandoc does not +understand; this filter removes them so the *same* source can also be converted +to the ISO Word document by pandoc: + + * the ``%%% ... %%%`` TOML front matter block (mmark document metadata) — it is + parsed with ``tomllib`` and the ``title`` is re-emitted as pandoc metadata. + The mmark title encodes the document status after the last `` - `` (e.g. + ``"... - Editor's Copy"``); we split it back into a ``title`` and a + ``subtitle`` so the Word document gets a styled title + status block; + * the ``.# Abstract`` section (an RFC/mmark concept; ISO documents have no + abstract, so the whole abstract block is dropped — up to, but not including, + the next heading of *any* level, matching how mmark ends the abstract); + * the ``{frontmatter}`` / ``{mainmatter}`` / ``{backmatter}`` part markers; + * kramdown-style ``{: ...}`` inline-attribute-list lines; + * mmark special headings ``.# Heading`` become normal ``# Heading``. + +All of the above filtering is skipped *inside fenced code blocks* (``` ``` `` / +``~~~``), so normative request/response examples that happen to contain these +constructs pass through verbatim. Everything else (headings, paragraphs, lists, +tables, definition lists, notes) is common Markdown and is passed through +unchanged. Reads stdin, writes stdout. + +Requires Python 3.11+ for ``tomllib`` (or the ``tomli`` backport on 3.10). +""" +from __future__ import annotations + +import json +import re +import sys + +try: + import tomllib +except ModuleNotFoundError: # Python < 3.11 + import tomli as tomllib # type: ignore[no-redef] + + +# All block-level constructs may be indented up to 3 *spaces* and still count +# (CommonMark rules, which mmark inherits): 4+ spaces — or a tab, which counts +# as 4 columns — make the line indented-code content instead. Every regex +# below spells that tolerance the same way: `^ {0,3}`. +HEADING = re.compile(r"^ {0,3}\.?#{1,6}(?:\s|$)") # ATX, incl. mmark's ".#" +PART_MARKER = re.compile(r"^ {0,3}\{(?:frontmatter|mainmatter|backmatter)\}\s*$") +ABSTRACT_START = re.compile(r"^ {0,3}\.#\s+Abstract\b") +IAL_LINE = re.compile(r"^ {0,3}\{:.*\}\s*$") # kramdown inline attribute list +SETEXT_UNDERLINE = re.compile(r"^ {0,3}(=+|-+)\s*$") +FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$") # group(2) is the info string +SPECIAL_HEADING = re.compile(r"^( {0,3})\.(#{1,6})") + + +def _split_front_matter(lines: list[str]) -> tuple[dict, int]: + """Return (metadata, body_start_index) for a leading ``%%% ... %%%`` block. + + If the source does not open with ``%%%`` there is no front matter: the whole + input is the body. The closing delimiter is found by parsing: a ``%%%`` line + *inside* a TOML multi-line string does not close the block (the truncated + text is not valid TOML there), matching how mmark reads the file. A block + that never closes, or whose content is not valid TOML, raises ValueError so + the build fails loudly instead of leaking the metadata (author emails and + all) into the Word document body. + """ + if not lines or lines[0].strip() != "%%%": + return {}, 0 + error: Exception | None = None + for j in range(1, len(lines)): + if lines[j].strip() != "%%%": + continue + try: + return tomllib.loads("\n".join(lines[1:j])), j + 1 + except tomllib.TOMLDecodeError as e: + error = e + if error is not None: + raise ValueError(f"front matter is not valid TOML: {error}") + raise ValueError("front matter opened with '%%%' is never closed") + + +def _title_and_status(meta: dict) -> tuple[str | None, str | None]: + """Extract (title, status) from the parsed front matter. + + The status (e.g. ``Editor's Copy``) is encoded after the *last* `` - `` in + the single mmark ``title`` so the same title still drives the HTML draft. + Splitting on the last separator keeps titles that themselves contain `` - `` + intact. + """ + title = meta.get("title") + if not isinstance(title, str) or not title.strip(): + return None, None + main, sep, status = title.rpartition(" - ") + if sep: + return main.strip(), status.strip() + return title.strip(), None + + +def _process_body(lines: list[str]) -> list[str]: + out: list[str] = [] + in_fence = False + fence = "" + dropping_abstract = False + + for i, line in enumerate(lines): + m = FENCE.match(line) + if m: + marker, info = m.group(1), m.group(2) + if not in_fence: + # A backtick fence's info string may not contain backticks + # (such a line is a paragraph, not a fence); tilde fences + # carry no such restriction. + if marker[0] == "~" or "`" not in info: + in_fence, fence = True, marker + elif ( + marker[0] == fence[0] + and len(marker) >= len(fence) + and not info.strip() + ): + # A closer must use the opener's character, be at least as + # long, and be bare; any other fence-looking line is content. + in_fence = False + if not dropping_abstract: + out.append(line) + continue + if in_fence: + if not dropping_abstract: + out.append(line) + continue + + # The abstract runs until the next heading (any level) or part marker; + # everything in between is dropped from the ISO Word output. Headings + # may also be setext-style: text underlined with = or -. Approximation: + # we end the drop at the line right before the underline, whereas a + # multi-line setext paragraph would make mmark end it at the + # paragraph's first line — close enough for a filter, and it keeps + # whole sections from vanishing. + if dropping_abstract: + next_line = lines[i + 1] if i + 1 < len(lines) else "" + starts_setext = ( + line.strip() != "" + and not IAL_LINE.match(line) + and SETEXT_UNDERLINE.match(next_line) + ) + if HEADING.match(line) or PART_MARKER.match(line) or starts_setext: + dropping_abstract = False # fall through and handle this line + else: + continue + + if ABSTRACT_START.match(line): + dropping_abstract = True + continue + if PART_MARKER.match(line): + continue + if IAL_LINE.match(line): + continue + + line = SPECIAL_HEADING.sub(r"\1\2", line) # ".# Heading" -> "# Heading" + out.append(line) + + # Collapse leading blank lines produced by the removals. + while out and out[0].strip() == "": + out.pop(0) + return out + + +def convert(text: str) -> str: + # An editor may save the file with a UTF-8 BOM; it must not hide the + # front-matter delimiter (str.strip() does not remove U+FEFF). + text = text.removeprefix("\ufeff") + # Split on real newlines only: str.splitlines() also breaks on U+2028, + # U+0085, form feed, etc., which markdown treats as ordinary characters \u2014 + # rejoining with "\n" would turn them into hard line breaks. + lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + if lines and lines[-1] == "": + lines.pop() # a trailing newline terminates the last line + front, start = _split_front_matter(lines) + title, status = _title_and_status(front) + + body = "\n".join(_process_body(lines[start:])) + "\n" + + # Re-emit the title (and status subtitle) as a pandoc YAML metadata block; + # the Lua filter renders them as the Word document's title block. + if title: + meta = ["---", "title: " + json.dumps(title)] + if status: + meta.append("subtitle: " + json.dumps(status)) + meta += ["---", "", ""] + body = "\n".join(meta) + body + return body + + +if __name__ == "__main__": + try: + sys.stdout.write(convert(sys.stdin.read())) + except ValueError as e: + raise SystemExit(f"error: {e}") diff --git a/tools/template/iso-reference.docx b/tools/template/iso-reference.docx new file mode 100644 index 0000000..5797535 Binary files /dev/null and b/tools/template/iso-reference.docx differ diff --git a/tools/tests/fixtures/sample.expected.md b/tools/tests/fixtures/sample.expected.md new file mode 100644 index 0000000..448da57 --- /dev/null +++ b/tools/tests/fixtures/sample.expected.md @@ -0,0 +1,35 @@ +--- +title: "Sample Spec - Volume 2" +subtitle: "Editor Copy" +--- + +## Notice + +This frontmatter subsection comes after the abstract and MUST survive: mmark +ends the abstract at the next heading of any level. + + +# Foreword + +Foreword text (unnumbered in both renditions). + +# Scope + +A normative example follows and must pass through verbatim: + +``` +%%% +title = "not the real title" +%%% +{mainmatter} +.# This dot-hash line is example content, not a heading +{: title="keep me"} +``` + +Body continues after the example. + + + +# Bibliography + +[1] Some reference. diff --git a/tools/tests/fixtures/sample.md b/tools/tests/fixtures/sample.md new file mode 100644 index 0000000..ab48b14 --- /dev/null +++ b/tools/tests/fixtures/sample.md @@ -0,0 +1,49 @@ +%%% +title = 'Sample Spec - Volume 2 - Editor Copy' +abbrev = "sample" +ipr = "none" + +[seriesInfo] +name = "Internet-Draft" +value = "sample" +status = "standard" +stream = "independent" +%%% + +.# Abstract + +This abstract must be dropped from the ISO Word output. + +## Notice + +This frontmatter subsection comes after the abstract and MUST survive: mmark +ends the abstract at the next heading of any level. + +{mainmatter} + +.# Foreword + +Foreword text (unnumbered in both renditions). + +# Scope + +A normative example follows and must pass through verbatim: + +``` +%%% +title = "not the real title" +%%% +{mainmatter} +.# This dot-hash line is example content, not a heading +{: title="keep me"} +``` + +Body continues after the example. + +{: .stray-attribute-list} + +{backmatter} + +# Bibliography + +[1] Some reference. diff --git a/tools/tests/test_iso_reference.py b/tools/tests/test_iso_reference.py new file mode 100644 index 0000000..aa9f845 --- /dev/null +++ b/tools/tests/test_iso_reference.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Hygiene tests for committed Word (WordprocessingML) files. + +``tools/template/iso-reference.docx`` is derived from the ISO Word template by +``tools/make-iso-reference.py`` and committed to the repo (the build uses it +directly; the generator is not part of the build). The tool promises that no +ISO branding, copyright/IPR boilerplate, or document metadata from the ISO +template travels into the public repo. A regression here is silent — the +files are binary and nobody re-opens them on review — so these tests pin +those guarantees against *every* committed .docx/.dotx (the ISO template +itself is deliberately not committed — see tools/make-iso-reference.py). + +Run: python3 tools/tests/test_iso_reference.py +""" +from __future__ import annotations + +import importlib.util +import io +import pathlib +import subprocess +import sys +import zipfile + +ROOT = pathlib.Path(__file__).resolve().parent.parent.parent +REFDOC = ROOT / "tools" / "template" / "iso-reference.docx" +TEMPLATE = ROOT / "tools" / "template" / "Word_template_for_ISO_standards.dotx" + +# The generator's filename has a hyphen, so load it by path rather than import. +_spec = importlib.util.spec_from_file_location( + "make_iso_reference", ROOT / "tools" / "make-iso-reference.py" +) +assert _spec and _spec.loader +make_iso_reference = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(make_iso_reference) + + +def check_no_document_properties(z: zipfile.ZipFile) -> list[str]: + """No docProps/ part may ship: core.xml carries the ISO template author's + personal metadata (dc:creator / cp:lastModifiedBy), app.xml and custom.xml + the ISO company/classification metadata. Nothing may still reference them. + """ + problems = [ + f"{name}: docProps part shipped" + for name in z.namelist() + if name.startswith("docProps/") + ] + for name in ("[Content_Types].xml", "_rels/.rels"): + if b"docProps/" in z.read(name): + problems.append(f"{name}: dangling docProps reference") + return problems + + +# Tripwire strings from the ISO title page, copyright/IPR notice, the +# document-number placeholders, and the ISO classification label. Any one of +# them surviving in any XML part means ISO boilerplate leaked into the +# public file. +ISO_BOILERPLATE_MARKERS = ( + "© ISO", + "All rights reserved", + "copyright@iso.org", + "ISO copyright office", + "#####", + "ISO - Internal", +) + + +def committed_word_files() -> list[pathlib.Path]: + """Every WordprocessingML package committed to the repo.""" + out = subprocess.run( + ["git", "ls-files", "-z", "--", "*.docx", "*.dotx", "*.docm", "*.dotm"], + cwd=ROOT, + capture_output=True, + check=True, + text=True, + ).stdout + return [ROOT / name for name in out.split("\0") if name] + + +def check_no_iso_boilerplate(z: zipfile.ZipFile) -> list[str]: + """No ISO copyright/IPR or title-page text may remain in any XML part + (body, headers, footers, notes); footers carry the draft label instead.""" + problems = [] + for name in z.namelist(): + if not name.endswith(".xml"): + continue + text = z.read(name).decode("utf-8") + problems += [ + f"{name}: contains {marker!r}" + for marker in ISO_BOILERPLATE_MARKERS + if marker in text + ] + return problems + + +def check_no_trash_entries(z: zipfile.ZipFile) -> list[str]: + """The source .dotx contains [trash]/*.dat zip-housekeeping junk left by + whatever tool last edited the ISO template; it must not be copied over.""" + return [ + f"{name}: junk zip entry copied from the template" + for name in z.namelist() + if name.startswith("[trash]/") + ] + + +def check_section_properties_kept(z: zipfile.ZipFile) -> list[str]: + """pandoc derives page geometry and the running footers from the body-level + ; cleaning the body must not lose it.""" + doc = z.read("word/document.xml").decode("utf-8") + return [] if " lost"] + + +def check_in_sync_with_generator(z: zipfile.ZipFile) -> list[str]: + """Regenerating from the ISO template must reproduce the committed + reference document part-for-part. Otherwise the ISO template or the + generator changed without `python3 tools/make-iso-reference.py` being + re-run — and the build, which reads only the committed file, would + silently ship the stale styles. (Parts are compared decompressed, so + zlib differences between environments cannot cause false failures.) + + The ISO template is not committed (see tools/make-iso-reference.py), so + this check runs only where an editor has placed it locally; CI skips it. + """ + if not TEMPLATE.is_file(): + print(f"SKIP: generator sync check ({TEMPLATE.name} not present)") + return [] + fresh = zipfile.ZipFile(io.BytesIO(make_iso_reference.build(TEMPLATE))) + if fresh.namelist() != z.namelist(): + return [ + "part list differs from regeneration " + f"(committed {z.namelist()} vs fresh {fresh.namelist()}); " + "re-run tools/make-iso-reference.py and commit the result" + ] + return [ + f"{name}: differs from regeneration; " + "re-run tools/make-iso-reference.py and commit the result" + for name in fresh.namelist() + if fresh.read(name) != z.read(name) + ] + + +# Hygiene rules applied to every committed Word file. +FILE_CHECKS = [ + check_no_document_properties, + check_no_iso_boilerplate, + check_no_trash_entries, +] + +# Structural/consistency rules for the pandoc reference document itself. +REFDOC_CHECKS = [ + check_section_properties_kept, + check_in_sync_with_generator, +] + + +def main() -> int: + problems: list[str] = [] + files = committed_word_files() + if REFDOC not in files: + problems.append(f"{REFDOC.relative_to(ROOT)}: not committed") + for path in files: + checks = FILE_CHECKS + (REFDOC_CHECKS if path == REFDOC else []) + with zipfile.ZipFile(path) as z: + for check in checks: + problems += [ + f"{path.relative_to(ROOT)}: {p}" for p in check(z) + ] + for p in problems: + print(f"FAIL: {p}", file=sys.stderr) + if problems: + return 1 + print(f"OK: hygiene checks pass for {len(files)} committed Word file(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/tests/test_mmark_to_pandoc.py b/tools/tests/test_mmark_to_pandoc.py new file mode 100644 index 0000000..c665785 --- /dev/null +++ b/tools/tests/test_mmark_to_pandoc.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Golden-output test for tools/mmark-to-pandoc.py. + +The converter runs three stacked line transformations (front-matter parsing, +abstract dropping, fence-aware filtering) whose failures are otherwise silent — +a mangled normative example or a dropped subsection produces no build error, only +a wrong Word document. This test pins the converter's output for a sample that +exercises every transformation: + + * a single-quoted TOML title containing embedded `` - `` separators (title vs. + status split); + * an ``.# Abstract`` followed by a ``## Notice`` subsection that must survive; + * a fenced code block whose contents (``%%%``, ``{mainmatter}``, ``.#``, + ``{: ...}``) must pass through verbatim. + +Run: python3 tools/tests/test_mmark_to_pandoc.py (requires Python 3.11+ / tomllib) +""" +from __future__ import annotations + +import difflib +import importlib.util +import pathlib +import sys + +HERE = pathlib.Path(__file__).resolve().parent +FIXTURES = HERE / "fixtures" + +# The converter's filename has a hyphen, so load it by path rather than import. +_spec = importlib.util.spec_from_file_location( + "mmark_to_pandoc", HERE.parent / "mmark-to-pandoc.py" +) +assert _spec and _spec.loader +mmark_to_pandoc = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(mmark_to_pandoc) + + +# Focused cases for the individual line transformations: (name, source, +# expected convert() output). Each pins one behavior the golden sample does +# not exercise; the fence cases follow CommonMark's fence rules, which mmark +# inherits. +CASES: list[tuple[str, str, str]] = [ + ( + "inner 3-backtick line does not close a 4-backtick fence " + "(closer must be at least as long as the opener)", + "# Scope\n\n````\n```\n{: #example-ial}\n```\n````\n\n{backmatter}\n\n# After\n", + "# Scope\n\n````\n```\n{: #example-ial}\n```\n````\n\n\n# After\n", + ), + ( + "a fence line with an info string does not close an open fence " + "(closing fences must be bare)", + '# Scope\n\n```\n```python\n{mainmatter}\n.# not a heading\n```\n\nAfter text.\n', + '# Scope\n\n```\n```python\n{mainmatter}\n.# not a heading\n```\n\nAfter text.\n', + ), + ( + "backticks indented 4+ spaces are indented-code content, not a fence", + "# Scope\n\n ```\n\n{mainmatter}\n\nAfter text.\n", + "# Scope\n\n ```\n\n\nAfter text.\n", + ), + ( + "an ordinary fenced block with an info string still passes through " + "and filtering resumes after it", + '```json\n{"a": 1}\n```\n\n{: .x}\n', + '```json\n{"a": 1}\n```\n\n', + ), + ( + "a UTF-8 BOM does not hide the front-matter delimiter", + '%%%\ntitle = "T - S"\n%%%\n\n# Scope\n', + '---\ntitle: "T"\nsubtitle: "S"\n---\n\n# Scope\n', + ), + ( + "a %%% line inside a TOML multi-line string does not close the " + "front matter", + '%%%\ntitle = "T - S"\ndescription = """\n%%%\n"""\n%%%\n\n# Scope\n', + '---\ntitle: "T"\nsubtitle: "S"\n---\n\n# Scope\n', + ), + ( + "a setext heading terminates the abstract drop " + "(mmark ends the abstract at the next heading of any style)", + ".# Abstract\n\nAbstract text.\n\nForeword\n========\n\n" + "Foreword body.\n\n# Scope\n", + "Foreword\n========\n\nForeword body.\n\n# Scope\n", + ), + ( + "an abstract heading indented 1-3 spaces (still a heading) is " + "dropped, not rewritten to a literal '# Abstract' clause", + " .# Abstract\n\nAbstract text.\n\n# Scope\n", + "# Scope\n", + ), + ( + "an attribute list indented 1-3 spaces is dropped, not passed " + "through as literal text", + "# Scope\n\n- item\n {: .keep-with-next}\n\nAfter text.\n", + "# Scope\n\n- item\n\nAfter text.\n", + ), + ( + "a part marker indented 1-3 spaces is dropped, not passed through " + "as literal text", + "# Scope\n\n {mainmatter}\n\nAfter text.\n", + "# Scope\n\n\nAfter text.\n", + ), + ( + "unicode line boundaries (U+2028, form feed) pasted from Word/PDF " + "are ordinary characters to markdown, not line breaks", + "# Sco\u2028pe\n\nBody\fmore.\n", + "# Sco\u2028pe\n\nBody\fmore.\n", + ), + ( + "CRLF line endings still split correctly", + "# Scope\r\n\r\n{mainmatter}\r\nBody.\r\n", + "# Scope\n\nBody.\n", + ), +] + + +def _convert_or_error(source: str) -> str: + try: + return mmark_to_pandoc.convert(source) + except Exception as e: # an unexpected raise is a failure; show it as one + return f"\n" + + +def _diff(expected: str, got: str) -> str: + return "".join( + difflib.unified_diff( + expected.splitlines(keepends=True), + got.splitlines(keepends=True), + fromfile="expected", + tofile="got", + ) + ) + + +def check_golden() -> list[str]: + source = (FIXTURES / "sample.md").read_text() + expected = (FIXTURES / "sample.expected.md").read_text() + got = mmark_to_pandoc.convert(source) + if got != expected: + return ["golden output mismatch:\n" + _diff(expected, got)] + return [] + + +def check_cases() -> list[str]: + return [ + f"{name}:\n" + _diff(expected, got) + for name, source, expected in CASES + if (got := _convert_or_error(source)) != expected + ] + + +def check_unclosed_front_matter() -> list[str]: + """A ``%%%`` block that never closes must fail the build loudly — both + silently dropping the rest of the file and emitting the raw TOML (author + emails and all) as Word body text are wrong.""" + source = '%%%\ntitle = "T"\n\n# Scope\n' + try: + mmark_to_pandoc.convert(source) + except ValueError as e: + if "front matter" in str(e): + return [] + return [f"unclosed front matter: error message lacks context: {e}"] + return ["unclosed front matter: convert() did not raise"] + + +def main() -> int: + failures = check_golden() + check_cases() + check_unclosed_front_matter() + for failure in failures: + print(f"FAIL: {failure}", file=sys.stderr) + if failures: + return 1 + print(f"OK: mmark-to-pandoc golden output and {len(CASES)} cases match") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())