feat(Djot): Support Lua filters for altering the AST before processing

Exposed as a "filter" option on the Djot inputter.

Author: Omikhleia

examples/custom-filter.lua

@@ -0,0 +1,99 @@
+--- An example Lua filter for Djot.
+--
+-- This filter does two things in sequence:
+--
+--  - On both spans and strings, it re-maps some classes to custom styles.
+--  - On strings only, it transforms numbers with the class "siecle" into roman numerals with a superscript "e", as per French typographic conventions.
+--
+-- @license MIT
+-- @copyright (c) 2025 Omikhleia / Didier Willis
+
+-- luacheck: globals djot
+
+local CLASS2STYLE = {
+  software = "Software",
+  hardware = "Hardware",
+}
+
+local function classToStyle (e)
+  if e.attr and e.attr['class'] then
+    local styles = {}
+    local classes = pl.Set(pl.stringx.split(e.attr['class']))
+    for class, style in pairs(CLASS2STYLE) do
+      if classes[class] then
+        styles[#styles+1] = style
+        classes[class] = nil
+      end
+    end
+    if #styles > 0 then
+      if #styles > 1 then
+        SU.warn("Multiple styles implied by classes, using the first one '" .. styles[1] .. "'")
+      end
+      if e.attr['custom-style'] then
+        SU.warn("Ignoring custom-style '" .. e.attr['custom-style'] .. "' because class implies style '" .. styles[1] .. "'")
+      end
+      e.attr['custom-style'] = styles[1]
+      e.attr.class = table.concat(pl.Set.values(classes), " ") -- Unused classes are kept
+    end
+  end
+end
+
+local function numberToRoman (num)
+  local val = { 1000, 900, 500, 400, 100, 90, 50, 40, 10, 9, 5, 4, 1 }
+  local syms = { "m", "cm", "d", "cd", "c", "xc", "l", "xl", "x", "ix", "v", "iv", "i" }
+  local roman = ""
+  for i = 1, #val do
+    while num >= val[i] do
+      num = num - val[i]
+      roman = roman .. syms[i]
+    end
+  end
+  return roman
+end
+
+return {
+  -- A first filter that maps classes to custom styles on spans and strings.
+  -- Ex. [Pandoc]{.software} --> equivalent to [Pandoc]{custom-style="Software"}
+  {
+    span = function(e)
+      classToStyle(e)
+    end,
+    str = function(e)
+      classToStyle(e)
+    end,
+  },
+  -- A second filter that transforms numbers with the class "siecle" into small caps roman
+  -- numerals with a superscript "e", as per French typographic conventions.
+  -- Ex. 21{.siecle} --> equivalent to xxi{.smallcaps}^e^
+  -- It somewhat specific to French, and we should rather delegate to a SILE command,
+  -- but it's a good example of AST manipulation.
+  {
+    str = function(e)
+      if e.attr and e.attr['class'] and tonumber(e.text) then
+        local classes = pl.Set(pl.stringx.split(e.attr['class']))
+        if not classes["siecle"] then
+          return -- Nothing to do
+        end
+        -- Unused classes are kept
+        classes["siecle"] = nil
+        e.attr['class'] = table.concat(pl.Set.values(classes), " ")
+        local num = tonumber(e.text)
+
+        -- Create the roman numeral
+        local century = djot.ast.new_node("str")
+        century.text = numberToRoman(num)
+        century.attr = djot.ast.new_attributes({ class = "smallcaps" })
+        -- Create the superscript "e"
+        local exp = djot.ast.new_node("str")
+        exp.text = num == 1 and "er" or "e"
+        -- Create the superscript node
+        local super = djot.ast.new_node("superscript")
+        super.children = { exp }
+        -- Transform the original str node into a span
+        e.tag = "span"
+        e.text = nil
+        e.children = { century, super }
+      end
+    end,
+  }
+}

inputters/djot.lua

@@ -3,7 +3,7 @@
 -- Using the djot Lua library for parsing.
 -- Reusing the common commands initially made for the "markdown" inputter/package.
 --
--- @copyright License: MIT (c) 2023-2024 Omikhleia, Didier Willis
+-- @copyright License: MIT (c) 2023-2025 Omikhleia, Didier Willis
 -- @module inputters.djot
 --
 local utils = require("packages.markdown.utils")
@@ -775,6 +775,25 @@ function inputter.appropriate (round, filename, _)
   return false
 end
 
+function inputter:_loadFilter (name, env)
+  -- Resolve the Lua file name in the same way as SILE for documents in the user tree.
+  local qname = name:match("%.lua$") and name or (name .. ".lua")
+  local filename = SILE.resolveFile(qname)
+  if filename then
+    local filter, err = utils.sandboxedLoadfile(filename, env)
+    if not filter then
+      SU.error("Failure loading filter '".. name .. "': " .. err)
+    end
+    if #filter == 0 then
+      -- Just a single filter.
+      -- (The Djot filter logic expects a list of filters applied in sequence.)
+      filter = { filter }
+    end
+    return filter
+  end
+  SU.error("Cannot find filter '" .. name .. "'")
+end
+
 function inputter:parse (doc)
   local djot = require("djot")
   local djast = djot.parse(doc, true, function (warning)
@@ -791,6 +810,28 @@ function inputter:parse (doc)
     local snippet = luautf8.sub(doc, sp + 1, ep)
     SU.warn(warning.message .. " near [[…" .. snippet .. "]]")
   end)
+
+  local fname = self.options.filter
+  if fname then
+    if type(fname) ~= "string" then
+      SU.error("The 'filter' option to the Djot inputter must be a string")
+    end
+    local filter = self:_loadFilter(fname, {
+      -- Allow Djot AST manipulations in the filter environment.
+      djot = {
+        ast = {
+          insert_attribute = djot.ast.insert_attribute,
+          copy_attributes = djot.ast.copy_attributes,
+          new_attributes = djot.ast.new_attributes,
+          new_node = djot.ast.new_node,
+          add_child = djot.ast.add_child,
+          has_children = djot.ast.has_children,
+        }
+      }
+    })
+    djot.filter.apply_filter(djast, filter)
+  end
+
   local renderer = Renderer(self.options, doc)
   local tree = renderer:render(djast)
 

packages/markdown/utils.lua

@@ -1,21 +1,25 @@
 --- A few utilities for the markdown / pandocast inputters
 --
--- @copyright License: MIT (c) 2022 Omikhleia
+-- @copyright License: MIT (c) 2022-2025 Omikhleia
 -- @module packages.markdown.utils
 --
 local createCommand = SU.ast.createCommand
 local createStructuredCommand = SU.ast.createStructuredCommand
 
 --- Extract the extension from a file name.
+--
 -- Assumes a POSIX-compliant name (with a slash as path separators).
+--
 -- @tparam string fname File name
 -- @treturn string File extension
 local function getFileExtension (fname)
   return fname:match("[^/]+$"):match("[^.]+$")
 end
 
 --- Non-breakable space extraction from a string.
+--
 -- It replaces them with an appropriate non-breakable inter-word space command.
+--
 -- @tparam string str Input string
 -- @treturn string|table Filtered string or SILE AST table
 local function nbspFilter (str)
@@ -33,6 +37,7 @@ local function nbspFilter (str)
 end
 
 --- Check if a given class is present in the options.
+--
 -- @tparam table options Command options
 -- @tparam string classname Pseudo-class specifier
 -- @treturn boolean
@@ -45,6 +50,7 @@ local function hasClass (options, classname)
 end
 
 --- Find the first raw handler suitable for the given pseudo-class attributes.
+--
 -- @tparam table options Command options
 -- @treturn function|nil Handler function (if found)
 local function hasRawHandler (options)
@@ -58,6 +64,7 @@ local function hasRawHandler (options)
 end
 
 --- Find the first embedder suitable for the given pseudo-class attributes.
+--
 -- @tparam table options Command options with class attribute (nil or list of comma-separated classes)
 -- @treturn string|nil Embedder name (if found)
 -- @treturn function|nil Embedder handler function (if applicable)
@@ -95,8 +102,10 @@ local metrics = require("fontmetrics")
 local bsratiocache = {}
 
 --- Compute the baseline ratio for the current font.
---- This is a ratio of the descender to the theoretical height of the font.
----@treturn number Descender ratio
+--
+-- This is a ratio of the descender to the theoretical height of the font.
+--
+-- @treturn number Descender ratio
 local function computeBaselineRatio ()
   local fontoptions = SILE.font.loadDefaults({})
   local bsratio = bsratiocache[SILE.font._key(fontoptions)]
@@ -110,8 +119,10 @@ local function computeBaselineRatio ()
 end
 
 --- Naive citation reference parser.
+--
 -- We only support a very simple syntax for now: `@key[, ]+[locator]`,
 -- where the unique locator consists of a name and a value separated by spaces.
+--
 -- @tparam string str Citation string
 -- @tparam[opt] table pos Position in the source (for error reporting)
 -- @treturn table AST for the citation command
@@ -143,6 +154,72 @@ local function naiveCitations (str, pos)
   return createStructuredCommand("cites", {}, refs, pos)
 end
 
+--- A sandboxed loadfile implementation.
+--
+-- Load and run a Lua file in a restricted environment.
+--
+-- @tparam string filename File name
+-- @tparam[opt] table env Additional environment entries
+-- @treturn unknown|nil Loaded chunk
+-- @treturn string|nil Error message
+local function sandboxedLoadfile(filename, env)
+  local envbase = {
+    -- Handy for debugging: print, SU logging functions, pl.pretty.dump.
+    -- Handy for table and string manipulations: table, pl.tablex, string, pl.stringx, pl.List, pl.Map, pl.Set.
+    print = print,
+    SU = {
+      debug = SU.debug,
+      error = SU.error,
+      warn = SU.warn,
+    },
+    pl = {
+      pretty = {
+        dump = function (data) pl.pretty.dump(data) end -- To avoid the second unsafe argument
+      },
+      tablex = pl.tablex,
+      stringx = pl.stringx,
+      List = pl.List,
+      Map = pl.Map,
+      Set = pl.Set,
+    },
+    table = table,
+    string = string,
+    -- And a few basic safe functions...
+    math = math,
+    ipairs = ipairs,
+    pairs = pairs,
+    type = type,
+    tostring = tostring,
+    tonumber = tonumber,
+    next = next,
+    error = error,
+    pcall = pcall,
+  }
+  env = pl.tablex.union(envbase, env or {}, true)
+  local f, err
+  -- Load in a sandboxed environment:
+  -- Strategies differ between Lua 5.1 and later versions.
+  if _VERSION == "Lua 5.1" then
+    f, err = loadfile(filename)
+    if not f then
+      return nil, err
+    end
+    -- luacheck: push globals setfenv
+    setfenv(f, env)
+    -- luacheck: pop
+  else
+    f, err = loadfile(filename, "t", env)
+    if not f then
+      return nil, err
+    end
+  end
+  -- Run the chunk in protected mode
+  local ok, res = pcall(f)
+  if not ok then
+    return nil, res end
+  return res
+end
+
 --- @export
 return {
   getFileExtension = getFileExtension,
@@ -152,4 +229,5 @@ return {
   hasEmbedHandler = hasEmbedHandler,
   computeBaselineRatio = computeBaselineRatio,
   naiveCitations = naiveCitations,
+  sandboxedLoadfile = sandboxedLoadfile,
 }