Support Pandoc floating tables and native two-column table layouts

configuration

@@ -13,7 +13,7 @@
 export DENO=v2.7.14
 # TODO figure out where 0.1.41 apple silicon libs are available
 export DENO_DOM=v0.1.41-alpha-artifacts
-export PANDOC=3.10
+export PANDOC=3.10.1
 export DARTSASS=1.101.0
 export ESBUILD=0.28.1
 export TYPST=0.15.1

news/changelog-1.11.md

@@ -12,6 +12,8 @@ All changes included in 1.11:
 
 ## Dependencies
 
+- Update Pandoc to 3.10.1.
+
 - ([#15043](https://github.com/quarto-dev/quarto-cli/issues/15043)): Update the OJS runtime to acorn 8.17, Observable Inspector 5.0.1, and Observable Inputs 0.10.6. `FileAttachment(...).mimeType` now reports `text/javascript` instead of `application/javascript` for `.js` and `.mjs` files.
 
 ## Accessibility
@@ -79,6 +81,8 @@ All changes included in 1.11:
 
 ### `pdf`
 
+- ([#14846](https://github.com/quarto-dev/quarto-cli/issues/14846)): Use Pandoc's floating tables for ordinary Markdown tables in documents with `classoption: twocolumn`, including tables with cross-references. Single-column documents retain `longtable` by default and can opt into floating tables with `.float`. Respect `tbl-pos` unless `latex-placement` is explicitly set. Render full-width, margin, bottom-captioned, and panel tables without `longtable`, retaining Pandoc's widths, spans, citations, and footnotes. Preserve native double-column `table*` and `figure*` environments when loading `sidenotes`, including documents that switch columns in the body.
+
 - ([#14982](https://github.com/quarto-dev/quarto-cli/issues/14982)): Fix a footnote in document metadata, such as `title` or `author`, making PDF and Typst output shift heading levels, which turned the first level-1 heading into the document title.
 - ([#15027](https://github.com/quarto-dev/quarto-cli/issues/15027)): Fix PDF rendering with regional German variants such as `lang: de-CH` failing when German hyphenation patterns are not installed, instead of installing them automatically.
 - ([#15035](https://github.com/quarto-dev/quarto-cli/issues/15035)): Fix automatic installation of missing hyphenation patterns for the pinned German patterns (`ngerman-x-latest`), for several languages at once, and for languages loaded with `polyglossia`.

src/command/check/check.ts

@@ -300,7 +300,7 @@ async function checkVersions(conf: CheckConfiguration) {
   // file is in an awkward format and it is not packaged
   // with our installers
   const versionConstraints: [string | undefined, string, string][] = [
-    [pandocVersion, "3.10.0", "Pandoc"],
+    [pandocVersion, "3.10.1", "Pandoc"],
     [sassVersion, "1.101.0", "Dart Sass"],
     [denoVersion, "2.7.14", "Deno"],
     [typstVersion, "0.15.1", "Typst"],

src/resources/filters/customnodes/floatreftarget.lua

@@ -326,7 +326,8 @@ end, function(float)
       return {}
     end
     caption_cmd_name = "subcaption"
-  elseif float.content.t == "Table" and float_type == "tbl" then -- float.parent_id is nil here
+  elseif float.content.t == "Table" and float_type == "tbl" and
+      figEnv == "table" and capLoc == "top" then -- float.parent_id is nil here
     -- special-case the situation where the figure is Table and the content is Table
     --
     -- just return the table itself with the caption inside the table
@@ -336,9 +337,24 @@ end, function(float)
     caption_cmd_name = "caption"
     float.content.caption.long = float.caption_long
     float.content.attr = pandoc.Attr(float.identifier, float.classes or {}, float.attributes or {})
+    if float.content.classes:includes("float") and figPos and
+        float.content.attributes["latex-placement"] == nil then
+      float.content.attributes["latex-placement"] = figPos
+    end
     return float.content
   end
 
+  -- Only floating tables need Quarto's environment and bottom caption here.
+  -- Ordinary single-column tables must retain longtable and its caption foot.
+  if not float.parent_id and float.content.t == "Table" and float_type == "tbl" and
+      (figEnv ~= "table" or float.classes:includes("float")) then
+    figPos = float.attributes["latex-placement"] or figPos
+    if figEnv == "marginfigure" then
+      figEnv = "margintable"
+    end
+    float.content = latexTabular(float.content)
+  end
+
   local fig_scap = attribute(float, kFigScap, nil)
   if fig_scap then
     fig_scap = pandoc.Span(markdownToInlines(fig_scap))
@@ -400,6 +416,9 @@ end, function(float)
       end
     end,
     Table = function(tbl)
+      if tbl.classes:includes("float") then
+        return latexTabular(tbl)
+      end
       local cites = pandoc.List({})
       local guid_id = global_table_guid_id
       local uuid = "85b77c8a-261c-4f58-9b04-f21c67e0a758"
@@ -646,13 +665,13 @@ end, function(float)
     -- the panel layout code
     return figure_content
   else
-    return quarto.LatexEnvironment({
+    return latexTableNoteEnvironment(quarto.LatexEnvironment({
       name = figEnv,
       pos = figPos,
       content = _quarto.ast.walk(figure_content, {
         Image = drop_class("column-margin")
       })
-    })
+    }))
   end
 end)
 

src/resources/filters/layout/latex.lua

@@ -8,14 +8,14 @@ end, function (layout)
   local rendered_panel = latexPanel(layout)
   local preamble = layout.preamble
   if preamble == nil then
-    return rendered_panel
+    return latexTableNoteEnvironment(rendered_panel)
   end
 
   local result = pandoc.Blocks({})
   panel_insert_preamble(result, preamble)
   result:insert(rendered_panel)
 
-  return result
+  return latexTableNoteEnvironment(pandoc.Div(result))
 end)
 
 _quarto.ast.add_renderer("PanelLayout", function(_)
@@ -365,6 +365,16 @@ function latexCell(cell, vAlign, endOfRow, endOfTable)
   end
 
 
+  -- Unlabelled tables in panels also need a non-floating tabular. Otherwise
+  -- Pandoc emits longtable inside the minipage, which still fails in twocolumn.
+  if not isSubRef and tbl then
+    cell.content = _quarto.ast.walk(cell.content, {
+      Table = function(table)
+        return latexTabular(table, vAlign)
+      end
+    })
+  end
+
   -- convert to latex percent as necessary
   width = asLatexSize(width)
 
@@ -444,47 +454,87 @@ function latexCell(cell, vAlign, endOfRow, endOfTable)
 end
 
 function latexTabular(tbl, vAlign)
-
-  -- convert to simple table
-  tbl = pandoc.utils.to_simple_table(tbl)
-
-  -- list of inlines
-  local tabular = pandoc.List()
-
-  -- vertically align the minipage
-  local tabularVAlign = latexMinipageValign(vAlign)
-
-  -- caption
-  if #tbl.caption > 0 then
-    latexAppend(tabular, "\\caption{")
-    tappend(tabular, tbl.caption)
-    latexAppend(tabular, "}\n")
+  -- Let Pandoc serialize the full Table, including widths, spans, and blocks.
+  -- Only remove its outer float: Quarto's layout owns the enclosing environment.
+  tbl = tbl:clone()
+  local has_notes = false
+  _quarto.ast.walk(tbl, {Note = function() has_notes = true end})
+  local caption = quarto.utils.as_inlines(tbl.caption.long)
+  tbl.caption = pandoc.Caption()
+  tbl.identifier = ""
+  if not tbl.classes:includes("float") then
+    tbl.classes:insert("float")
+  end
+  quarto.doc.use_latex_package("multirow")
+  local content = latexWriteTable(tbl, function(text)
+    local opens, closes
+    text, opens = text:gsub("\\begin{table}%b[]%s*\\centering%s*", "", 1)
+    text, closes = text:gsub("\\end{tabular}%s*\\end{table}", "\\end{tabular}", 1)
+    assert(opens == 1 and closes == 1, "Could not unwrap Pandoc's floating table")
+    text = text:gsub("\\begin{tabular}", "\\begin{tabular}" .. latexMinipageValign(vAlign), 1)
+    return text
+  end)
+  if has_notes then
+    quarto.doc.use_latex_package("footnotehyper")
+    content = pandoc.Div({content}, pandoc.Attr("", {"quarto-table-notes"}))
+  end
+  if caption and #caption > 0 then
+    return pandoc.Div({
+      quarto.LatexInlineCommand({name = "caption", arg = pandoc.Span(caption)}),
+      content
+    })
   end
-
-  -- header
-  local aligns = table.concat(tbl.aligns:map(latexTabularAlign), "")
-  latexAppend(tabular, "\\begin{tabular}" .. tabularVAlign .. "{" .. aligns .. "}\n")
-  latexAppend(tabular, "\\toprule\n")
-
-  -- headers (optional)
-  local headers = latexTabularRow(tbl.headers)
-  if latexTabularRowHasContent(headers) then
-    latexTabularRowAppend(tabular, headers)
-    latexAppend(tabular, "\\midrule\n")
+  return content
+end
+
+-- Footnote text emitted while constructing a float would otherwise be lost.
+-- Collect it around the outermost float/panel and release it afterwards.
+function latexTableNoteEnvironment(content)
+  local has_notes = false
+  _quarto.ast.walk(content, {
+    Div = function(div)
+      if div.classes:includes("quarto-table-notes") then
+        has_notes = true
+      end
+    end
+  })
+  if has_notes then
+    return quarto.LatexEnvironment({name = "savenotes", content = content})
+  end
+  return content
+end
+
+-- Keep citations in the AST for citeproc/natbib processing even when a table
+-- must be serialized before the document's writer runs.
+function latexWriteTable(tbl, transform)
+  local cites = pandoc.List({})
+  local uuid = "85b77c8a-261c-4f58-9b04-f21c67e0a758"
+  local first_id = global_table_guid_id
+  tbl = _quarto.ast.walk(tbl, {
+    Cite = function(cite)
+      cites:insert(cite)
+      return pandoc.Str(uuid .. "-" .. (first_id + #cites) .. "-" .. uuid)
+    end
+  })
+  local text = pandoc.write(pandoc.Pandoc({tbl}), "latex")
+  if transform then
+    text = transform(text)
   end
-
-  -- rows
-  for _,row in ipairs(tbl.rows) do
-    latexTabularRowAppend(tabular, latexTabularRow(row))
+  local raw = pandoc.RawBlock("latex", text)
+  if #cites == 0 then
+    return raw
   end
-
-  -- footer
-  latexAppend(tabular, "\\bottomrule\n")
-  latexAppend(tabular, "\\end{tabular}")
-
-  -- return tabular
-  return pandoc.Para(tabular)
-
+  global_table_guid_id = first_id + #cites
+  local result = pandoc.Blocks({})
+  for i, cite in ipairs(cites) do
+    result:insert(_quarto.ast.make_scaffold(pandoc.Span, pandoc.Inlines({
+      pandoc.RawInline("latex", "%quarto-define-uuid: " .. uuid .. "-" .. (first_id + i) .. "-" .. uuid .. "\n"),
+      cite,
+      pandoc.RawInline("latex", "\n%quarto-end-define-uuid\n")
+    })))
+  end
+  result:insert(raw)
+  return pandoc.Div(result)
 end
 
 function latexTabularRow(row)

src/resources/filters/layout/meta.lua

@@ -106,9 +106,44 @@ function layout_meta_inject_latex_packages()
       if (layoutState.hasColumns or marginReferences() or marginCitations()) and _quarto.format.isLatexOutput() then
         -- inject sidenotes package
         metaInjectLatex(meta, function(inject)
-          inject(
-            usePackage("sidenotes")
-          )
+          -- sidenotes redefines table*/figure* as single-column floats widened
+          -- into the margin. In twocolumn documents these must remain native
+          -- double-column floats. Choose at use time, since journal styles may
+          -- switch columns in the document rather than via classoption.
+          inject([[
+% Install the dispatchers only when Quarto loads sidenotes itself. If the
+% document already loaded it, leave that existing package setup untouched.
+\@ifpackageloaded{sidenotes}{}{%
+  % Save the document class's native double-column float environments before
+  % sidenotes replaces them. Save both the opening and closing commands.
+  % csname addresses environment commands whose names contain a literal '*';
+  % expandafter lets \let copy the resulting command's current definition.
+  \expandafter\let\expandafter\quarto@native@table\csname table*\endcsname
+  \expandafter\let\expandafter\quarto@native@endtable\csname endtable*\endcsname
+  \expandafter\let\expandafter\quarto@native@figure\csname figure*\endcsname
+  \expandafter\let\expandafter\quarto@native@endfigure\csname endfigure*\endcsname
+  \usepackage{sidenotes}
+  % Keep sidenotes' versions too: in single-column documents they extend a
+  % regular float into the margin, which is the intended column-page layout.
+  \expandafter\let\expandafter\quarto@margin@table\csname table*\endcsname
+  \expandafter\let\expandafter\quarto@margin@endtable\csname endtable*\endcsname
+  \expandafter\let\expandafter\quarto@margin@figure\csname figure*\endcsname
+  \expandafter\let\expandafter\quarto@margin@endfigure\csname endfigure*\endcsname
+  % Select the implementation when each environment is used, not when the
+  % preamble is read. A journal style or the document body may call
+  % \twocolumn or \onecolumn after sidenotes has already been loaded.
+  % Expand past \else or \fi before calling an opening command so its
+  % optional placement argument (such as [t]) is still the next input token.
+  \expandafter\def\csname table*\endcsname{%
+    \if@twocolumn\expandafter\quarto@native@table\else\expandafter\quarto@margin@table\fi}
+  \expandafter\def\csname endtable*\endcsname{%
+    \if@twocolumn\quarto@native@endtable\else\quarto@margin@endtable\fi}
+  \expandafter\def\csname figure*\endcsname{%
+    \if@twocolumn\expandafter\quarto@native@figure\else\expandafter\quarto@margin@figure\fi}
+  \expandafter\def\csname endfigure*\endcsname{%
+    \if@twocolumn\quarto@native@endfigure\else\quarto@margin@endfigure\fi}
+}
+]])
           inject(
             usePackage("marginnote")
           )

src/resources/filters/main.lua

@@ -567,6 +567,10 @@ local quarto_post_filters = {
     filter = render_typst_brand_yaml(),
     traverser = 'jog',
   },
+  { name = "post-render-markdown-code-languages",
+    filter = preserve_markdown_code_languages(),
+    traverser = 'jog',
+  },
   { name = "post-render-gfm-fixups",
     filter = render_gfm_fixups(),
     traverser = 'jog',
@@ -638,6 +642,7 @@ local quarto_finalize_filters = {
 }
 
 local quarto_layout_filters = {
+  { name = "layout-latex-table-floats", filter = latex_table_floats() },
   { name = "manuscript filtering",
     filter = manuscript(),
     traverser = 'jog',

src/resources/filters/quarto-post/code.lua

@@ -35,4 +35,33 @@ function removeCodeOptions()
       end
     end
   }
-end
+end
+
+function preserve_markdown_code_languages()
+  return {
+    CodeBlock = function(code)
+      if not _quarto.format.isMarkdownOutput() or
+          tcontains(PANDOC_WRITER_OPTIONS.extensions, "fenced_code_attributes") then
+        return nil
+      end
+      -- Since Pandoc 3.10.1, bare classes are only treated as languages if
+      -- Skylighting knows them. Explicit language-* classes still preserve
+      -- arbitrary names, including those interpreted by downstream renderers.
+      -- Match the previous writer's first non-sourceCode class convention,
+      -- while respecting an already-explicit language anywhere in the list.
+      local language
+      for _, class in ipairs(code.classes) do
+        if class:match("^language%-") then
+          return nil
+        end
+        if language == nil and class ~= "sourceCode" then
+          language = class
+        end
+      end
+      if language then
+        code.classes:insert("language-" .. language)
+        return code
+      end
+    end
+  }
+end