Merge branch 'main' into main

.github/workflows/tests.yml

@@ -126,6 +126,28 @@ jobs:
           PIP_CONSTRAINT= hatch env run -e test -- pip install 'pip>=24.2'
           xvfb-run --auto-servernum hatch run test:nowarn || xvfb-run --auto-servernum hatch run test:nowarn --lf
 
+  test_mistune_30:
+    name: Test Mistune 3.0
+    timeout-minutes: 20
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Base Setup
+        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install texlive-plain-generic inkscape texlive-xetex latexmk
+          sudo apt-get install xvfb x11-utils libxkbcommon-x11-0  libxcb-xinerama0 python3-pyqt5
+
+          # pandoc is not up to date in the ubuntu repos, so we install directly
+          wget https://github.com/jgm/pandoc/releases/download/2.9.2.1/pandoc-2.9.2.1-1-amd64.deb && sudo dpkg -i pandoc-2.9.2.1-1-amd64.deb
+
+      - name: Run tests
+        run: |
+          hatch env run -e test -- pip install 'mistune~=3.0.0'
+          xvfb-run --auto-servernum hatch run test:nowarn || xvfb-run --auto-servernum hatch run test:nowarn --lf
+
   test_prereleases:
     name: Test Prereleases
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -2,6 +2,35 @@
 
 <!-- <START NEW CHANGELOG ENTRY> -->
 
+## 7.16.5
+
+([Full Changelog](https://github.com/jupyter/nbconvert/compare/v7.16.4...18e10f6484afdfee39105248c28abe5ef0835b22))
+
+### Enhancements made
+
+- Add support for mistune 3.1.0 [#2199](https://github.com/jupyter/nbconvert/pull/2199) ([@fcollonval](https://github.com/fcollonval))
+- Allow including `text/x-rst` outputs in rst conversion, transition away from `text/restructuredtext` [#2167](https://github.com/jupyter/nbconvert/pull/2167) ([@takluyver](https://github.com/takluyver))
+
+### Bugs fixed
+
+- Do not display mathjax overlay [#2181](https://github.com/jupyter/nbconvert/pull/2181) ([@timkpaine](https://github.com/timkpaine))
+- Work around pip 24.1 bug which prevents installing pandocfilters 1.4.1 [#2168](https://github.com/jupyter/nbconvert/pull/2168) ([@takluyver](https://github.com/takluyver))
+- Don't die if template path cannot be read [#2162](https://github.com/jupyter/nbconvert/pull/2162) ([@stuaxo](https://github.com/stuaxo))
+- Fix markdown2asciidoc function for pandoc >= 3.0 (closes #2017) [#2152](https://github.com/jupyter/nbconvert/pull/2152) ([@thomasjm](https://github.com/thomasjm))
+
+### Maintenance and upkeep improvements
+
+- enhancement dep-chain: directly depend on bleach\[css\], instead of pulling in tinycss2. [#2166](https://github.com/jupyter/nbconvert/pull/2166) ([@xiacunshun](https://github.com/xiacunshun))
+- chore: update pre-commit hooks [#2146](https://github.com/jupyter/nbconvert/pull/2146) ([@pre-commit-ci](https://github.com/pre-commit-ci))
+
+### Contributors to this release
+
+([GitHub contributors page for this release](https://github.com/jupyter/nbconvert/graphs/contributors?from=2024-04-29&to=2025-01-02&type=c))
+
+[@bollwyvl](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Abollwyvl+updated%3A2024-04-29..2025-01-02&type=Issues) | [@fcollonval](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Afcollonval+updated%3A2024-04-29..2025-01-02&type=Issues) | [@krassowski](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Akrassowski+updated%3A2024-04-29..2025-01-02&type=Issues) | [@pre-commit-ci](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Apre-commit-ci+updated%3A2024-04-29..2025-01-02&type=Issues) | [@stuaxo](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Astuaxo+updated%3A2024-04-29..2025-01-02&type=Issues) | [@t-makaro](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3At-makaro+updated%3A2024-04-29..2025-01-02&type=Issues) | [@takluyver](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Atakluyver+updated%3A2024-04-29..2025-01-02&type=Issues) | [@thomasjm](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Athomasjm+updated%3A2024-04-29..2025-01-02&type=Issues) | [@timkpaine](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Atimkpaine+updated%3A2024-04-29..2025-01-02&type=Issues) | [@xiacunshun](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Axiacunshun+updated%3A2024-04-29..2025-01-02&type=Issues)
+
+<!-- <END NEW CHANGELOG ENTRY> -->
+
 ## 7.16.4
 
 ([Full Changelog](https://github.com/jupyter/nbconvert/compare/v7.16.3...71fde294f623083f364bf68e14f07459dee952e6))
@@ -22,8 +51,6 @@
 
 [@blink1073](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Ablink1073+updated%3A2024-03-21..2024-04-29&type=Issues) | [@mgeier](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Amgeier+updated%3A2024-03-21..2024-04-29&type=Issues) | [@pre-commit-ci](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Apre-commit-ci+updated%3A2024-03-21..2024-04-29&type=Issues)
 
-<!-- <END NEW CHANGELOG ENTRY> -->
-
 ## 7.16.3
 
 ([Full Changelog](https://github.com/jupyter/nbconvert/compare/v7.16.2...9019037e300dfdbac5a8cb330d0b09eb82e12f62))

docs/source/nbconvert_library.ipynb

@@ -486,7 +486,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "@damianavila wrote the Nikola Plugin to [write blog post as Notebooks](http://damianavila.github.io/blog/posts/one-line-deployment-of-your-site-to-gh-pages.html) and is developing a js-extension to publish notebooks via one click from the web app."
+    "@damianavila wrote the Nikola Plugin to [write blog post as Notebooks](https://damianavila.github.io/blog/posts/one-line-deployment-of-your-site-to-gh-pages) and is developing a js-extension to publish notebooks via one click from the web app."
    ]
   }
  ],

nbconvert/_version.py

@@ -4,7 +4,7 @@
 from typing import List
 
 # Version string must appear intact for versioning
-__version__ = "7.16.4"
+__version__ = "7.16.5"
 
 # Build up version_info tuple for backwards compatibility
 pattern = r"(?P<major>\d+).(?P<minor>\d+).(?P<patch>\d+)(?P<rest>.*)"

nbconvert/exporters/html.py

@@ -15,6 +15,7 @@
 from bs4 import BeautifulSoup
 from jupyter_core.paths import jupyter_path
 from traitlets import Bool, Unicode, default, validate
+from traitlets import Dict as TraitletsDict
 from traitlets.config import Config
 
 if tuple(int(x) for x in jinja2.__version__.split(".")[:3]) < (3, 0, 0):
@@ -183,6 +184,14 @@ def _template_name_default(self):
 
     output_mimetype = "text/html"
 
+    lexer_options = TraitletsDict(
+        {},
+        help=(
+            "Options to be passed to the pygments lexer for highlighting markdown code blocks. "
+            "See https://pygments.org/docs/lexers/#available-lexers for available options."
+        ),
+    ).tag(config=True)
+
     @property
     def default_config(self):
         c = Config(
@@ -239,6 +248,7 @@ def markdown2html(self, context, source):
             path=path,
             anchor_link_text=self.anchor_link_text,
             exclude_anchor_links=self.exclude_anchor_links,
+            **self.lexer_options,
         )
         return MarkdownWithMath(renderer=renderer).render(source)
 

nbconvert/exporters/templateexporter.py

@@ -151,7 +151,11 @@ class TemplateExporter(Exporter):
     """
 
     # finish the docstring
-    __doc__ = __doc__.format(filters="- " + "\n    - ".join(sorted(default_filters.keys())))
+    __doc__ = (
+        __doc__.format(filters="- " + "\n    - ".join(sorted(default_filters.keys())))
+        if __doc__
+        else None
+    )
 
     _template_cached = None
 

nbconvert/filters/highlight.py

@@ -136,7 +136,9 @@ def __call__(self, source, language=None, metadata=None, strip_verbatim=False):
         return latex
 
 
-def _pygments_highlight(source, output_formatter, language="ipython", metadata=None):
+def _pygments_highlight(
+    source, output_formatter, language="ipython", metadata=None, **lexer_options
+):
     """
     Return a syntax-highlighted version of the input source
 
@@ -149,6 +151,10 @@ def _pygments_highlight(source, output_formatter, language="ipython", metadata=N
         language to highlight the syntax of
     metadata : NotebookNode cell metadata
         metadata of the cell to highlight
+    lexer_options : dict
+        Options to pass to the pygments lexer. See
+        https://pygments.org/docs/lexers/#available-lexers for more information about
+        valid lexer options
     """
     from pygments import highlight
     from pygments.lexers import get_lexer_by_name
@@ -179,7 +185,7 @@ def _pygments_highlight(source, output_formatter, language="ipython", metadata=N
 
     if lexer is None:
         try:
-            lexer = get_lexer_by_name(language, stripall=True)
+            lexer = get_lexer_by_name(language, **lexer_options)
         except ClassNotFound:
             warn("No lexer found for language %r. Treating as plain text." % language, stacklevel=2)
             from pygments.lexers.special import TextLexer

nbconvert/filters/markdown_mistune.py

@@ -9,7 +9,7 @@
 import mimetypes
 import os
 from html import escape
-from typing import Any, Callable, Dict, Iterable, Match, Optional, Tuple
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, Iterable, Match, Optional, Protocol, Tuple
 
 import bs4
 from pygments import highlight
@@ -20,6 +20,19 @@
 
 from nbconvert.filters.strings import add_anchor
 
+if TYPE_CHECKING:
+    try:
+        from mistune.plugins import Plugin
+    except ImportError:
+
+        class Plugin(Protocol):  # type: ignore[no-redef]
+            """Mistune plugin interface."""
+
+            def __call__(self, markdown: "Markdown") -> None:
+                """Apply the plugin on the markdown document."""
+                ...
+
+
 try:  # for Mistune >= 3.0
     from mistune import (  # type:ignore[attr-defined]
         BlockParser,
@@ -32,6 +45,7 @@
     )
 
     MISTUNE_V3 = True
+    MISTUNE_V3_ATX = "atx_heading" in BlockParser.SPECIFICATION
 
 except ImportError:  # for Mistune >= 2.0
     import re
@@ -45,8 +59,9 @@
     )
 
     MISTUNE_V3 = False
+    MISTUNE_V3_ATX = False
 
-    def import_plugin(name: str) -> "MarkdownPlugin":  # type: ignore[misc]
+    def import_plugin(name: str) -> "Plugin":  # type: ignore[misc]
         """Simple implementation of Mistune V3's import_plugin for V2."""
         return PLUGINS[name]  # type: ignore[no-any-return]
 
@@ -75,8 +90,10 @@ class MathBlockParser(BlockParser):
         is ignored here.
         """
 
-        AXT_HEADING_WITHOUT_LEADING_SPACES = (
-            r"^ {0,3}(?P<axt_1>#{1,6})(?!#+)(?P<axt_2>[ \t]*(.*?)?)$"
+        ATX_HEADING_WITHOUT_LEADING_SPACES = (
+            r"^ {0,3}(?P<atx_1>#{1,6})(?!#+)(?P<atx_2>[ \t]*(.*?)?)$"
+            if MISTUNE_V3_ATX
+            else r"^ {0,3}(?P<axt_1>#{1,6})(?!#+)(?P<axt_2>[ \t]*(.*?)?)$"
         )
 
         MULTILINE_MATH = _dotall(
@@ -92,12 +109,14 @@ class MathBlockParser(BlockParser):
 
         SPECIFICATION = {
             **BlockParser.SPECIFICATION,
-            "axt_heading": AXT_HEADING_WITHOUT_LEADING_SPACES,
+            (
+                "atx_heading" if MISTUNE_V3_ATX else "axt_heading"
+            ): ATX_HEADING_WITHOUT_LEADING_SPACES,
             "multiline_math": MULTILINE_MATH,
         }
 
         # Multiline math must be searched before other rules
-        DEFAULT_RULES: Tuple[str, ...] = ("multiline_math", *BlockParser.DEFAULT_RULES)  # type: ignore[assignment]
+        DEFAULT_RULES: ClassVar[Iterable[str]] = ("multiline_math", *BlockParser.DEFAULT_RULES)  # type: ignore[assignment]
 
         def parse_multiline_math(self, m: Match[str], state: BlockState) -> int:
             """Send mutiline math as a single paragraph to MathInlineParser."""
@@ -139,7 +158,7 @@ class MathInlineParser(InlineParser):
         }
 
         # Block math must be matched first, and all math must come before text
-        DEFAULT_RULES: Tuple[str, ...] = (
+        DEFAULT_RULES: ClassVar[Iterable[str]] = (
             "block_math_tex",
             "block_math_latex",
             "inline_math_tex",
@@ -274,13 +293,15 @@ def __init__(
         anchor_link_text: str = "¶",
         path: str = "",
         attachments: Optional[Dict[str, Dict[str, str]]] = None,
+        **lexer_options,
     ):
         """Initialize the renderer."""
         super().__init__(escape, allow_harmful_protocols)
         self.embed_images = embed_images
         self.exclude_anchor_links = exclude_anchor_links
         self.anchor_link_text = anchor_link_text
         self.path = path
+        self.lexer_options = lexer_options
         if attachments is not None:
             self.attachments = attachments
         else:
@@ -298,7 +319,7 @@ def block_code(self, code: str, info: Optional[str] = None) -> str:
             try:
                 if info.strip().split(None, 1):
                     lang = info.strip().split(maxsplit=1)[0]
-                    lexer = get_lexer_by_name(lang, stripall=True)
+                    lexer = get_lexer_by_name(lang, **self.lexer_options)
             except ClassNotFound:
                 code = f"{lang}\n{code}"
                 lang = None
@@ -442,10 +463,6 @@ def _html_embed_images(self, html: str) -> str:
         return str(parsed_html)
 
 
-# Represents an already imported plugin for Mistune
-MarkdownPlugin = Callable[[Markdown], None]
-
-
 class MarkdownWithMath(Markdown):
     """Markdown text with math enabled."""
 
@@ -464,7 +481,7 @@ def __init__(
         renderer: HTMLRenderer,
         block: Optional[BlockParser] = None,
         inline: Optional[InlineParser] = None,
-        plugins: Optional[Iterable[MarkdownPlugin]] = None,
+        plugins: Optional[Iterable["Plugin"]] = None,
     ):
         """Initialize the parser."""
         if block is None:

pyproject.toml

@@ -224,7 +224,6 @@ ignore = [
   "T201",   # `print` found
   "RUF012", # Mutable class attributes should be annotated
   "UP031",  # Use format specifiers instead of percent format
-
 ]
 unfixable = [
   "T201",   # Don't touch print statements
@@ -253,6 +252,7 @@ unfixable = [
 "nbconvert/__init__.py" = ["F401", "F403"]
 # PLR2004 Magic value used in comparison
 "nbconvert/filters/ansi.py" = ["PLR2004"]
+"tests/exporters/test_html.py" = ["RUF001"]
 
 [tool.interrogate]
 ignore-init-module=true

share/templates/base/mathjax.html.j2

@@ -21,6 +21,7 @@
                     processEnvironments: true
                 },
                 displayAlign: 'center',
+                messageStyle: 'none',
                 CommonHTML: {
                     linebreaks: {
                     automatic: true

share/templates/latex/base.tex.j2

@@ -164,7 +164,26 @@ override this.-=))
     \newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.49,0.56,0.16}{{#1}}}
     \newcommand{\InformationTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}}
     \newcommand{\WarningTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}}
-
+    \makeatletter
+    \newsavebox\pandoc@box
+    \newcommand*\pandocbounded[1]{%
+      \sbox\pandoc@box{#1}%
+      % scaling factors for width and height
+      \Gscale@div\@tempa\textheight{\dimexpr\ht\pandoc@box+\dp\pandoc@box\relax}%
+      \Gscale@div\@tempb\linewidth{\wd\pandoc@box}%
+      % select the smaller of both
+      \ifdim\@tempb\p@<\@tempa\p@
+        \let\@tempa\@tempb
+      \fi
+      % scaling accordingly (\@tempa < 1)
+      \ifdim\@tempa\p@<\p@
+        \scalebox{\@tempa}{\usebox\pandoc@box}%
+      % scaling not needed, use as it is
+      \else
+        \usebox{\pandoc@box}%
+      \fi
+    }
+    \makeatother
 
     % Define a nice break command that doesn't care if a line doesn't already
     % exist.