diff --git a/docs/syntax/optional.md b/docs/syntax/optional.md index c8b9f24c..55bb9301 100644 --- a/docs/syntax/optional.md +++ b/docs/syntax/optional.md @@ -539,13 +539,15 @@ The slug generation function is selected with the `myst_heading_slug_func` optio The recommended form is a named preset: ```python -myst_heading_slug_func = "github" # the default; also available: "gitlab" +myst_heading_slug_func = "github" # the default; also available: "gitlab", "docutils" ``` The `"gitlab"` preset follows [GitLab's algorithm](https://docs.gitlab.com/ee/user/markdown.html#heading-ids-and-links) (which, unlike GitHub, strips surrounding whitespace, squeezes repeated hyphens and prefixes digit-only slugs with `anchor-`). +The `"docutils"` preset produces slugs byte-compatible with `docutils.nodes.make_id` — the `id` style reStructuredText headings receive — for a uniform anchor style across mixed rST + Markdown projects; its slugs are ASCII-only, so a non-Latin title produces no anchor (see the anchor contract below). + :::{versionchanged} 5.2.0 -`myst_heading_slug_func` now accepts the preset names `"github"` (default) and `"gitlab"`. +`myst_heading_slug_func` now accepts the preset names `"github"` (default), `"gitlab"` and `"docutils"`. As a legacy form, it can still be set to an import path string (e.g. `myst_heading_slug_func = "mypackage.mymodule.slugify"`, added in 0.19.0) @@ -574,6 +576,41 @@ $ myst-anchors -l 2 --slug-func github docs/syntax/optional.md

``` +Note this is a quick preview using the underlying markdown-it plugin: +it does not apply the renderer-only empty-slug rule described below, +so a heading whose slug is empty is shown with a literal empty (or suffixed) id, +where the renderer would emit no anchor at all. + +### The anchor contract + +:::{versionadded} 5.2.0 +::: + +- **Slug generation**: anchors are computed by the configured slug function. + The named presets are specified by `myst_parser/slugs.py` (pure and + standard-library-only, so alternative implementations can port them from that + one file) and pinned by the machine-readable corpus `tests/fixtures/slugs.json` + (versioned via a `version` field; append-only within a version, with a version + bump on any semantic change). +- **Deduplication**: repeated slugs are given fixed-base suffixes `slug`, + `slug-1`, `slug-2`, ... by `unique_slug` — the base is never mutated. This + applies whatever the preset; in particular docutils' own duplicate-id scheme is + *not* emulated by the `docutils` preset, which governs the slug text only. +- **Empty slugs**: a slug function may return an empty string (a punctuation-only + title, or any non-Latin title under `docutils`); such a heading gets no anchor + and takes no part in deduplication. +- **Explicit-id priority**: an explicit target/id on a heading beats the + auto-slug as the primary id; the superseded slug is kept as a secondary anchor, + so previously published links keep working. +- **HTML id emission**: slugs are emitted as real (secondary) HTML `id`s in the + output, so the anchors exist in the published HTML. The docutils `id_prefix` + setting is deliberately bypassed — the raw slug is the anchor. Set + `myst_heading_anchors_html_ids = False` to disable this. +- **docutils drift policy**: the `docutils` preset is pinned byte-for-byte + against `docutils.nodes.make_id` by CI. If a future docutils changes `make_id`, + the preset and corpus stay byte-stable (they are the contract) and the + divergence will be documented. + (syntax/definition-lists)= ## Definition Lists diff --git a/myst_parser/config/main.py b/myst_parser/config/main.py index 1e203fa6..bd19dc7a 100644 --- a/myst_parser/config/main.py +++ b/myst_parser/config/main.py @@ -313,7 +313,7 @@ def __repr__(self) -> str: "validator": check_heading_slug_func, "help": ( "Function for creating heading anchors: " - 'a preset name ("github", "gitlab"), ' + 'a preset name ("docutils", "github", "gitlab"), ' "or (legacy) a python import path " "e.g. `my_package.my_module.my_function`" ), diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index 164b3ba6..68cfae72 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -877,6 +877,12 @@ def generate_heading_target( append_to=self.current_node, ) else: + if not slug: + # an empty slug (e.g. a punctuation-only title, or a + # non-Latin title under the docutils preset) means the + # heading gets no anchor; it also takes no part in + # deduplication, so a later empty slug is not suffixed -1 + return node["slug"] = slug # note: the slug is additionally emitted as a (secondary) HTML id # by the `AddSlugIds` transform, which runs only after *all* diff --git a/myst_parser/slugs.py b/myst_parser/slugs.py index 5a32c657..52a6ad81 100644 --- a/myst_parser/slugs.py +++ b/myst_parser/slugs.py @@ -3,15 +3,20 @@ This is the single source of truth for heading slugs, shared by the renderer (``myst_heading_anchors``) and the ``myst-anchors`` CLI. -The functions here are pure and dependency-free (standard library ``re`` only), +The functions here are pure and dependency-free (standard library only), so that alternative implementations of MyST can replicate them from this file; for the same reason, the unit-test corpus is kept as a language-agnostic data file (``tests/fixtures/slugs.json``). + +A slug function may return an empty string (e.g. for a punctuation-only +title, or a non-Latin title under the ``docutils`` preset); an empty slug +means the heading gets *no* anchor. """ from __future__ import annotations import re +import unicodedata from collections.abc import Callable, Container _GITHUB_CLEAN = re.compile(r"[^\w\u4e00-\u9fff\- ]") @@ -57,7 +62,82 @@ def gitlab_slugify(title: str) -> str: return slug +# ported verbatim from ``docutils.nodes`` (public domain), so that this +# module stays free of third-party imports; byte-fidelity to +# ``docutils.nodes.make_id`` is pinned by tests/test_slugs.py +_DOCUTILS_DIGRAPHS = { + 0x00DF: "sz", # ß + 0x00E6: "ae", # æ + 0x0153: "oe", # œ + 0x0238: "db", # ȸ + 0x0239: "qp", # ȹ +} +_DOCUTILS_TRANSLATE = { + 0x00F8: "o", # ø + 0x0111: "d", # đ + 0x0127: "h", # ħ + 0x0131: "i", # dotless i + 0x0142: "l", # ł + 0x0167: "t", # ŧ + 0x0180: "b", # ƀ + 0x0183: "b", # ƃ + 0x0188: "c", # ƈ + 0x018C: "d", # ƌ + 0x0192: "f", # ƒ + 0x0199: "k", # ƙ + 0x019A: "l", # ƚ + 0x019E: "n", # ƞ + 0x01A5: "p", # ƥ + 0x01AB: "t", # ƫ + 0x01AD: "t", # ƭ + 0x01B4: "y", # ƴ + 0x01B6: "z", # ƶ + 0x01E5: "g", # ǥ + 0x0225: "z", # ȥ + 0x0234: "l", # ȴ + 0x0235: "n", # ȵ + 0x0236: "t", # ȶ + 0x0237: "j", # ȷ + 0x023C: "c", # ȼ + 0x023F: "s", # ȿ + 0x0240: "z", # ɀ + 0x0247: "e", # ɇ + 0x0249: "j", # ɉ + 0x024B: "q", # ɋ + 0x024D: "r", # ɍ + 0x024F: "y", # ɏ +} +_DOCUTILS_NON_ID = re.compile(r"[^a-z0-9]+") +_DOCUTILS_AT_ENDS = re.compile(r"^[-0-9]+|-+$") + + +def docutils_slugify(title: str) -> str: + """Convert a heading title to a docutils/reStructuredText-style id. + + Byte-identical to ``docutils.nodes.make_id``, for uniform anchors across + mixed rST + Markdown projects. Algorithm: lowercase the title; map the + digraph letters ß æ œ ȸ ȹ and 33 further stroked/hooked Latin letters to + ASCII equivalents; NFKD-normalize and drop every remaining non-ASCII + character; normalize whitespace runs to single spaces (kept for + ``make_id`` step-parity, though subsumed by the next step); collapse each + run of characters outside ``[a-z0-9]`` to a single hyphen; strip leading + digits/hyphens and trailing hyphens. + + The result matches ``[a-z](-?[a-z0-9]+)*`` or is *empty* — the latter for + any title with no Latin letters (so such headings get no anchor), and + docutils' own duplicate-id handling (``idN``) is not emulated: + deduplication is governed by :func:`unique_slug`, whatever the preset. + """ + slug = title.lower() + slug = slug.translate(_DOCUTILS_DIGRAPHS) + slug = slug.translate(_DOCUTILS_TRANSLATE) + slug = unicodedata.normalize("NFKD", slug).encode("ascii", "ignore").decode() + slug = _DOCUTILS_NON_ID.sub("-", " ".join(slug.split())) + return _DOCUTILS_AT_ENDS.sub("", slug) + + SLUG_PRESETS: dict[str, Callable[[str], str]] = { + "docutils": docutils_slugify, "github": github_slugify, "gitlab": gitlab_slugify, } diff --git a/tests/fixtures/slugs.json b/tests/fixtures/slugs.json index 9eeb9f2b..fe4de410 100644 --- a/tests/fixtures/slugs.json +++ b/tests/fixtures/slugs.json @@ -1,4 +1,6 @@ { + "version": 1, + "description": "Normative conformance corpus for MyST heading-anchor slugs. 'slugify' rows pin SLUG_PRESETS[preset](input) == expected, byte-for-byte; 'unique' rows pin unique_slug(slug, existing) == expected. The set is append-only within a version; any semantic change to an existing vector or algorithm requires a version bump. An empty expected slug means the heading receives no anchor.", "slugify": [ { "preset": "github", @@ -104,6 +106,76 @@ "preset": "gitlab", "input": "!!!", "expected": "" + }, + { + "preset": "docutils", + "input": "Ubuntu 20.04", + "expected": "ubuntu-20-04" + }, + { + "preset": "docutils", + "input": "lxc.env for environment variables", + "expected": "lxc-env-for-environment-variables" + }, + { + "preset": "docutils", + "input": "2.0", + "expected": "" + }, + { + "preset": "docutils", + "input": "3rd party", + "expected": "rd-party" + }, + { + "preset": "docutils", + "input": "Привет Мир", + "expected": "" + }, + { + "preset": "docutils", + "input": "中文标题", + "expected": "" + }, + { + "preset": "docutils", + "input": " a b c ", + "expected": "a-b-c" + }, + { + "preset": "docutils", + "input": "Title with UPPER", + "expected": "title-with-upper" + }, + { + "preset": "docutils", + "input": "Hello 🎉 World!", + "expected": "hello-world" + }, + { + "preset": "docutils", + "input": "!!!", + "expected": "" + }, + { + "preset": "docutils", + "input": "Straße & Œuvre", + "expected": "strasze-oeuvre" + }, + { + "preset": "docutils", + "input": "łódź ȸȹ", + "expected": "lodz-dbqp" + }, + { + "preset": "docutils", + "input": "café naïve", + "expected": "cafe-naive" + }, + { + "preset": "docutils", + "input": "__init__ (API)", + "expected": "init-api" } ], "unique": [ diff --git a/tests/test_anchors.py b/tests/test_anchors.py index df676cbb..edfd6e29 100644 --- a/tests/test_anchors.py +++ b/tests/test_anchors.py @@ -34,6 +34,22 @@ def test_print_anchors_gitlab(): assert re.findall(r'id="([^"]*)"', out) == ["anchor-20"] +def test_print_anchors_docutils(): + """``--slug-func docutils`` applies the make_id-style preset (with dedup).""" + out = _run_cli( + "# Hello World\n\n# Straße & Œuvre\n\n# Hello World", + "-l", + "1", + "--slug-func", + "docutils", + ) + assert re.findall(r'id="([^"]*)"', out) == [ + "hello-world", + "strasze-oeuvre", + "hello-world-1", + ] + + # a corpus exercising duplicates, dotted/digit titles, non-latin scripts, # inline code, emphasis and internal whitespace CROSS_CHECK_CORPUS = """\ @@ -84,3 +100,54 @@ def test_anchor_slugs_match_renderer(): ] assert cli_ids == doc_slugs + + +# docutils cross-check corpus: duplicates, dotted/digit titles, digraphs, +# inline code and internal whitespace, all with *non-empty* docutils slugs. +# Empty-slug titles (e.g. `# 2.0`, `# 中文标题`) are deliberately excluded: +# the renderer drops them from deduplication (empty slug -> no anchor), a +# renderer-only rule that the CLI's anchors plugin does not implement. +DOCUTILS_CROSS_CHECK_CORPUS = """\ +# Dup + +# Dup + +# Ubuntu 20.04 + +# lxc.env for environment variables + +# 3rd party + +# Straße & Œuvre + +# Some `code` here + +# Some *emphasis* text + +# x y +""" + + +def test_anchor_slugs_match_renderer_docutils(): + """The CLI and the docutils renderer agree under the ``docutils`` preset.""" + cli_out = _run_cli( + DOCUTILS_CROSS_CHECK_CORPUS, "-l", "6", "--slug-func", "docutils" + ) + cli_ids = re.findall(r'id="([^"]*)"', cli_out) + + doctree = publish_doctree( + DOCUTILS_CROSS_CHECK_CORPUS, + parser=Parser(), + settings_overrides={ + "myst_heading_anchors": 6, + "doctitle_xform": False, + "myst_heading_slug_func": "docutils", + }, + ) + doc_slugs = [ + section["slug"] + for section in doctree.findall(nodes.section) + if "slug" in section + ] + + assert cli_ids == doc_slugs diff --git a/tests/test_docutils.py b/tests/test_docutils.py index c27ef692..890badbc 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -597,6 +597,48 @@ def test_text_node_line_stamping(): assert lines["note body"] is None +def test_empty_slug_means_no_anchor(): + """A heading whose title slugifies to nothing gets no anchor. + + In particular the empty slug takes no part in deduplication, so a later + empty-slugging heading must not receive a nonsense ``-1`` anchor id. + """ + doctree = publish_doctree( + source="# !!!\n\n# ???\n", + parser=Parser(), + settings_overrides={ + "warning_stream": io.StringIO(), + "doctitle_xform": False, + "myst_heading_anchors": 2, + }, + ) + for section in doctree.findall(nodes.section): + assert "slug" not in section.attributes, section.attributes + assert section["ids"] and all(not i.startswith("-") for i in section["ids"]), ( + section["ids"] + ) + + +def test_docutils_slug_preset(): + """The ``docutils`` preset produces make_id-style anchors. + + Non-Latin titles slugify to nothing under this preset, so those headings + get no anchor. + """ + doctree = publish_doctree( + source="# Привет\n\n# 2.0\n\n# Hello World\n\n# Straße & Œuvre\n", + parser=Parser(), + settings_overrides={ + "warning_stream": io.StringIO(), + "doctitle_xform": False, + "myst_heading_anchors": 2, + "myst_heading_slug_func": "docutils", + }, + ) + slugs = [section.get("slug") for section in doctree.findall(nodes.section)] + assert slugs == [None, None, "hello-world", "strasze-oeuvre"] + + def test_definition_list_orphan_definition(): """A definition with no preceding term errors, but keeps its content. diff --git a/tests/test_slugs.py b/tests/test_slugs.py index 0ffa4c01..a1953094 100644 --- a/tests/test_slugs.py +++ b/tests/test_slugs.py @@ -39,12 +39,46 @@ def test_unique_slug(record): def test_slug_presets_keys(): - """The presets are exactly the two documented ones.""" - assert set(SLUG_PRESETS) == {"github", "gitlab"} + """The presets are exactly the documented ones.""" + assert set(SLUG_PRESETS) == {"docutils", "github", "gitlab"} + + +@pytest.mark.parametrize( + "title", + [record["input"] for record in CORPUS["slugify"]] + + [ + # extra fidelity vectors, beyond the corpus + "Ærøskøbing", # digraph + stroked o + "øđħıłŧ", # the whole translate table: head ... + "ƀƃƈƌƒƙƚƞƥƫƭƴƶ", # ... middle ... + "ǥȥȴȵȶȷȼȿɀɇɉɋɍɏ", # ... and tail (6 + 13 + 14 = all 33 entries) + "élégant", # combining acute accents (NFKD) + "① ² ⅓ Ⅳ", # numeric compatibility forms + "fine flow džungle", # ligature compatibility forms + " tabs\tand\nnewlines ", # whitespace normalization + "-- leading -- trailing --", + "123", # digits only -> empty + "𝔘𝔫𝔦𝔠𝔬𝔡𝔢", # noqa: RUF001 (mathematical alphanumerics, NFKD to ascii) + "ıstanbul I", # noqa: RUF001 (dotless i / turkish-locale hazard) + "", # empty input + ], + ids=repr, +) +def test_docutils_preset_fidelity(title): + """The ``docutils`` preset is byte-identical to ``docutils.nodes.make_id``. + + This pins fidelity by CI: a future docutils change to ``make_id`` fails + here, surfacing the drift instead of letting the two silently diverge + (contract policy: the preset and corpus then stay byte-stable, and the + divergence is documented). + """ + from docutils.nodes import make_id + + assert SLUG_PRESETS["docutils"](title) == make_id(title) def test_slugs_module_has_no_third_party_deps(): - """``myst_parser.slugs`` is pure standard library (only ``re``, ``typing``). + """``myst_parser.slugs`` is pure standard library. This keeps it trivially portable for other MyST implementations. """ @@ -57,4 +91,10 @@ def test_slugs_module_has_no_third_party_deps(): imported.update(alias.name.split(".")[0] for alias in node.names) elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: imported.add(node.module.split(".")[0]) - assert imported <= {"__future__", "re", "collections", "typing"} + assert imported <= {"__future__", "re", "collections", "typing", "unicodedata"} + + +def test_corpus_metadata(): + """The corpus carries its contract version and policy.""" + assert CORPUS["version"] == 1 + assert "append-only" in CORPUS["description"]