Mercurial > hgrepos > Python > libs > pygments-lexer-pseudocode2

changeset 289:6fc7f9c1d89d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Remove the old implementation of explicit token types completely
author Franz Glasner <fzglas.hg@dom66.de>
date Fri, 22 May 2026 12:41:08 +0200
parents 298841bc4dee
children 3e2f7ca64894
files docs/lexer-algpseudocode.rst pygments_lexer_pseudocode2/lexers/algpseudocode.py tests/test_algpseudo.py
diffstat 3 files changed, 7 insertions(+), 295 deletions(-) [+]
line wrap: on
line diff
--- a/docs/lexer-algpseudocode.rst	Fri May 22 12:32:38 2026 +0200
+++ b/docs/lexer-algpseudocode.rst	Fri May 22 12:41:08 2026 +0200
@@ -468,19 +468,10 @@
 ====================
 
 They allow to handle keywords and operators that are not recognized by default.
-And they allow the user to explicitely highlight some input text at low-level.
-
-.. note:: Explicit token types are **case-sensitive**.
-
-.. note:: Explicit token types work in all `expression` and `text` contexts.
+And they allow the user to explicitely highlight some input text with
+a low-level command.
 
-.. note:: Nested explicit token types are *not supported*.
-
-
-Current and Recommended Syntax
-------------------------------
-
-The current and recommended use is the ``\ttX{ARG1}{ARG2}`` command.
+They are implemented with the ``\ttX{ARG1}{ARG2}`` command.
 
 This command has two required parameters:
 
@@ -517,6 +508,8 @@
 
    Standard `Escaping Rules`_ apply to this argument!
 
+.. note:: The command for explicit token types is **case-sensitive**.
+
 .. rubric:: Examples:
 
 .. code-block:: algpseudocode
@@ -574,77 +567,9 @@
 
    \text{• \\ttX{Generic.Not.Yet.Existing\}{∈_∌\}(p1, p2)}     \ttX{Generic.Not.Yet.Existing}{∈_∌}(p1, p2)
 
-
-Old Syntax (Deprecated)
------------------------
-
-.. deprecated:: 3.0
-   Use `Current and Recommended Syntax`_ instead.
-
-.. note:: The lower-case ``x`` in ``\ttx-``!
-
-`XX` represents a `value` in the :py:data:`pygments.token.STANDARD_TYPES`
-dict.
-Its corresponding token type (the associated `key` in this `dict`) is
-used as token type.
-
-``\tt-XX/SINGLE-CHAR``
-
-  no escaping needed
-
-  `SINGLE-CHAR` is a single character and can be *every* character
-  (including a carriage-return or line-feed)
-
-``\ttx-XX{CHARACTERS}``
-
-``\ttx-XX(CHARACTERS)``
-
-``\ttx-XX[CHARACTERS]``
-
-``\ttx-XX<CHARACTERS>``
-
-``\ttx-XX<SEP>CHARACTERS<SEP>``
-
-  No escaping possible! There are enough alternatives available!
-
-  `SEP` is exactly one character of ``/?.,:;%|=*+!\$~"'_-#@``.
-
+.. note:: Explicit token types work in all `expression` and `text` contexts.
 
-Examples:
-
-.. code-block:: algpseudocode
-
-   \text{• \\tt-kc/C}      \tt-kc/C            \rem C as Keyword.Constant
-   \text{• \\tt-ow/∈}      \tt-ow/∈            \rem ∈ as Operator.Word
-   \text{• \\ttx-kc{A Constant Keyword\}}    \ttx-kc{A Constant Keyword}  \rem An explicit Keyword.Constant
-   \text{• \\ttx-nv{A Variable Name\}}       \ttx-nv{A Variable Name}     \rem An explicit Name.Variable
-   \text{• \\ttx-ni{An Entity*Name\}}        \ttx-ni{An Entity*Name}      \rem An explicit Name.Entity
-   \text{• \\ttx-k(∈ ∌)}   \ttx-k(∈ ∌)         \rem ∈ and ∌ as (ordinary) Keywords
-   \text{• \\ttx-o<∈ ∌>}   \ttx-o<∈ ∌>         \rem ∈ and ∌ as (ordinary) Operators
-     /*
-      * The line below has ∈_∌ as (peculiar) function name.
-      * Their params are automatic (i.e. a normal expression).
-      */
-   \text{• \\ttx-nf<∈_∌>(p1, p2)}                \ttx-nf<∈_∌>(p1, p2)
-     /*
-      * The line below has ∈_∌ as (peculiar) decorator name (as used in Python).
-      * Their params are automatic (i.e. a normal expression).
-      */
-   \text{• \\ttx-nd[∈_∌](p1, p2)}                \ttx-nd[∈_∌](p1, p2)
-     /*
-      * Normal emphasis ("strong")
-      */
-   \text{• \\ttx-gs$this is strong$}             \ttx-gs$this is strong$
-     /*
-      * A strong emphasis.
-      * Note that the backslash is a valid delimiter!
-      */
-   \text{• \\ttx-ges\\strong emphasis!\\}          \ttx-ges\strong emphasis!\
-     /*
-      * This is a non-existing token type: you get some generic error markup
-      * with a Generic.Error token and no expansion.
-      */
-   \text{• \\ttx-NON-EXISTING?∈_∌?(p1, p2)}      \ttx-NON_EXISTING?∈_∌?(p1, p2)
+.. note:: Nested explicit token types are *not supported*.
 
 
 .. _escaping-rules:
--- a/pygments_lexer_pseudocode2/lexers/algpseudocode.py	Fri May 22 12:32:38 2026 +0200
+++ b/pygments_lexer_pseudocode2/lexers/algpseudocode.py	Fri May 22 12:41:08 2026 +0200
@@ -223,20 +223,6 @@
 
         return _op_symbol
 
-    def op_explicit_tokentype(lexer, match, ctx=None):
-        needed_css = match.group("type")
-        toktype = REVERSED_STANDARD_TYPES.get(needed_css, None)
-        if toktype is None:
-            # Be more error friendly
-            toktype = Generic.Error
-            val = match.group()
-            _logger.warning("Unhandled explicit token type: %s", val)
-        else:
-            val = match.group("characters")
-        yield match.start(), toktype, val
-        if ctx:
-            ctx.pos = match.end()
-
     def op_explicit_tokentype_ex_start(lexer, match, ctx):
         needed_css = match.group("type")
         toktype = REVERSED_STANDARD_TYPES.get(needed_css, None)
@@ -539,28 +525,6 @@
             (r"""\\ttX[ \t]*\{(?P<type>[^}]*)\}[ \t]*\{""",
              op_explicit_tokentype_ex_start,
              "extended-explicit-tokentype"),
-
-            #
-            # Old variants
-            #
-
-            # Multiple characters possible, but no escaping!
-            (r"""\\ttx\-(?P<type>[a-zA-Z0-9_-]+?)"""
-             r"""(?P<sep>[/?.,:;%|=*+!\\$~"'#@_-])"""
-             r"""(?P<characters>(.|\n)+?)(?P=sep)""",
-             op_explicit_tokentype),
-            (r"\\ttx\-(?P<type>[a-zA-Z0-9_-]+?)\{(?P<characters>[^}]+?)\}",
-             op_explicit_tokentype),
-            (r"\\ttx\-(?P<type>[a-zA-Z0-9_-]+?)\((?P<characters>[^)]+?)\)",
-             op_explicit_tokentype),
-            (r"\\ttx\-(?P<type>[a-zA-Z0-9_-]+?)<(?P<characters>[^>]+?)>",
-             op_explicit_tokentype),
-            (r"\\ttx\-(?P<type>[a-zA-Z0-9_-]+?)\[(?P<characters>[^\]]+?)\]",
-             op_explicit_tokentype),
-
-            # Every character is possible: no escaping needed!
-            (r"\\tt-(?P<type>[^/]+?)/(?P<characters>(?:.|\n))",
-             op_explicit_tokentype),
         ],
         "extended-explicit-tokentype": [
             (r"([^\\}]+)", op_explicit_tokentype_ex_value),
--- a/tests/test_algpseudo.py	Fri May 22 12:32:38 2026 +0200
+++ b/tests/test_algpseudo.py	Fri May 22 12:41:08 2026 +0200
@@ -577,183 +577,6 @@
                 r"\EXPR{second expression}}",
                 self.lexer))
 
-    def test_explicit_extended_single_tokentype_1(self):
-        self.assertTokenStreamEqualComplete(
-            [("Operator", "%"),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(r"\ttx-o:%:", self.lexer))
-
-    def test_explicit_extended_single_tokentype_2(self):
-        self.assertTokenStreamEqualComplete(
-            [("Operator", "{"),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(r"\ttx-o{{}", self.lexer))
-
-    def test_explicit_extended_single_tokentype_3(self):
-        self.assertTokenStreamEqualComplete(
-            [("Operator", "<"),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(r"\ttx-o<<>", self.lexer))
-
-    def test_explicit_extended_single_tokentype_4(self):
-        self.assertTokenStreamEqualComplete(
-            [("Operator", "("),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(r"\ttx-o(()", self.lexer))
-
-    def test_explicit_extended_multi_tokentype_1(self):
-        self.assertTokenStreamEqualComplete(
-            [("Operator", "xxx in A"),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(r"\ttx-o/xxx in A/", self.lexer))
-
-    def test_explicit_extended_multi_tokentype_2(self):
-        self.assertTokenStreamEqualComplete(
-            [("Operator", "xxx in B"),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(r"\ttx-o{xxx in B}", self.lexer))
-
-    def test_explicit_extended_multi_tokentype_3(self):
-        self.assertTokenStreamEqualComplete(
-            [("Operator", "xxx in C"),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(r"\ttx-o<xxx in C>", self.lexer))
-
-    def test_explicit_extended_multi_tokentype_4(self):
-        self.assertTokenStreamEqualComplete(
-            [("Operator", "xxx in D"),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(r"\ttx-o(xxx in D)", self.lexer))
-
-    def test_explicit_tokentype_simple_1(self):
-        self.assertTokenStreamEqualComplete(
-            [("Operator", "}"),
-             ("Operator", "/"),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(r"\tt-o/}\tt-o//", self.lexer))
-
-    def test_explicit_tokentype_simple_2(self):
-        self.assertTokenStreamEqualComplete(
-            [("Operator", "\n"),
-             ("Operator", "/"),
-             ("Text", " "),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex("\\tt-o/\n\\tt-o// ", self.lexer))
-
-    def test_explicit_tokentype_with_remark(self):
-        self.assertTokenStreamEqualComplete(
-            [("Operator", u"∈ ∌"),
-             ("Text", "    "),
-             ("Comment.Single", u"▷"),
-             ("Comment.Single", u" ∈ ∌ as (ordinary) operators"),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(
-                u"""\\ttx-o<∈ ∌>    \\rem ∈ ∌ as (ordinary) operators""",
-                self.lexer))
-
-    def test_explicit_tokentype_with_remark_2(self):
-        self.assertTokenStreamEqualComplete(
-            [("Operator", "new_operator"),
-             ("Text", "  "),
-             ("Comment.Single", u"▷"),
-             ("Comment.Single", " a (synthesized) operator"),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(
-                r"""\ttx-o<new_operator>  \rem a (synthesized) operator""",
-                self.lexer))
-
-    def test_explicit_tokentype_with_possibly_conflicting_parens(self):
-        self.assertTokenStreamEqualComplete(
-            [("Name.Function", u"∈_∌"),
-             ("Punctuation", "("),
-             ("Name.Entity", "p1"),
-             ("Punctuation", ","),
-             ("Text", " "),
-             ("Name.Entity", "p2"),
-             ("Punctuation", ")"),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(
-                u"""\\ttx-nf<∈_∌>(p1, p2)""",
-                self.lexer))
-
-    def test_explicit_tokentype_with_possibly_conflicting_parens_2(self):
-        self.assertTokenStreamEqualComplete(
-            [("Name.Decorator", "a_Decorator"),
-             ("Punctuation", "("),
-             ("Name.Entity", "p1"),
-             ("Punctuation", ","),
-             ("Text", " "),
-             ("Name.Entity", "p2"),
-             ("Punctuation", ")"),
-             ("Text", "   "),
-             ("Comment.Single", u"▷"),
-             ("Comment.Single", " (Python) decorator"),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(
-                r"""\ttx-nd<a_Decorator>(p1, p2)   \rem (Python) decorator""",
-                self.lexer))
-
-    def test_explicit_tokentype_with_possibly_conflicting_parens_3(self):
-        self.assertTokenStreamEqualComplete(
-            [("Name.Decorator", "a_Decorator"),
-             ("Punctuation", "("),
-             ("Name.Entity", "p1"),
-             ("Punctuation", ","),
-             ("Text", " "),
-             ("Name.Entity", "p2"),
-             ("Punctuation", ")"),
-             ("Text", "   "),
-             ("Comment.Single", u"▷"),
-             ("Comment.Single", " (Python) annotation"),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(
-                r"""\ttx-nd[a_Decorator](p1, p2)   \rem (Python) annotation""",
-                self.lexer))
-
-    def test_explicit_tokentype_all_seps(self):
-        for sep in r"""/?.,:;%|=*+!$"'~_-#@""":
-            self.assertTokenStreamEqualComplete(
-                [("Name.Decorator", "word"),
-                 ("Text", sep),
-                 ("Text.Whitespace", "\n"),
-                 ],
-                pygments.lex(
-                    r"\text{\ttx-nd%sword%s%s}" % (sep, sep, sep),
-                    self.lexer))
-
-    def test_explicit_tokentype_backslash(self):
-        self.assertTokenStreamEqualComplete(
-            [("Name.Decorator", "word"),
-             ("Text", "\\"),
-             ("Text", " "),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(r"\text{\ttx-nd\word\\ }", self.lexer))
-
-    def test_explicit_tokentype_error(self):
-        self.assertTokenStreamEqualComplete(
-            [("Generic.Error", r"""\ttx-non-existing[a_Decorator]"""),
-             ("Text.Whitespace", "\n"),
-             ],
-            pygments.lex(
-                r"""\ttx-non-existing[a_Decorator]""",
-                self.lexer))
-
     def test_extended_explicit_tokentype_empty(self):
         self.assertTokenStreamEqualComplete(
             [("Text.Whitespace", "\n")],