changeset 258:e03e8cfdcece

Make the TokenReplaceFilter more flexible by allowing a replacement map
author Franz Glasner <fzglas.hg@dom66.de>
date Sun, 17 May 2026 16:08:34 +0200
parents 265b759f1f08
children e9e605e837fc
files docs/filters.rst pygments_lexer_pseudocode2/filters/__init__.py tests/test_filter.py
diffstat 3 files changed, 54 insertions(+), 17 deletions(-) [+]
line wrap: on
line diff
--- a/docs/filters.rst	Sun May 17 14:30:07 2026 +0200
+++ b/docs/filters.rst	Sun May 17 16:08:34 2026 +0200
@@ -53,18 +53,30 @@
 ==================
 
 :Name: tokenreplace
-:Required Filter Options:
+:Filter Options:
+   ``replacements``
+     **Type:** :py:class:`dict[str | pygments.token.Token, str | pygments.token.Token]`
+
+     A map from tokens to their replacements.
+
    ``token_from``
      **Type:** :py:class:`str` or :py:class:`pygments.token.Token`
 
      The name of a token type (like ``Error``) or a token object
      (like :py:class:`pygments.token.Token.Error`).
 
+     If given the `token_to` options is required and `replacements` will be
+     augmented with their respective values.
+
    ``token_to``
      **Type:** :py:class:`str` or :py:class:`pygments.token.Token`
 
      The name of a token type (like ``Generic.Error``) or a token object
      (like :py:class:`pygments.token.Token.Generic.Error`).
 
-Replace all token types given in `token_from` by the token type given
-in `token_to`.
+     This option is required if `token_from` is given.
+
+Replace all token types given as `replacements` keys or in `token_from`
+with the token types given in `replacements` values or in `token_to`.
+
+The values in the token stream are retained.
--- a/pygments_lexer_pseudocode2/filters/__init__.py	Sun May 17 14:30:07 2026 +0200
+++ b/pygments_lexer_pseudocode2/filters/__init__.py	Sun May 17 16:08:34 2026 +0200
@@ -18,36 +18,49 @@
 
 class TokenReplaceFilter(Filter):
 
-    """Replace a given fixed token type with another token."""
+    """Replace given fixed token types with other tokens."""
 
     def __init__(self, **options):
         """Specifiy the replacement options:
 
+        :param replacements: A map from tokens to their replacements.
+        :type replacements:
+                dict[str | pygments.token.Token, str | pygments.token.Token]
+
         :param token_from:
         :type token_from: str or pygments.token.Token
 
         :param token_to:
         :type token_to: str or pygments.token.Token
 
-        Both these arguments are *required*!
+        `token_to` is required if `token_from` is given.
+        If they are given they **augment** `replacements`.
 
         """
         Filter.__init__(self, **options)
-        # The option "token_from" is required!
-        self.token_from = options["token_from"]
-        if not is_token_subtype(self.token_from, Token):
-            self.token_from = string_to_tokentype(self.token_from)
-        # The option "token_to" is required!
-        self.token_to = options["token_to"]
-        if not is_token_subtype(self.token_to, Token):
-            self.token_to = string_to_tokentype(self.token_to)
+        self.replacements = {}
+        repl = options.get("replacements")
+        if repl:
+            for k, v in repl.items():
+                if not is_token_subtype(k, Token):
+                    k = string_to_tokentype(k)
+                if not is_token_subtype(v, Token):
+                    v = string_to_tokentype(v)
+                self.replacements[k] = v
+        # The option "token_from" augments the replacements if given
+        token_from = options.get("token_from", None)
+        if token_from:
+            if not is_token_subtype(token_from, Token):
+                token_from = string_to_tokentype(token_from)
+            # The option "token_to" is required if "token_from" is given
+            token_to = options["token_to"]
+            if not is_token_subtype(token_to, Token):
+                token_to = string_to_tokentype(token_to)
+            self.replacements[token_from] = token_to
 
     def filter(self, lexer, stream):
         for ttype, value in stream:
-            if ttype is self.token_from:
-                yield self.token_to, value
-            else:
-                yield ttype, value
+            yield self.replacements.get(ttype, ttype), value
 
 
 class ErrorToGenericErrorTokenFilter(TokenReplaceFilter):
--- a/tests/test_filter.py	Sun May 17 14:30:07 2026 +0200
+++ b/tests/test_filter.py	Sun May 17 16:08:34 2026 +0200
@@ -10,6 +10,7 @@
 
 import pygments
 import pygments.lexers
+import pygments.token
 
 import _testhelper
 import pygments_lexer_pseudocode2.filters
@@ -57,5 +58,16 @@
             pygments.lex(r"\nonexisting{", lexer))
 
 
+class TestPygmentsTokenAssumptions(unittest.TestCase):
+
+    def test_tokens_are_hashable(self):
+        self.assertTrue(hash(pygments.token.Token.Error))
+
+    def test_token_equality(self):
+        self.assertEqual(
+            pygments.token.Token.Text.Whitespace,
+            pygments.token.string_to_tokentype("Text.Whitespace"))
+
+
 if __name__ == "__main__":
     unittest.main()