# HG changeset patch # User Franz Glasner # Date 1779026914 -7200 # Node ID e03e8cfdceceaef42b0a3f94a2315936e7af618c # Parent 265b759f1f081ad3a6061874735de6d2e13fa156 Make the TokenReplaceFilter more flexible by allowing a replacement map diff -r 265b759f1f08 -r e03e8cfdcece docs/filters.rst --- a/docs/filters.rst Sun May 17 14:30:07 2026 +0200 +++ b/docs/filters.rst Sun May 17 16:08:34 2026 +0200 @@ -53,18 +53,30 @@ ================== :Name: tokenreplace -:Required Filter Options: +:Filter Options: + ``replacements`` + **Type:** :py:class:`dict[str | pygments.token.Token, str | pygments.token.Token]` + + A map from tokens to their replacements. + ``token_from`` **Type:** :py:class:`str` or :py:class:`pygments.token.Token` The name of a token type (like ``Error``) or a token object (like :py:class:`pygments.token.Token.Error`). + If given the `token_to` options is required and `replacements` will be + augmented with their respective values. + ``token_to`` **Type:** :py:class:`str` or :py:class:`pygments.token.Token` The name of a token type (like ``Generic.Error``) or a token object (like :py:class:`pygments.token.Token.Generic.Error`). -Replace all token types given in `token_from` by the token type given -in `token_to`. + This option is required if `token_from` is given. + +Replace all token types given as `replacements` keys or in `token_from` +with the token types given in `replacements` values or in `token_to`. + +The values in the token stream are retained. diff -r 265b759f1f08 -r e03e8cfdcece pygments_lexer_pseudocode2/filters/__init__.py --- a/pygments_lexer_pseudocode2/filters/__init__.py Sun May 17 14:30:07 2026 +0200 +++ b/pygments_lexer_pseudocode2/filters/__init__.py Sun May 17 16:08:34 2026 +0200 @@ -18,36 +18,49 @@ class TokenReplaceFilter(Filter): - """Replace a given fixed token type with another token.""" + """Replace given fixed token types with other tokens.""" def __init__(self, **options): """Specifiy the replacement options: + :param replacements: A map from tokens to their replacements. + :type replacements: + dict[str | pygments.token.Token, str | pygments.token.Token] + :param token_from: :type token_from: str or pygments.token.Token :param token_to: :type token_to: str or pygments.token.Token - Both these arguments are *required*! + `token_to` is required if `token_from` is given. + If they are given they **augment** `replacements`. """ Filter.__init__(self, **options) - # The option "token_from" is required! - self.token_from = options["token_from"] - if not is_token_subtype(self.token_from, Token): - self.token_from = string_to_tokentype(self.token_from) - # The option "token_to" is required! - self.token_to = options["token_to"] - if not is_token_subtype(self.token_to, Token): - self.token_to = string_to_tokentype(self.token_to) + self.replacements = {} + repl = options.get("replacements") + if repl: + for k, v in repl.items(): + if not is_token_subtype(k, Token): + k = string_to_tokentype(k) + if not is_token_subtype(v, Token): + v = string_to_tokentype(v) + self.replacements[k] = v + # The option "token_from" augments the replacements if given + token_from = options.get("token_from", None) + if token_from: + if not is_token_subtype(token_from, Token): + token_from = string_to_tokentype(token_from) + # The option "token_to" is required if "token_from" is given + token_to = options["token_to"] + if not is_token_subtype(token_to, Token): + token_to = string_to_tokentype(token_to) + self.replacements[token_from] = token_to def filter(self, lexer, stream): for ttype, value in stream: - if ttype is self.token_from: - yield self.token_to, value - else: - yield ttype, value + yield self.replacements.get(ttype, ttype), value class ErrorToGenericErrorTokenFilter(TokenReplaceFilter): diff -r 265b759f1f08 -r e03e8cfdcece tests/test_filter.py --- a/tests/test_filter.py Sun May 17 14:30:07 2026 +0200 +++ b/tests/test_filter.py Sun May 17 16:08:34 2026 +0200 @@ -10,6 +10,7 @@ import pygments import pygments.lexers +import pygments.token import _testhelper import pygments_lexer_pseudocode2.filters @@ -57,5 +58,16 @@ pygments.lex(r"\nonexisting{", lexer)) +class TestPygmentsTokenAssumptions(unittest.TestCase): + + def test_tokens_are_hashable(self): + self.assertTrue(hash(pygments.token.Token.Error)) + + def test_token_equality(self): + self.assertEqual( + pygments.token.Token.Text.Whitespace, + pygments.token.string_to_tokentype("Text.Whitespace")) + + if __name__ == "__main__": unittest.main()