Mercurial > hgrepos > Python > libs > pygments-lexer-pseudocode2
changeset 258:e03e8cfdcece
Make the TokenReplaceFilter more flexible by allowing a replacement map
| author | Franz Glasner <fzglas.hg@dom66.de> |
|---|---|
| date | Sun, 17 May 2026 16:08:34 +0200 |
| parents | 265b759f1f08 |
| children | e9e605e837fc |
| files | docs/filters.rst pygments_lexer_pseudocode2/filters/__init__.py tests/test_filter.py |
| diffstat | 3 files changed, 54 insertions(+), 17 deletions(-) [+] |
line wrap: on
line diff
--- a/docs/filters.rst Sun May 17 14:30:07 2026 +0200 +++ b/docs/filters.rst Sun May 17 16:08:34 2026 +0200 @@ -53,18 +53,30 @@ ================== :Name: tokenreplace -:Required Filter Options: +:Filter Options: + ``replacements`` + **Type:** :py:class:`dict[str | pygments.token.Token, str | pygments.token.Token]` + + A map from tokens to their replacements. + ``token_from`` **Type:** :py:class:`str` or :py:class:`pygments.token.Token` The name of a token type (like ``Error``) or a token object (like :py:class:`pygments.token.Token.Error`). + If given the `token_to` options is required and `replacements` will be + augmented with their respective values. + ``token_to`` **Type:** :py:class:`str` or :py:class:`pygments.token.Token` The name of a token type (like ``Generic.Error``) or a token object (like :py:class:`pygments.token.Token.Generic.Error`). -Replace all token types given in `token_from` by the token type given -in `token_to`. + This option is required if `token_from` is given. + +Replace all token types given as `replacements` keys or in `token_from` +with the token types given in `replacements` values or in `token_to`. + +The values in the token stream are retained.
--- a/pygments_lexer_pseudocode2/filters/__init__.py Sun May 17 14:30:07 2026 +0200 +++ b/pygments_lexer_pseudocode2/filters/__init__.py Sun May 17 16:08:34 2026 +0200 @@ -18,36 +18,49 @@ class TokenReplaceFilter(Filter): - """Replace a given fixed token type with another token.""" + """Replace given fixed token types with other tokens.""" def __init__(self, **options): """Specifiy the replacement options: + :param replacements: A map from tokens to their replacements. + :type replacements: + dict[str | pygments.token.Token, str | pygments.token.Token] + :param token_from: :type token_from: str or pygments.token.Token :param token_to: :type token_to: str or pygments.token.Token - Both these arguments are *required*! + `token_to` is required if `token_from` is given. + If they are given they **augment** `replacements`. """ Filter.__init__(self, **options) - # The option "token_from" is required! - self.token_from = options["token_from"] - if not is_token_subtype(self.token_from, Token): - self.token_from = string_to_tokentype(self.token_from) - # The option "token_to" is required! - self.token_to = options["token_to"] - if not is_token_subtype(self.token_to, Token): - self.token_to = string_to_tokentype(self.token_to) + self.replacements = {} + repl = options.get("replacements") + if repl: + for k, v in repl.items(): + if not is_token_subtype(k, Token): + k = string_to_tokentype(k) + if not is_token_subtype(v, Token): + v = string_to_tokentype(v) + self.replacements[k] = v + # The option "token_from" augments the replacements if given + token_from = options.get("token_from", None) + if token_from: + if not is_token_subtype(token_from, Token): + token_from = string_to_tokentype(token_from) + # The option "token_to" is required if "token_from" is given + token_to = options["token_to"] + if not is_token_subtype(token_to, Token): + token_to = string_to_tokentype(token_to) + self.replacements[token_from] = token_to def filter(self, lexer, stream): for ttype, value in stream: - if ttype is self.token_from: - yield self.token_to, value - else: - yield ttype, value + yield self.replacements.get(ttype, ttype), value class ErrorToGenericErrorTokenFilter(TokenReplaceFilter):
--- a/tests/test_filter.py Sun May 17 14:30:07 2026 +0200 +++ b/tests/test_filter.py Sun May 17 16:08:34 2026 +0200 @@ -10,6 +10,7 @@ import pygments import pygments.lexers +import pygments.token import _testhelper import pygments_lexer_pseudocode2.filters @@ -57,5 +58,16 @@ pygments.lex(r"\nonexisting{", lexer)) +class TestPygmentsTokenAssumptions(unittest.TestCase): + + def test_tokens_are_hashable(self): + self.assertTrue(hash(pygments.token.Token.Error)) + + def test_token_equality(self): + self.assertEqual( + pygments.token.Token.Text.Whitespace, + pygments.token.string_to_tokentype("Text.Whitespace")) + + if __name__ == "__main__": unittest.main()
