Mercurial > hgrepos > Python > libs > pygments-lexer-pseudocode2
changeset 18:859ab8abce0a
The first real tests about lexing with the Pygments API for the tests.
Also implemented a simple helper to compare token streams with shorter code.
| author | Franz Glasner <fzglas.hg@dom66.de> |
|---|---|
| date | Mon, 20 Apr 2026 14:12:35 +0200 |
| parents | 251898f2f0c7 |
| children | 2e67c4eae6d9 |
| files | tests/_testhelper.py tests/_tsetup.py tests/test_fr.py |
| diffstat | 3 files changed, 129 insertions(+), 9 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/_testhelper.py Mon Apr 20 14:12:35 2026 +0200 @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# :- +# SPDX-FileCopyrightText: © 2026 Franz Glasner +# SPDX-License-Identifier: MIT +# :- +"""Test helper""" + +__all__ = ["TokenAssertHelper"] + + +from pygments.token import Token, is_token_subtype, string_to_tokentype + + +class TokenAssertHelper(object): + """Mixin to test for token stream equality""" + + def assertTokenEqual(self, tok_or_str, txt, token): + if is_token_subtype(tok_or_str, Token): + t = tok_or_str + else: + t = string_to_tokentype(tok_or_str) + self.assertEqual((t, txt), token) + + def assertNextTokenEqual(self, tok_or_str, txt, tokens): + self.assertTokenEqual(tok_or_str, txt, next(tokens)) + + def assertTokenStreamEqual(self, expected_tokens, given_tokens): + for tok, txt in expected_tokens: + self.assertNextTokenEqual(tok, txt, given_tokens) + + def assertTokenStreamEqualComplete(self, expected_tokens, given_tokens): + self.assertTokenStreamEqual(expected_tokens, given_tokens) + self.assertRaises(StopIteration, next, given_tokens)
--- a/tests/_tsetup.py Mon Apr 20 13:34:20 2026 +0200 +++ b/tests/_tsetup.py Mon Apr 20 14:12:35 2026 +0200 @@ -13,12 +13,12 @@ import sys -sys.path.insert( - 0, - os.path.abspath( - os.path.normpath(os.path.join(os.path.dirname(__file__), "..")))) - - +PROJECTDIR = os.path.abspath( + os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) +sys.path.insert(0, PROJECTDIR) +LEXERFILENAME = os.path.join(PROJECTDIR, + "pygments_lexer_pseudocode2/__init__.py") +LEXERCLASS = "PseudocodeLexer2" TESTSNIPPETSDIR = os.path.join( os.path.abspath(os.path.dirname(__file__)), "snippets")
--- a/tests/test_fr.py Mon Apr 20 13:34:20 2026 +0200 +++ b/tests/test_fr.py Mon Apr 20 14:12:35 2026 +0200 @@ -4,12 +4,99 @@ # SPDX-License-Identifier: MIT # :- -import _tsetup # noqa:F401 (imported but unused) +from _tsetup import LEXERFILENAME, LEXERCLASS import unittest +import pygments +import pygments.lexers +from pygments.token import Token + +import _testhelper + + +class TestLoading(unittest.TestCase): + + def test_load(self): + pygments.lexers.load_lexer_from_file(LEXERFILENAME, LEXERCLASS) + + +class TestFrLexer(unittest.TestCase, _testhelper.TokenAssertHelper): + + def setUp(self): + self.lexer = pygments.lexers.load_lexer_from_file( + LEXERFILENAME, LEXERCLASS) + + def test_types(self): + tokens = pygments.lex("huhu", self.lexer) + self.assertEqual((Token.Name.Variable, "huhu"), next(tokens)) + self.assertEqual((Token.Text, "\n"), next(tokens)) + self.assertRaises(StopIteration, next, tokens) + + def test_op_replace(self): + tokens = pygments.lex("<= >= <> <- ^", self.lexer) + self.assertEqual((Token.Operator, "≤"), next(tokens)) + self.assertEqual((Token.Text, " "), next(tokens)) + self.assertEqual((Token.Operator, "≥"), next(tokens)) + self.assertEqual((Token.Text, " "), next(tokens)) + self.assertEqual((Token.Operator, "≠"), next(tokens)) + self.assertEqual((Token.Text, " "), next(tokens)) + self.assertEqual((Token.Operator, "←"), next(tokens)) + self.assertEqual((Token.Text, " "), next(tokens)) + self.assertEqual((Token.Operator, "↑"), next(tokens)) + self.assertEqual((Token.Text, "\n"), next(tokens)) + self.assertRaises(StopIteration, next, tokens) + + def test_op_replace_list(self): + tokens = pygments.lex("<= >= <> <- ^", self.lexer) + self.assertTokenStreamEqual( + [("Token.Operator", "≤"), + ("Token.Text", " "), + ("Token.Operator", "≥"), + ("Token.Text", " "), + ("Token.Operator", "≠"), + ("Token.Text", " "), + ("Token.Operator", "←"), + ("Token.Text", " "), + ("Token.Operator", "↑"), + ("Token.Text", "\n"), + ], + tokens) + self.assertRaises(StopIteration, next, tokens) + + def test_op_replace_list_with_short_names(self): + tokens = pygments.lex("<= >= <> <- ^", self.lexer) + self.assertTokenStreamEqual( + [("Operator", "≤"), + ("Text", " "), + ("Operator", "≥"), + ("Text", " "), + ("Operator", "≠"), + ("Text", " "), + ("Operator", "←"), + ("Text", " "), + ("Operator", "↑"), + ("Text", "\n"), + ], + tokens) + self.assertRaises(StopIteration, next, tokens) + + def test_op_replace_complete(self): + tokens = pygments.lex("<= >= <> <- ^", self.lexer) + self.assertTokenStreamEqualComplete( + [("Operator", "≤"), + ("Text", " "), + ("Operator", "≥"), + ("Text", " "), + ("Operator", "≠"), + ("Text", " "), + ("Operator", "←"), + ("Text", " "), + ("Operator", "↑"), + ("Text", "\n"), + ], + tokens) + if __name__ == "__main__": unittest.main() - -