diff tests/test_fr.py @ 18:859ab8abce0a

The first real tests about lexing with the Pygments API for the tests. Also implemented a simple helper to compare token streams with shorter code.
author Franz Glasner <fzglas.hg@dom66.de>
date Mon, 20 Apr 2026 14:12:35 +0200
parents b92f660f2461
children 2e67c4eae6d9
line wrap: on
line diff
--- a/tests/test_fr.py	Mon Apr 20 13:34:20 2026 +0200
+++ b/tests/test_fr.py	Mon Apr 20 14:12:35 2026 +0200
@@ -4,12 +4,99 @@
 # SPDX-License-Identifier: MIT
 # :-
 
-import _tsetup    # noqa:F401 (imported but unused)
+from _tsetup import LEXERFILENAME, LEXERCLASS
 
 import unittest
 
+import pygments
+import pygments.lexers
+from pygments.token import Token
+
+import _testhelper
+
+
+class TestLoading(unittest.TestCase):
+
+    def test_load(self):
+        pygments.lexers.load_lexer_from_file(LEXERFILENAME, LEXERCLASS)
+
+
+class TestFrLexer(unittest.TestCase, _testhelper.TokenAssertHelper):
+
+    def setUp(self):
+        self.lexer = pygments.lexers.load_lexer_from_file(
+            LEXERFILENAME, LEXERCLASS)
+
+    def test_types(self):
+        tokens = pygments.lex("huhu", self.lexer)
+        self.assertEqual((Token.Name.Variable, "huhu"), next(tokens))
+        self.assertEqual((Token.Text, "\n"), next(tokens))
+        self.assertRaises(StopIteration, next, tokens)
+
+    def test_op_replace(self):
+        tokens = pygments.lex("<= >= <> <- ^", self.lexer)
+        self.assertEqual((Token.Operator, "≤"), next(tokens))
+        self.assertEqual((Token.Text, " "), next(tokens))
+        self.assertEqual((Token.Operator, "≥"), next(tokens))
+        self.assertEqual((Token.Text, " "), next(tokens))
+        self.assertEqual((Token.Operator, "≠"), next(tokens))
+        self.assertEqual((Token.Text, " "), next(tokens))
+        self.assertEqual((Token.Operator, "←"), next(tokens))
+        self.assertEqual((Token.Text, " "), next(tokens))
+        self.assertEqual((Token.Operator, "↑"), next(tokens))
+        self.assertEqual((Token.Text, "\n"), next(tokens))
+        self.assertRaises(StopIteration, next, tokens)
+
+    def test_op_replace_list(self):
+        tokens = pygments.lex("<= >= <> <- ^", self.lexer)
+        self.assertTokenStreamEqual(
+            [("Token.Operator", "≤"),
+             ("Token.Text", " "),
+             ("Token.Operator", "≥"),
+             ("Token.Text", " "),
+             ("Token.Operator", "≠"),
+             ("Token.Text", " "),
+             ("Token.Operator", "←"),
+             ("Token.Text", " "),
+             ("Token.Operator", "↑"),
+             ("Token.Text", "\n"),
+             ],
+            tokens)
+        self.assertRaises(StopIteration, next, tokens)
+
+    def test_op_replace_list_with_short_names(self):
+        tokens = pygments.lex("<= >= <> <- ^", self.lexer)
+        self.assertTokenStreamEqual(
+            [("Operator", "≤"),
+             ("Text", " "),
+             ("Operator", "≥"),
+             ("Text", " "),
+             ("Operator", "≠"),
+             ("Text", " "),
+             ("Operator", "←"),
+             ("Text", " "),
+             ("Operator", "↑"),
+             ("Text", "\n"),
+             ],
+            tokens)
+        self.assertRaises(StopIteration, next, tokens)
+
+    def test_op_replace_complete(self):
+        tokens = pygments.lex("<= >= <> <- ^", self.lexer)
+        self.assertTokenStreamEqualComplete(
+            [("Operator", "≤"),
+             ("Text", " "),
+             ("Operator", "≥"),
+             ("Text", " "),
+             ("Operator", "≠"),
+             ("Text", " "),
+             ("Operator", "←"),
+             ("Text", " "),
+             ("Operator", "↑"),
+             ("Text", "\n"),
+             ],
+            tokens)
+
 
 if __name__ == "__main__":
     unittest.main()
-
-