annotate tests/test_fr.py @ 19:2e67c4eae6d9

Use an alternate replace algorithm with a dict
author Franz Glasner <fzglas.hg@dom66.de>
date Mon, 20 Apr 2026 15:25:45 +0200
parents 859ab8abce0a
children 3db60b64abf6
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
16
b92f660f2461 Begin a test infrastructure: using unittest and flake8.
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
1 # -*- coding: utf-8 -*-
b92f660f2461 Begin a test infrastructure: using unittest and flake8.
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
2 # :-
b92f660f2461 Begin a test infrastructure: using unittest and flake8.
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
3 # SPDX-FileCopyrightText: © 2026 Franz Glasner
b92f660f2461 Begin a test infrastructure: using unittest and flake8.
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
4 # SPDX-License-Identifier: MIT
b92f660f2461 Begin a test infrastructure: using unittest and flake8.
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
5 # :-
b92f660f2461 Begin a test infrastructure: using unittest and flake8.
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
6
18
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
7 from _tsetup import LEXERFILENAME, LEXERCLASS
16
b92f660f2461 Begin a test infrastructure: using unittest and flake8.
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
8
b92f660f2461 Begin a test infrastructure: using unittest and flake8.
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
9 import unittest
b92f660f2461 Begin a test infrastructure: using unittest and flake8.
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
10
18
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
11 import pygments
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
12 import pygments.lexers
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
13 from pygments.token import Token
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
14
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
15 import _testhelper
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
16
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
17
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
18 class TestLoading(unittest.TestCase):
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
19
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
20 def test_load(self):
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
21 pygments.lexers.load_lexer_from_file(LEXERFILENAME, LEXERCLASS)
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
22
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
23
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
24 class TestFrLexer(unittest.TestCase, _testhelper.TokenAssertHelper):
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
25
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
26 def setUp(self):
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
27 self.lexer = pygments.lexers.load_lexer_from_file(
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
28 LEXERFILENAME, LEXERCLASS)
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
29
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
30 def test_types(self):
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
31 tokens = pygments.lex("huhu", self.lexer)
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
32 self.assertEqual((Token.Name.Variable, "huhu"), next(tokens))
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
33 self.assertEqual((Token.Text, "\n"), next(tokens))
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
34 self.assertRaises(StopIteration, next, tokens)
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
35
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
36 def test_op_replace(self):
19
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
37 tokens = pygments.lex("<= >= <> <- ^ < = >", self.lexer)
18
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
38 self.assertEqual((Token.Operator, "≤"), next(tokens))
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
39 self.assertEqual((Token.Text, " "), next(tokens))
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
40 self.assertEqual((Token.Operator, "≥"), next(tokens))
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
41 self.assertEqual((Token.Text, " "), next(tokens))
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
42 self.assertEqual((Token.Operator, "≠"), next(tokens))
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
43 self.assertEqual((Token.Text, " "), next(tokens))
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
44 self.assertEqual((Token.Operator, "←"), next(tokens))
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
45 self.assertEqual((Token.Text, " "), next(tokens))
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
46 self.assertEqual((Token.Operator, "↑"), next(tokens))
19
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
47 self.assertEqual((Token.Text, " "), next(tokens))
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
48 self.assertEqual((Token.Operator, "<"), next(tokens))
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
49 self.assertEqual((Token.Text, " "), next(tokens))
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
50 self.assertEqual((Token.Operator, "="), next(tokens))
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
51 self.assertEqual((Token.Text, " "), next(tokens))
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
52 self.assertEqual((Token.Operator, ">"), next(tokens))
18
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
53 self.assertEqual((Token.Text, "\n"), next(tokens))
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
54 self.assertRaises(StopIteration, next, tokens)
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
55
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
56 def test_op_replace_list(self):
19
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
57 tokens = pygments.lex("<= >= <> <- ^ < = >", self.lexer)
18
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
58 self.assertTokenStreamEqual(
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
59 [("Token.Operator", "≤"),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
60 ("Token.Text", " "),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
61 ("Token.Operator", "≥"),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
62 ("Token.Text", " "),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
63 ("Token.Operator", "≠"),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
64 ("Token.Text", " "),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
65 ("Token.Operator", "←"),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
66 ("Token.Text", " "),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
67 ("Token.Operator", "↑"),
19
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
68 ("Token.Text", " "),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
69 ("Token.Operator", "<"),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
70 ("Token.Text", " "),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
71 ("Token.Operator", "="),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
72 ("Token.Text", " "),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
73 ("Token.Operator", ">"),
18
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
74 ("Token.Text", "\n"),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
75 ],
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
76 tokens)
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
77 self.assertRaises(StopIteration, next, tokens)
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
78
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
79 def test_op_replace_list_with_short_names(self):
19
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
80 tokens = pygments.lex("<= >= <> <- ^ < = > ", self.lexer)
18
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
81 self.assertTokenStreamEqual(
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
82 [("Operator", "≤"),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
83 ("Text", " "),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
84 ("Operator", "≥"),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
85 ("Text", " "),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
86 ("Operator", "≠"),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
87 ("Text", " "),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
88 ("Operator", "←"),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
89 ("Text", " "),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
90 ("Operator", "↑"),
19
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
91 ("Text", " "),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
92 ("Operator", "<"),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
93 ("Text", " "),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
94 ("Operator", "="),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
95 ("Text", " "),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
96 ("Operator", ">"),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
97 ("Text", " \n"),
18
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
98 ],
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
99 tokens)
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
100 self.assertRaises(StopIteration, next, tokens)
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
101
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
102 def test_op_replace_complete(self):
19
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
103 tokens = pygments.lex("<= >= <> <- ^ > = < ", self.lexer)
18
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
104 self.assertTokenStreamEqualComplete(
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
105 [("Operator", "≤"),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
106 ("Text", " "),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
107 ("Operator", "≥"),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
108 ("Text", " "),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
109 ("Operator", "≠"),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
110 ("Text", " "),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
111 ("Operator", "←"),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
112 ("Text", " "),
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
113 ("Operator", "↑"),
19
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
114 ("Text", " "),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
115 ("Operator", ">"),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
116 ("Text", " "),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
117 ("Operator", "="),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
118 ("Text", " "),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
119 ("Operator", "<"),
2e67c4eae6d9 Use an alternate replace algorithm with a dict
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
120 ("Text", " \n"),
18
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
121 ],
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
122 tokens)
859ab8abce0a The first real tests about lexing with the Pygments API for the tests.
Franz Glasner <fzglas.hg@dom66.de>
parents: 16
diff changeset
123
16
b92f660f2461 Begin a test infrastructure: using unittest and flake8.
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
124
b92f660f2461 Begin a test infrastructure: using unittest and flake8.
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
125 if __name__ == "__main__":
b92f660f2461 Begin a test infrastructure: using unittest and flake8.
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
126 unittest.main()