view tests/test_fr.py @ 56:661461fb4dfc

Make the "py-name" rules parameterized: allow to provide the token type. For this to work the implemting function must be global, because the access to the class is not yet possible at construction time. So consistently make some previons LexBase members module globals. Make some LexBase members module globals consistently.
author Franz Glasner <fzglas.hg@dom66.de>
date Mon, 27 Apr 2026 12:37:27 +0200
parents c5fa8ee08059
children
line wrap: on
line source

# -*- coding: utf-8 -*-
# :-
# SPDX-FileCopyrightText: © 2026 Franz Glasner
# SPDX-License-Identifier: MIT
# :-

from _tsetup import FRLEXERFILENAME, FRLEXERCLASS

import unittest

import pygments
import pygments.lexers
from pygments.token import Token

import _testhelper


class TestLoading(unittest.TestCase):

    def test_load(self):
        pygments.lexers.load_lexer_from_file(FRLEXERFILENAME, FRLEXERCLASS)


class TestFrLexer(unittest.TestCase, _testhelper.TokenAssertHelper):

    def setUp(self):
        self.lexer = pygments.lexers.load_lexer_from_file(
            FRLEXERFILENAME, FRLEXERCLASS)

    def test_types(self):
        tokens = pygments.lex("huhu", self.lexer)
        self.assertEqual((Token.Name.Variable, "huhu"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, "\n"), next(tokens))
        self.assertRaises(StopIteration, next, tokens)

    def test_op_replace(self):
        tokens = pygments.lex("<= >= <> != <- -> => ^ < = > <-> <=>",
                              self.lexer)
        self.assertEqual((Token.Operator, "≤"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "≥"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "≠"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "≠"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "←"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "→"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "⇒"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "↑"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "<"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "="), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, ">"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "↔"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "⇔"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, "\n"), next(tokens))
        self.assertRaises(StopIteration, next, tokens)

    def test_op_replace_list(self):
        tokens = pygments.lex("<= >= <> != <- -> => ^ < = > <-> <=>",
                              self.lexer)
        self.assertTokenStreamEqual(
            [("Token.Operator", "≤"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "≥"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "≠"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "≠"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "←"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "→"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "⇒"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "↑"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "<"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "="),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", ">"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "↔"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "⇔"),
             ("Token.Text.Whitespace", "\n"),
             ],
            tokens)
        self.assertRaises(StopIteration, next, tokens)

    def test_op_replace_list_with_short_names(self):
        tokens = pygments.lex("<= >= <> != <- -> => ^ < = > <-> <=> ",
                              self.lexer)
        self.assertTokenStreamEqual(
            [("Operator", "≤"),
             ("Text.Whitespace", " "),
             ("Operator", "≥"),
             ("Text.Whitespace", " "),
             ("Operator", "≠"),
             ("Text.Whitespace", " "),
             ("Operator", "≠"),
             ("Text.Whitespace", " "),
             ("Operator", "←"),
             ("Text.Whitespace", " "),
             ("Operator", "→"),
             ("Text.Whitespace", " "),
             ("Operator", "⇒"),
             ("Text.Whitespace", " "),
             ("Operator", "↑"),
             ("Text.Whitespace", " "),
             ("Operator", "<"),
             ("Text.Whitespace", " "),
             ("Operator", "="),
             ("Text.Whitespace", " "),
             ("Operator", ">"),
             ("Text.Whitespace", " "),
             ("Operator", "↔"),
             ("Text.Whitespace", " "),
             ("Operator", "⇔"),
             ("Text.Whitespace", " \n"),
             ],
            tokens)
        self.assertRaises(StopIteration, next, tokens)

    def test_op_replace_complete(self):
        tokens = pygments.lex("<= >= <> != <- -> => ^ > = < <=> <->  ",
                              self.lexer)
        self.assertTokenStreamEqualComplete(
            [("Operator", "≤"),
             ("Text.Whitespace", " "),
             ("Operator", "≥"),
             ("Text.Whitespace", " "),
             ("Operator", "≠"),
             ("Text.Whitespace", " "),
             ("Operator", "≠"),
             ("Text.Whitespace", " "),
             ("Operator", "←"),
             ("Text.Whitespace", " "),
             ("Operator", "→"),
             ("Text.Whitespace", " "),
             ("Operator", "⇒"),
             ("Text.Whitespace", " "),
             ("Operator", "↑"),
             ("Text.Whitespace", " "),
             ("Operator", ">"),
             ("Text.Whitespace", " "),
             ("Operator", "="),
             ("Text.Whitespace", " "),
             ("Operator", "<"),
             ("Text.Whitespace", " "),
             ("Operator", "⇔"),
             ("Text.Whitespace", " "),
             ("Operator", "↔"),
             ("Text.Whitespace", "  \n"),
             ],
            tokens)


if __name__ == "__main__":
    unittest.main()