view tests/test_fr.py @ 18:859ab8abce0a

The first real tests about lexing with the Pygments API for the tests. Also implemented a simple helper to compare token streams with shorter code.
author Franz Glasner <fzglas.hg@dom66.de>
date Mon, 20 Apr 2026 14:12:35 +0200
parents b92f660f2461
children 2e67c4eae6d9
line wrap: on
line source

# -*- coding: utf-8 -*-
# :-
# SPDX-FileCopyrightText: © 2026 Franz Glasner
# SPDX-License-Identifier: MIT
# :-

from _tsetup import LEXERFILENAME, LEXERCLASS

import unittest

import pygments
import pygments.lexers
from pygments.token import Token

import _testhelper


class TestLoading(unittest.TestCase):

    def test_load(self):
        pygments.lexers.load_lexer_from_file(LEXERFILENAME, LEXERCLASS)


class TestFrLexer(unittest.TestCase, _testhelper.TokenAssertHelper):

    def setUp(self):
        self.lexer = pygments.lexers.load_lexer_from_file(
            LEXERFILENAME, LEXERCLASS)

    def test_types(self):
        tokens = pygments.lex("huhu", self.lexer)
        self.assertEqual((Token.Name.Variable, "huhu"), next(tokens))
        self.assertEqual((Token.Text, "\n"), next(tokens))
        self.assertRaises(StopIteration, next, tokens)

    def test_op_replace(self):
        tokens = pygments.lex("<= >= <> <- ^", self.lexer)
        self.assertEqual((Token.Operator, "≤"), next(tokens))
        self.assertEqual((Token.Text, " "), next(tokens))
        self.assertEqual((Token.Operator, "≥"), next(tokens))
        self.assertEqual((Token.Text, " "), next(tokens))
        self.assertEqual((Token.Operator, "≠"), next(tokens))
        self.assertEqual((Token.Text, " "), next(tokens))
        self.assertEqual((Token.Operator, "←"), next(tokens))
        self.assertEqual((Token.Text, " "), next(tokens))
        self.assertEqual((Token.Operator, "↑"), next(tokens))
        self.assertEqual((Token.Text, "\n"), next(tokens))
        self.assertRaises(StopIteration, next, tokens)

    def test_op_replace_list(self):
        tokens = pygments.lex("<= >= <> <- ^", self.lexer)
        self.assertTokenStreamEqual(
            [("Token.Operator", "≤"),
             ("Token.Text", " "),
             ("Token.Operator", "≥"),
             ("Token.Text", " "),
             ("Token.Operator", "≠"),
             ("Token.Text", " "),
             ("Token.Operator", "←"),
             ("Token.Text", " "),
             ("Token.Operator", "↑"),
             ("Token.Text", "\n"),
             ],
            tokens)
        self.assertRaises(StopIteration, next, tokens)

    def test_op_replace_list_with_short_names(self):
        tokens = pygments.lex("<= >= <> <- ^", self.lexer)
        self.assertTokenStreamEqual(
            [("Operator", "≤"),
             ("Text", " "),
             ("Operator", "≥"),
             ("Text", " "),
             ("Operator", "≠"),
             ("Text", " "),
             ("Operator", "←"),
             ("Text", " "),
             ("Operator", "↑"),
             ("Text", "\n"),
             ],
            tokens)
        self.assertRaises(StopIteration, next, tokens)

    def test_op_replace_complete(self):
        tokens = pygments.lex("<= >= <> <- ^", self.lexer)
        self.assertTokenStreamEqualComplete(
            [("Operator", "≤"),
             ("Text", " "),
             ("Operator", "≥"),
             ("Text", " "),
             ("Operator", "≠"),
             ("Text", " "),
             ("Operator", "←"),
             ("Text", " "),
             ("Operator", "↑"),
             ("Text", "\n"),
             ],
            tokens)


if __name__ == "__main__":
    unittest.main()