view tests/test_fr.py @ 160:b4028838e0c8

Implement lexer option "prohibit_raiseonerror_filter". Sphinx raises by default when an Error token is seen (by means of the "raiseonerror" filter that is applied by default to lexers in Sphinx). This option skips this and allows error locations to be seen and highlighted properly. While there convert most Generic.Error tokens to Error tokens because now they can be handled by a lexer with "prohibit_raiseonerror_filter=True".
author Franz Glasner <fzglas.hg@dom66.de>
date Fri, 08 May 2026 17:46:28 +0200
parents c5fa8ee08059
children
line wrap: on
line source

# -*- coding: utf-8 -*-
# :-
# SPDX-FileCopyrightText: © 2026 Franz Glasner
# SPDX-License-Identifier: MIT
# :-

from _tsetup import FRLEXERFILENAME, FRLEXERCLASS

import unittest

import pygments
import pygments.lexers
from pygments.token import Token

import _testhelper


class TestLoading(unittest.TestCase):

    def test_load(self):
        pygments.lexers.load_lexer_from_file(FRLEXERFILENAME, FRLEXERCLASS)


class TestFrLexer(unittest.TestCase, _testhelper.TokenAssertHelper):

    def setUp(self):
        self.lexer = pygments.lexers.load_lexer_from_file(
            FRLEXERFILENAME, FRLEXERCLASS)

    def test_types(self):
        tokens = pygments.lex("huhu", self.lexer)
        self.assertEqual((Token.Name.Variable, "huhu"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, "\n"), next(tokens))
        self.assertRaises(StopIteration, next, tokens)

    def test_op_replace(self):
        tokens = pygments.lex("<= >= <> != <- -> => ^ < = > <-> <=>",
                              self.lexer)
        self.assertEqual((Token.Operator, "≤"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "≥"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "≠"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "≠"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "←"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "→"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "⇒"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "↑"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "<"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "="), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, ">"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "↔"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, " "), next(tokens))
        self.assertEqual((Token.Operator, "⇔"), next(tokens))
        self.assertEqual((Token.Text.Whitespace, "\n"), next(tokens))
        self.assertRaises(StopIteration, next, tokens)

    def test_op_replace_list(self):
        tokens = pygments.lex("<= >= <> != <- -> => ^ < = > <-> <=>",
                              self.lexer)
        self.assertTokenStreamEqual(
            [("Token.Operator", "≤"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "≥"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "≠"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "≠"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "←"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "→"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "⇒"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "↑"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "<"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "="),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", ">"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "↔"),
             ("Token.Text.Whitespace", " "),
             ("Token.Operator", "⇔"),
             ("Token.Text.Whitespace", "\n"),
             ],
            tokens)
        self.assertRaises(StopIteration, next, tokens)

    def test_op_replace_list_with_short_names(self):
        tokens = pygments.lex("<= >= <> != <- -> => ^ < = > <-> <=> ",
                              self.lexer)
        self.assertTokenStreamEqual(
            [("Operator", "≤"),
             ("Text.Whitespace", " "),
             ("Operator", "≥"),
             ("Text.Whitespace", " "),
             ("Operator", "≠"),
             ("Text.Whitespace", " "),
             ("Operator", "≠"),
             ("Text.Whitespace", " "),
             ("Operator", "←"),
             ("Text.Whitespace", " "),
             ("Operator", "→"),
             ("Text.Whitespace", " "),
             ("Operator", "⇒"),
             ("Text.Whitespace", " "),
             ("Operator", "↑"),
             ("Text.Whitespace", " "),
             ("Operator", "<"),
             ("Text.Whitespace", " "),
             ("Operator", "="),
             ("Text.Whitespace", " "),
             ("Operator", ">"),
             ("Text.Whitespace", " "),
             ("Operator", "↔"),
             ("Text.Whitespace", " "),
             ("Operator", "⇔"),
             ("Text.Whitespace", " \n"),
             ],
            tokens)
        self.assertRaises(StopIteration, next, tokens)

    def test_op_replace_complete(self):
        tokens = pygments.lex("<= >= <> != <- -> => ^ > = < <=> <->  ",
                              self.lexer)
        self.assertTokenStreamEqualComplete(
            [("Operator", "≤"),
             ("Text.Whitespace", " "),
             ("Operator", "≥"),
             ("Text.Whitespace", " "),
             ("Operator", "≠"),
             ("Text.Whitespace", " "),
             ("Operator", "≠"),
             ("Text.Whitespace", " "),
             ("Operator", "←"),
             ("Text.Whitespace", " "),
             ("Operator", "→"),
             ("Text.Whitespace", " "),
             ("Operator", "⇒"),
             ("Text.Whitespace", " "),
             ("Operator", "↑"),
             ("Text.Whitespace", " "),
             ("Operator", ">"),
             ("Text.Whitespace", " "),
             ("Operator", "="),
             ("Text.Whitespace", " "),
             ("Operator", "<"),
             ("Text.Whitespace", " "),
             ("Operator", "⇔"),
             ("Text.Whitespace", " "),
             ("Operator", "↔"),
             ("Text.Whitespace", "  \n"),
             ],
            tokens)


if __name__ == "__main__":
    unittest.main()