# -*- coding: utf-8 -*-
# :-
# SPDX-FileCopyrightText: © 2026 Franz Glasner
# SPDX-License-Identifier: MIT
# :-

from _tsetup import ALGLEXERFILENAME, ALGLEXERCLASS

import unittest

import pygments
import pygments.lexers
import pygments.formatters

import _testhelper


class TestSnippets(unittest.TestCase, _testhelper.TokenAssertHelper):

    def setUp(self):
        self.lexer = pygments.lexers.load_lexer_from_file(
            ALGLEXERFILENAME, ALGLEXERCLASS)

    def test_lf(self):
        self.assertTokenStreamEqualComplete(
            [("Text.Whitespace", "\n")],
            pygments.lex("\n", self.lexer))

    def test_protected_lf(self):
        self.assertTokenStreamEqualComplete(
            [("Text", "\\\n")],
            pygments.lex("\\\n", self.lexer))

    def test_number_int(self):
        self.assertTokenStreamEqualComplete(
            [("Number.Integer", "10"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("10", self.lexer))

    def test_number_float_1(self):
        self.assertTokenStreamEqualComplete(
            [("Number.Float", "3.1415926"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("3.1415926", self.lexer))

    def test_number_float_2(self):
        self.assertTokenStreamEqualComplete(
            [("Number.Float", "3.14e-12"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("3.14e-12", self.lexer))

    def test_string_s_1(self):
        self.assertTokenStreamEqualComplete(
            [("String.Single", "'"),
             ("String.Single", "HU"),
             ("String.Single", '"'),
             ("String.Single", "HE HA"),
             ("String.Escape", "\\'"),
             ("String.Single", "HO"),
             ("String.Single", "'"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("""'HU"HE HA\\'HO'""", self.lexer))

    def test_string_s_2(self):
        self.assertTokenStreamEqual(
            [("String.Single", "'"),
             ("String.Single", "HUHU"),
             ("Error", "\n"),
             ],
            pygments.lex("'HUHU\nHEHE'", self.lexer))

    def test_string_ts_1(self):
        self.assertTokenStreamEqualComplete(
            [("String.Single", "'''"),
             ("String.Single", "HUHU HEHE"),
             ("String.Single", "'''"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("'''HUHU HEHE'''", self.lexer))

    def test_string_ts_2(self):
        self.assertTokenStreamEqualComplete(
            [("String.Single", "'''"),
             ("String.Single", "HI"),
             ("String.Single", "'"),
             ("String.Single", "HU"),
             ("String.Single", "\n"),
             ("String.Single", "HE"),
             ("String.Single", '"'),
             ("String.Single", "HA"),
             ("String.Single", "'''"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("""'''HI'HU\nHE"HA'''""", self.lexer))

    def test_string_d_1(self):
        self.assertTokenStreamEqualComplete(
            [("String.Double", '"'),
             ("String.Double", 'HU'),
             ("String.Double", "'"),
             ("String.Double", 'HE HA'),
             ("String.Escape", '\\"'),
             ("String.Double", 'HO'),
             ("String.Double", '"'),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex('''"HU'HE HA\\"HO"''', self.lexer))

    def test_string_d_2(self):
        self.assertTokenStreamEqual(
            [("String.Double", '"'),
             ("String.Double", "HUHU"),
             ("Error", "\n"),
             ],
            pygments.lex('"HUHU\nHEHE"', self.lexer))

    def test_string_td_1(self):
        self.assertTokenStreamEqualComplete(
            [("String.Double", '"""'),
             ("String.Double", 'HUHU HAHA'),
             ("String.Double", '"""'),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex('"""HUHU HAHA"""', self.lexer))

    def test_string_td_2(self):
        self.assertTokenStreamEqualComplete(
            [("String.Double", '"""'),
             ("String.Double", 'HU'),
             ("String.Double", '"'),
             ("String.Double", "HO"),
             ("String.Double", "\n"),
             ("String.Double", "HE"),
             ("String.Double", "'"),
             ("String.Double", "HA"),
             ("String.Double", '"""'),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex('''"""HU"HO\nHE'HA"""''', self.lexer))

    def test_proc(self):
        self.assertTokenStreamEqualComplete(
            [("Keyword", "PROCEDURE"),
             ("Text.Whitespace", " "),
             ("Name.Entity", "the name"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("\\PROC {the name}", self.lexer))

    def test_proc_with_symbols_in_name(self):
        self.assertTokenStreamEqualComplete(
            [("Keyword", "PROCEDURE"),
             ("Text.Whitespace", " "),
             ("Name.Entity", "the name sqrt ! <="),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("\\PROC {the name sqrt ! <=}", self.lexer))

    def test_proc_with_escape_in_name(self):
        self.assertTokenStreamEqualComplete(
            [("Keyword", "PROCEDURE"),
             ("Text.Whitespace", " "),
             ("Name.Entity", "t"),
             ("Generic.Error", "\\"),
             ("Name.Entity", "he "),
             ("Name.Entity", "\\"),
             ("Name.Entity", "}"),
             ("Name.Entity", "name"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\PROC {t\he \\\}name}", self.lexer))

    def test_endproc(self):
        self.assertTokenStreamEqualComplete(
            [("Keyword", "END OF PROCEDURE"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("\\END-PROCEDURE", self.lexer))

    def test_endproc_with_entityname(self):
        self.assertTokenStreamEqualComplete(
            [("Keyword", "END OF PROCEDURE"),
             ("Text.Whitespace", " "),
             ("Name.Entity", "the procedure name"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("\\ENDPROCEDURE {the procedure name}", self.lexer))

    def test_endproc_with_entityname_2(self):
        self.assertTokenStreamEqualComplete(
            [("Keyword", "END OF PROCEDURE"),
             ("Text.Whitespace", " "),
             ("Name.Entity", "the procedure name"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("\\END-PROC {the procedure name}", self.lexer))

    def test_endproc_with_entityname_3(self):
        self.assertTokenStreamEqualComplete(
            [("Keyword", "END OF PROCEDURE"),
             ("Text.Whitespace", " "),
             ("Name.Entity", "the procedure name with pow and symbols ! <= "),
             ("Name.Entity", "}"),
             ("Name.Entity", "<-"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(
                "\\END-PROC"
                " {the procedure name with pow and symbols ! <= \\}<-}",
                self.lexer))

    def test_proc_de(self):
        lexer = pygments.lexers.load_lexer_from_file(
            ALGLEXERFILENAME, "AlgPseudocodeLexer_DE")
        self.assertTokenStreamEqualComplete(
            [("Keyword", "PROZEDUR"),
             ("Text.Whitespace", " "),
             ("Name.Entity", " also {nichtxs"),
             ("Name.Entity", "}"),
             ("Name.Entity", " hier"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\PROC  { also {nichtxs\} hier}", lexer))

    def test_loop_de_not_yet_translated(self):
        lexer = pygments.lexers.load_lexer_from_file(
            ALGLEXERFILENAME, "AlgPseudocodeLexer_DE")
        self.assertTokenStreamEqualComplete(
            [("Keyword", "LOOP"),
             ("Text", " "),
             ("Name.Entity", "a"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\LOOP a", lexer))

    def test_function_1(self):
        self.assertTokenStreamEqualComplete(
            [("Keyword", "FUNCTION"),
             ("Text.Whitespace", " "),
             ("Name.Entity", "1"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("\\FUNC{1}", self.lexer))

    def test_function_2(self):
        self.assertTokenStreamEqualComplete(
            [("Keyword", "FUNCTION"),
             ("Text.Whitespace", " "),
             ("Name.Entity", "line 1\nline 2\n"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("\\FUNCTION{line 1\nline 2\n}", self.lexer))

    def test_function_3(self):
        self.assertTokenStreamEqualComplete(
            [("Keyword", "FUNCTION"),
             ("Text.Whitespace", " "),
             ("Generic.Error", "\\"),
             ("Name.Entity", "n"),
             ("Name.Entity", "}"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\FN {\n\}}", self.lexer))

    def test_class(self):
        self.assertTokenStreamEqualComplete(
            [("Keyword", "CLASS"),
             ("Text.Whitespace", " "),
             ("Generic.Error", "\\"),
             ("Name.Entity", "n"),
             ("Name.Entity", "\\"),
             ("Name.Entity", "}"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\CLASS {\n\\\}}", self.lexer))

    def test_class_fr(self):
        lexer = pygments.lexers.load_lexer_from_file(
            ALGLEXERFILENAME, "AlgPseudocodeLexer_FR")
        self.assertTokenStreamEqualComplete(
            [("Keyword", "CLASSE"),
             ("Text.Whitespace", " "),
             ("Generic.Error", "\\"),
             ("Name.Entity", "n"),
             ("Name.Entity", "}"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("\\CLASS {\\n\\}}", lexer))

    def test_class_de_with_noend_option(self):
        lexer = pygments.lexers.load_lexer_from_file(
            ALGLEXERFILENAME, "AlgPseudocodeLexer_DE", no_end="True")
        self.assertTokenStreamEqualComplete(
            [("Keyword", "KLASSE"),
             ("Text.Whitespace", " "),
             ("Name.Entity", "\\"),
             ("Name.Entity", "n"),
             ("Name.Entity", "}"),
             ("Text.Whitespace", "\n"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("\\CLASS {\\\\n\\}}\n\\ENDCLASS", lexer))

    def test_class_de_with_noend_option_and_name(self):
        lexer = pygments.lexers.load_lexer_from_file(
            ALGLEXERFILENAME, "AlgPseudocodeLexer_DE", no_end="True")
        self.assertTokenStreamEqualComplete(
            [("Keyword", "KLASSE"),
             ("Text.Whitespace", " "),
             ("Generic.Error", "\\"),
             ("Name.Entity", "n"),
             ("Name.Entity", "}"),
             ("Text.Whitespace", "\n"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("\\CLASS {\\n\\}}\n\\ENDCLASS {end class}", lexer))

    def test_remark_1(self):
        self.assertTokenStreamEqualComplete(
            [("Comment.Single", u"▷"),
             ("Comment.Single", "  the remark"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("\\REMARK  the remark\n", self.lexer))

    def test_remark_2(self):
        self.assertTokenStreamEqualComplete(
            [("Comment.Single", u"▷"),
             ("Comment.Single", "  the remark 1"),
             ("Text.Whitespace", "\n"),
             ("Comment.Single", u"▷"),
             ("Comment.Single", "  the remark 2"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(
                """\\remArk  the remark 1
\\Rem  the remark 2
""", self.lexer))

    def test_remark_custom(self):
        # With the APL comment symbol
        lexer = pygments.lexers.load_lexer_from_file(
            ALGLEXERFILENAME, ALGLEXERCLASS, remark=u"⍝")
        self.assertTokenStreamEqualComplete(
            [("Comment.Single", u"⍝"),  # U+235D ⍝ APL FUNC. SYMBOL UP SHOE JOT
             ("Comment.Single", "  another remark"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(u"\\REMARK  another remark\n", lexer))

    def test_remark_in_text(self):
        self.assertTokenStreamEqualComplete(
            [("Text", "the text  "),
             ("Comment.Single", u"▷"),
             ("Comment.Single", " the remark"),
             ("Text.Whitespace", "\n"),
             ("Text", "the next text line"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(
                """\\TEXT{the text  \\rem the remark\nthe next text line}""",
                self.lexer))

    def test_comment_single_1(self):
        self.assertTokenStreamEqualComplete(
            [("Comment.Single", "// foo bar"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"// foo bar", self.lexer))

    def test_comment_single_2(self):
        self.assertTokenStreamEqualComplete(
            [("Comment.Single", "#  foo bar"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"#  foo bar", self.lexer))

    def test_expr_and_text(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Entity", "a"),
             ("Text", " "),
             ("Text", "multiplied by"),
             ("Text", " "),
             ("Name.Entity", "b"),
             ("Text", " "),
             ("Text", "is"),
             ("Text", " "),
             ("Number.Integer", "0"),
             ("Text", "  "),
             ("Text", "mod"),
             ("Text", " "),
             ("Name.Entity", "p"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(
                r"""a \TEXT{multiplied by} b \text{is} 0  \text{mod} p""",
                self.lexer))

    def test_expr_and_text_with_escaped_characters(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Entity", "a"),
             ("Text", " "),
             ("Text", "multiplied by {escaped"),
             ("Text", "}"),
             ("Text", " "),
             ("Text", " "),
             ("Name.Entity", "b"),
             ("Text", " "),
             ("Text", "is "),
             ("Text", "\\"),
             ("Text", " not"),
             ("Text", " "),
             ("Number.Integer", "0"),
             ("Text", "  "),
             ("Text", "mod"),
             ("Text", " "),
             ("Name.Entity", "p"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(
                r"""a \TEXT{multiplied by {escaped\} } b \text{is \ not} 0"""
                r"  \text{mod} p""",
                self.lexer))

    def test_punctuation(self):
        self.assertTokenStreamEqualComplete(
            [("Punctuation", "{"),
             ("Punctuation", "}"),
             ("Punctuation", ":"),
             ("Punctuation", "("),
             ("Punctuation", ")"),
             ("Punctuation", ","),
             ("Punctuation", ";"),
             ("Punctuation", "["),
             ("Punctuation", "]"),
             ("Punctuation", "?"),
             ("Punctuation", "@"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"{}:(),;[]?@", self.lexer))

    def test_block_empty(self):
        self.assertTokenStreamEqualComplete(
            [("Text", u"◆"),
             ("Text.Whitespace", " "),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\block{}", self.lexer))

    def test_block_with_text(self):
        self.assertTokenStreamEqualComplete(
            [("Text", u"◆"),
             ("Text.Whitespace", " "),
             ("Text", "a b c"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\block{\text{a b c}}", self.lexer))

    def test_block_with_escaped_text(self):
        self.assertTokenStreamEqualComplete(
            [("Text", u"◆"),
             ("Text.Whitespace", " "),
             ("Text", "\\"),
             ("Name.Entity", "text"),
             ("Punctuation", "{"),
             ("Name.Entity", "a"),
             ("Text", " "),
             ("Name.Entity", "b"),
             ("Text", " "),
             ("Name.Entity", "c"),
             ("Punctuation", "}"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\block{\\text{a b c\}}", self.lexer))

    def test_block(self):
        self.assertTokenStreamEqualComplete(
            [("Text", u"◆"),
             ("Text.Whitespace", " "),
             ("Name.Entity", "a"),
             ("Text", " "),
             ("Number.Float", "1.2"),
             ("Text", " "),
             ("Punctuation", "{"),
             ("Name.Entity", "x"),
             ("Text", " "),
             ("Operator.Word", "in"),
             ("Text", " "),
             ("Name.Entity", "X"),
             ("Punctuation", "}"),
             ("Text", " "),
             ("Name.Entity", "c"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\state{a 1.2 {x in X\} c}", self.lexer))

    def test_tstate_empty(self):
        self.assertTokenStreamEqualComplete(
            [("Text", u"▪"),
             ("Text.Whitespace", " "),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\tstate{}", self.lexer))

    def test_tstate_with_expr(self):
        self.assertTokenStreamEqualComplete(
            [("Text", u"▪"),
             ("Text.Whitespace", " "),
             ("Text", "a 1.2 "),
             ("Name.Entity", "x"),
             ("Text", " "),
             ("Operator.Word", "in"),
             ("Text", " "),
             ("Name.Entity", "X"),
             ("Punctuation", "}"),
             ("Text", " c"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\tstate{a 1.2 \expr{x in X\}} c}", self.lexer))

    def test_tstate_with_escaped_expr(self):
        self.assertTokenStreamEqualComplete(
            [("Text", u"▪"),
             ("Text.Whitespace", " "),
             ("Text", "a 1.2 "),
             ("Text", "\\"),
             ("Text", "expr{x in X"),
             ("Text", "}"),
             ("Text", " c"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\tstate{a 1.2 \\expr{x in X\} c}", self.lexer))

    def test_text_in_expr(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Entity", "first"),
             ("Text", " "),
             ("Name.Entity", "expression"),
             ("Text", "  "),
             ("Name.Entity", "second"),
             ("Text", " "),
             ("Name.Entity", "expression"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(
                r"\TEXT{\EXPRESSION{first expression}  "
                r"\EXPR{second expression}}",
                self.lexer))

    def test_explicit_extended_single_tokentype_1(self):
        self.assertTokenStreamEqualComplete(
            [("Operator", "%"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\ttx-o:%:", self.lexer))

    def test_explicit_extended_single_tokentype_2(self):
        self.assertTokenStreamEqualComplete(
            [("Operator", "{"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\ttx-o{{}", self.lexer))

    def test_explicit_extended_single_tokentype_3(self):
        self.assertTokenStreamEqualComplete(
            [("Operator", "<"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\ttx-o<<>", self.lexer))

    def test_explicit_extended_single_tokentype_4(self):
        self.assertTokenStreamEqualComplete(
            [("Operator", "("),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\ttx-o(()", self.lexer))

    def test_explicit_extended_multi_tokentype_1(self):
        self.assertTokenStreamEqualComplete(
            [("Operator", "xxx in A"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\ttx-o/xxx in A/", self.lexer))

    def test_explicit_extended_multi_tokentype_2(self):
        self.assertTokenStreamEqualComplete(
            [("Operator", "xxx in B"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\ttx-o{xxx in B}", self.lexer))

    def test_explicit_extended_multi_tokentype_3(self):
        self.assertTokenStreamEqualComplete(
            [("Operator", "xxx in C"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\ttx-o<xxx in C>", self.lexer))

    def test_explicit_extended_multi_tokentype_4(self):
        self.assertTokenStreamEqualComplete(
            [("Operator", "xxx in D"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\ttx-o(xxx in D)", self.lexer))

    def test_explicit_tokentype_simple_1(self):
        self.assertTokenStreamEqualComplete(
            [("Operator", "}"),
             ("Operator", "/"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\tt-o/}\tt-o//", self.lexer))

    def test_explicit_tokentype_simple_2(self):
        self.assertTokenStreamEqualComplete(
            [("Operator", "\n"),
             ("Operator", "/"),
             ("Text", " "),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("\\tt-o/\n\\tt-o// ", self.lexer))

    def test_explicit_tokentype_with_remark(self):
        self.assertTokenStreamEqualComplete(
            [("Operator", u"∈ ∌"),
             ("Text", "    "),
             ("Comment.Single", u"▷"),
             ("Comment.Single", u" ∈ ∌ as (ordinary) operators"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(
                u"""\\ttx-o<∈ ∌>    \\rem ∈ ∌ as (ordinary) operators""",
                self.lexer))

    def test_explicit_tokentype_with_remark_2(self):
        self.assertTokenStreamEqualComplete(
            [("Operator", "new_operator"),
             ("Text", "  "),
             ("Comment.Single", u"▷"),
             ("Comment.Single", " a (synthesized) operator"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(
                r"""\ttx-o<new_operator>  \rem a (synthesized) operator""",
                self.lexer))

    def test_explicit_tokentype_with_possibly_conflicting_parens(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Function", u"∈_∌"),
             ("Punctuation", "("),
             ("Name.Entity", "p1"),
             ("Punctuation", ","),
             ("Text", " "),
             ("Name.Entity", "p2"),
             ("Punctuation", ")"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(
                u"""\\ttx-nf<∈_∌>(p1, p2)""",
                self.lexer))

    def test_explicit_tokentype_with_possibly_conflicting_parens_2(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Decorator", "a_Decorator"),
             ("Punctuation", "("),
             ("Name.Entity", "p1"),
             ("Punctuation", ","),
             ("Text", " "),
             ("Name.Entity", "p2"),
             ("Punctuation", ")"),
             ("Text", "   "),
             ("Comment.Single", u"▷"),
             ("Comment.Single", " (Python) decorator"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(
                r"""\ttx-nd<a_Decorator>(p1, p2)   \rem (Python) decorator""",
                self.lexer))

    def test_explicit_tokentype_with_possibly_conflicting_parens_3(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Decorator", "a_Decorator"),
             ("Punctuation", "("),
             ("Name.Entity", "p1"),
             ("Punctuation", ","),
             ("Text", " "),
             ("Name.Entity", "p2"),
             ("Punctuation", ")"),
             ("Text", "   "),
             ("Comment.Single", u"▷"),
             ("Comment.Single", " (Python) annotation"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(
                r"""\ttx-nd[a_Decorator](p1, p2)   \rem (Python) annotation""",
                self.lexer))

    def test_explicit_tokentype_error(self):
        self.assertTokenStreamEqualComplete(
            [("Generic.Error", r"""\ttx-non-existing[a_Decorator]"""),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(
                r"""\ttx-non-existing[a_Decorator]""",
                self.lexer))

    def test_end_combinations(self):
        self.assertTokenStreamEqualComplete(
            [("Keyword", "BEGIN"),
             ("Text", "  "),
             ("Keyword", "END"),
             ("Text", "  "),
             ("Keyword", "END OF FUNCTION"),
             ("Text", "  "),
             ("Keyword", "END OF FUNCTION"),
             ("Text.Whitespace", " "),
             ("Name.Entity", "An End of a Function"),
             ("Text", "  "),
             ("Keyword", "END OF FUNCTION"),
             ("Text.Whitespace", " "),
             ("Name.Entity", "The End of the Next Function"),
             ("Text", " "),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(
                r"""\begin  \end  \end-fn  \end-fn {An End of a Function}"""
                r"""  \end fn {The End of the Next Function} """,
                self.lexer))

    def test_unicode_math_operator(self):
        self.assertTokenStreamEqualComplete(
            [("Operator", u"∈"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(u"∈", self.lexer))

    def test_ascii_math_operator(self):
        self.assertTokenStreamEqualComplete(
            [("Operator", "="),
             ("Operator", "!"),
             ("Operator", "&"),
             ("Operator", "<"),
             ("Text", " "),
             ("Operator", ">"),
             ("Operator", "+"),
             ("Operator", "-"),
             ("Operator", "*"),
             ("Operator", "/"),
             ("Operator", "%"),
             ("Operator", "|"),
             ("Operator", "~"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("=!&< >+-*/%|~", self.lexer))

    def test_ascii_math_operator_with_replacements(self):
        self.assertTokenStreamEqualComplete(
            [("Operator", u"⇔"),
             ("Text", " "),
             ("Operator", u"⟷"),
             ("Text", " "),
             ("Operator", u"⟵"),
             ("Text", " "),
             ("Operator", u"⟶"),
             ("Text", " "),
             ("Operator", u"⇒"),
             ("Text", " "),
             ("Operator", u"≤"),
             ("Text", " "),
             ("Operator", u"≥"),
             ("Text", " "),
             ("Operator", u"≠"),
             ("Text", " "),
             ("Operator", u"≠"),
             ("Text", " "),
             ("Operator", u"∶="),   # u"≔"),
             ("Text", " "),
             ("Operator", u"=∶"),   # u"≕"),
             ("Text", " "),
             ("Operator", u"≟"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(u"<=> <-> <- -> => <= >= <> != := =: ?=", self.lexer))

    def test_word_operators(self):
        self.assertTokenStreamEqualComplete(
            [("Operator.Word", "IN"),
             ("Text", " "),
             ("Operator.Word", "is"),
             ("Text", " "),
             ("Operator.Word", "And"),
             ("Text", " "),
             ("Operator.Word", "Or"),
             ("Text", " "),
             ("Operator.Word", "XOR"),
             ("Text", " "),
             ("Operator.Word", "not"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("IN is And Or XOR not", self.lexer))

    def test_keyword_constants(self):
        self.assertTokenStreamEqualComplete(
            [("Keyword.Constant", "true"),
             ("Text", " "),
             ("Keyword.Constant", "FALSE"),
             ("Text", " "),
             ("Keyword.Constant", "None"),
             ("Text", " "),
             ("Keyword.Constant", "nil"),
             ("Text", " "),
             ("Keyword.Constant", "NULL"),
             ("Text", " "),
             ("Keyword.Constant", "Empty"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("true FALSE None nil NULL Empty", self.lexer))

    def test_math_builtins(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Builtin", "sqrt"),
             ("Punctuation", "("),
             ("Name.Entity", "Foo"),
             ("Punctuation", ")"),
             ("Punctuation", ";"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("sqrt(Foo);", self.lexer))

    def test_math_builtins_nested(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Builtin", "pow"),
             ("Punctuation", "("),
             ("Number.Integer", "2"),
             ("Punctuation", ","),
             ("Text", " "),
             ("Number.Integer", "8"),
             ("Punctuation", ")"),
             ("Punctuation", ";"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex("\\text{\\expr{pow(2, 8);}}", self.lexer))

    def test_math_letters(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Entity", "a"),
             ("Text", " "),
             ("Operator.Word", "in"),
             ("Text", " "),
             ("Name.Entity", u"ℂ"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(u"a in ℂ", self.lexer))

    def test_other_symbols(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Entity", "b"),
             ("Text", " "),
             ("Operator.Word", "in"),
             ("Text", " "),
             ("Text", u"℀"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(u"b in ℀", self.lexer))

    def test_escaped_string_start_1(self):
        self.assertTokenStreamEqualComplete(
            [("Text", "flow "),
             ("Name.Entity", "f"),
             ("Punctuation", "'"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\TEXT{flow \expr{f\'}}", self.lexer))

    def test_escaped_string_start_2(self):
        self.assertTokenStreamEqualComplete(
            [("Text", "flow "),
             ("Name.Entity", "f"),
             ("Punctuation", '"'),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r'\TEXT{flow \expr{f\"}}', self.lexer))

    def test_gets_default(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Entity", "a"),
             ("Text", " "),
             ("Operator", u"⟵"),     # U+27F5 (Supplemental Arrows-A)
             ("Text", " "),
             ("Number.Integer", "2"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"a \gets 2", self.lexer))

    def test_gets_custom(self):
        lexer = pygments.lexers.load_lexer_from_file(
            ALGLEXERFILENAME, ALGLEXERCLASS, gets="::=")
        self.assertTokenStreamEqualComplete(
            [("Name.Entity", "a"),
             ("Text", " "),
             ("Operator", "::="),
             ("Text", " "),
             ("Number.Float", "2.7"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"a \gets 2.7", lexer))

    def test_call(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Entity", "a function"),
             ("Punctuation", "("),
             ("Name.Entity", "p1"),
             ("Punctuation", ","),
             ("Text", " "),
             ("Name.Entity", "p2"),
             ("Punctuation", ")"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\CALL{a function}(p1, p2)", self.lexer))

    def test_call_in_expr(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Entity", "a procedure"),
             ("Punctuation", "("),
             ("Name.Entity", "arg1"),
             ("Punctuation", ","),
             ("Text", " "),
             ("Name.Entity", "arg2"),
             ("Punctuation", ","),
             ("Text", " "),
             ("Name.Entity", "arg3"),
             ("Punctuation", ")"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\TEXT{\EXPR{\CALL{a procedure}(arg1, arg2, arg3)}}",
                         self.lexer))

    def test_name(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Entity", "an entity"),
             ("Punctuation", "("),
             ("Name.Entity", "p1"),
             ("Punctuation", ","),
             ("Text", " "),
             ("Name.Entity", "p2"),
             ("Punctuation", ")"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\NAME{an entity}(p1, p2)", self.lexer))

    def test_name_in_expr(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Entity", "an entity"),
             ("Punctuation", "("),
             ("Name.Entity", "arg1"),
             ("Punctuation", ","),
             ("Text", " "),
             ("Name.Entity", "arg2"),
             ("Punctuation", ","),
             ("Text", " "),
             ("Name.Entity", "arg3"),
             ("Punctuation", ")"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\TEXT{\EXPR{\NAME{an entity}(arg1, arg2, arg3)}}",
                         self.lexer))

    def test_comment_multi(self):
        self.assertTokenStreamEqualComplete(
            [("Comment.Multiline", "/*"),
             ("Comment.Multiline", " word1 "),
             ("Comment.Multiline", "*"),
             ("Comment.Multiline", " word2 "),
             ("Comment.Multiline", "/"),
             ("Comment.Multiline", " word3 "),
             ("Comment.Multiline", "/"),
             ("Comment.Multiline", "/"),
             ("Comment.Multiline", " word4 "),
             ("Comment.Multiline", "*/"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"/* word1 * word2 / word3 // word4 */", self.lexer))

    def test_comment_multi_alt(self):
        self.assertTokenStreamEqualComplete(
            [("Comment.Multiline", "(*"),
             ("Comment.Multiline", " word1 "),
             ("Comment.Multiline", "*"),
             ("Comment.Multiline", " word2 "),
             ("Comment.Multiline", "("),
             ("Comment.Multiline", " word3 "),
             ("Comment.Multiline", ")"),
             ("Comment.Multiline", " word4 "),
             ("Comment.Multiline", "("),
             ("Comment.Multiline", ")"),
             ("Comment.Multiline", " word5 "),
             ("Comment.Multiline", "*)"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"(* word1 * word2 ( word3 ) word4 () word5 *)",
                         self.lexer))

    def test_dots_one(self):
        self.assertTokenStreamEqualComplete(
            [("Punctuation", "."),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r".", self.lexer))

    def test_dots_two(self):
        self.assertTokenStreamEqualComplete(
            [("Punctuation", ".."),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"..", self.lexer))

    def test_dots_three(self):
        self.assertTokenStreamEqualComplete(
            [("Punctuation", "..."),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"...", self.lexer))

    def test_dots_four(self):
        self.assertTokenStreamEqualComplete(
            [("Punctuation", "...."),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"....", self.lexer))

    def test_nested_expr(self):
        self.assertTokenStreamEqualComplete(
            [("Name.Entity", "foo"),
             ("Text", " "),
             ("Operator", "+"),
             ("Text", " "),
             ("Name.Entity", "bar"),
             ("Text", " "),
             ("Operator", "*"),
             ("Text", " "),
             ("Number.Integer", '7'),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\EXPR{\EXPR{foo + bar} * 7}", self.lexer))

    def test_nested_expr_in_block(self):
        self.assertTokenStreamEqualComplete(
            [("Text", u"◆"),
             ("Text.Whitespace", " "),
             ("Name.Entity", "foo"),
             ("Text", " "),
             ("Operator", "+"),
             ("Text", " "),
             ("Name.Entity", "bar"),
             ("Text", " "),
             ("Operator", "*"),
             ("Text", " "),
             ("Number.Integer", '7'),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(u"\\BLOCK{\\EXPR{\\EXPR{foo + bar} * 7}}",
                         self.lexer))

    def test_nested_text(self):
        self.assertTokenStreamEqualComplete(
            [("Text", "foo + bar"),
             ("Text", " * 7"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(r"\TEXT{\TEXT{foo + bar} * 7}", self.lexer))

    def test_nested_text_in_tblock(self):
        self.assertTokenStreamEqualComplete(
            [("Text", u"▪"),
             ("Text.Whitespace", " "),
             ("Text", "bar + Foo"),
             ("Text", " * 7"),
             ("Text.Whitespace", "\n"),
             ],
            pygments.lex(u"\\TBLOCK{\\TEXT{\\TEXT{bar + Foo} * 7}}",
                         self.lexer))


class PygmentizeCompletely(unittest.TestCase):

    def setUp(self):
        self.lexer = pygments.lexers.load_lexer_from_file(
            ALGLEXERFILENAME, ALGLEXERCLASS)

    def test_pygmentize_html(self):
        html_formatter = pygments.formatters.get_formatter_by_name("html")
        highlighted = pygments.highlight(
            '"""FOO"""',
            self.lexer,
            html_formatter,
            outfile=None)
        self.assertTrue(highlighted.startswith("""<div class="highlight">"""))

    def test_pygmentize_latex(self):
        latex_formatter = pygments.formatters.get_formatter_by_name("latex")
        highlighted = pygments.highlight(
            '"""FOO"""',
            self.lexer,
            latex_formatter,
            outfile=None)
        self.assertTrue(highlighted.startswith(r"""\begin{Verbatim}"""))


if __name__ == "__main__":
    unittest.main()