Mercurial > hgrepos > Python > libs > pygments-lexer-pseudocode2
changeset 62:7153e945a3d6
Implement ignoring of \ENDxxx including its optional name parts
| author | Franz Glasner <fzglas.hg@dom66.de> |
|---|---|
| date | Tue, 28 Apr 2026 10:09:17 +0200 |
| parents | 392745b66969 |
| children | 414bf3cbb152 |
| files | pygments_lexer_pseudocode2/algpseudocode.py pygments_lexer_pseudocode2/bases.py tests/test_algpseudo.py |
| diffstat | 3 files changed, 41 insertions(+), 25 deletions(-) [+] |
line wrap: on
line diff
--- a/pygments_lexer_pseudocode2/algpseudocode.py Mon Apr 27 19:25:45 2026 +0200 +++ b/pygments_lexer_pseudocode2/algpseudocode.py Tue Apr 28 10:09:17 2026 +0200 @@ -121,6 +121,14 @@ return _op_end_translate + def op_opt_ignore(toktype): + + def _op_opt_ignore(lexer, match, ctx=None): + if not lexer.no_end: + yield match.start(), toktype, match.group() + + return _op_opt_ignore + def op_symbol(toktype): def _op_symbol(lexer, match, ctx=None): @@ -146,7 +154,7 @@ r"|(?:func(?:tion)?|(?:fn))" r"|(?:class)" r")(\s*)(\{)", - bygroups(op_translate(Keyword), Whitespace, Name.Entity), + bygroups(op_translate(Keyword), Whitespace, LexBase.op_ignore), "entity-name"), # ENDxxx keywords with optional entity name # with name @@ -157,8 +165,10 @@ r"|(?:func(?:tion)?)" r"|(?:class)" r")(\s*)(\{)", - bygroups(op_opt_end_translate(Keyword), Whitespace, Name.Entity), - "entity-name"), + bygroups(op_opt_end_translate(Keyword), + op_opt_ignore(Whitespace), + LexBase.op_ignore), + "entity-name-end"), # without name (r"(?i)\\end(?:_|-)?(" r"(?:prog(?:ram)?)" @@ -166,7 +176,7 @@ r"|(?:proc(?:edure)?)" r"|(?:func(?:tion)?)" r"|(?:class)" - r")((\s*)(\{))?", + r")\b", bygroups(op_opt_end_translate(Keyword))), # Keywords (r"(?i)\\(" @@ -176,11 +186,17 @@ include("expr"), (r"\s+", Text), ], - "entity-name": [ # may be multiline + "entity-name": [ # may be multiline (r"[^\\}]+", Name.Entity), (r"\\\}", Name.Entity), (r"\\", Name.Entity), - (r"\}", Name.Entity, "#pop"), + (r"\}", LexBase.op_ignore, "#pop"), + ], + "entity-name-end": [ # may be multiline -- suppressed if no_end + (r"[^\\}]+", op_opt_ignore(Name.Entity)), + (r"\\\}", op_opt_ignore(Name.Entity)), + (r"\\", op_opt_ignore(Name.Entity)), + (r"\}", LexBase.op_ignore, "#pop"), ], "expr": [ include("py-strings"),
--- a/pygments_lexer_pseudocode2/bases.py Mon Apr 27 19:25:45 2026 +0200 +++ b/pygments_lexer_pseudocode2/bases.py Tue Apr 28 10:09:17 2026 +0200 @@ -10,7 +10,7 @@ from pygments import unistring from pygments.lexer import RegexLexer, combined, bygroups, include -from pygments.token import Error, Name, Number, String, Comment +from pygments.token import Error, Name, Number, String, Comment, Other # @@ -68,6 +68,10 @@ """ + def op_ignore(lexer, match, ctx=None): + if False: + yield match.start(), Other, "" + tokens = { # # These states are borrowed from Pygment's Python lexer.
--- a/tests/test_algpseudo.py Mon Apr 27 19:25:45 2026 +0200 +++ b/tests/test_algpseudo.py Tue Apr 28 10:09:17 2026 +0200 @@ -145,9 +145,7 @@ self.assertTokenStreamEqualComplete( [("Keyword", "PROCEDURE"), ("Text.Whitespace", " "), - ("Name.Entity", "{"), ("Name.Entity", "the name"), - ("Name.Entity", "}"), ("Text.Whitespace", "\n"), ], pygments.lex("\\PROC {the name}", self.lexer)) @@ -163,9 +161,7 @@ self.assertTokenStreamEqualComplete( [("Keyword", "END OF PROCEDURE"), ("Text.Whitespace", " "), - ("Name.Entity", "{"), ("Name.Entity", "the procedure name"), - ("Name.Entity", "}"), ("Text.Whitespace", "\n"), ], pygments.lex("\\ENDPROCEDURE {the procedure name}", self.lexer)) @@ -176,11 +172,9 @@ self.assertTokenStreamEqualComplete( [("Keyword", "PROZEDUR"), ("Text.Whitespace", " "), - ("Name.Entity", "{"), ("Name.Entity", " also {nichtxs"), ("Name.Entity", "\\}"), ("Name.Entity", " hier"), - ("Name.Entity", "}"), ("Text.Whitespace", "\n"), ], pygments.lex("\\PROC { also {nichtxs\\} hier}", lexer)) @@ -188,9 +182,7 @@ def test_function_1(self): self.assertTokenStreamEqualComplete( [("Keyword", "FUNCTION"), - ("Name.Entity", "{"), ("Name.Entity", "1"), - ("Name.Entity", "}"), ("Text.Whitespace", "\n"), ], pygments.lex("\\FUNC{1}", self.lexer)) @@ -198,9 +190,7 @@ def test_function_2(self): self.assertTokenStreamEqualComplete( [("Keyword", "FUNCTION"), - ("Name.Entity", "{"), ("Name.Entity", "line 1\nline 2\n"), - ("Name.Entity", "}"), ("Text.Whitespace", "\n"), ], pygments.lex("\\FUNCTION{line 1\nline 2\n}", self.lexer)) @@ -209,11 +199,9 @@ self.assertTokenStreamEqualComplete( [("Keyword", "FUNCTION"), ("Text.Whitespace", " "), - ("Name.Entity", "{"), ("Name.Entity", "\\"), ("Name.Entity", "n"), ("Name.Entity", "\\}"), - ("Name.Entity", "}"), ("Text.Whitespace", "\n"), ], pygments.lex("\\FN {\\n\\}}", self.lexer)) @@ -222,11 +210,9 @@ self.assertTokenStreamEqualComplete( [("Keyword", "CLASS"), ("Text.Whitespace", " "), - ("Name.Entity", "{"), ("Name.Entity", "\\"), ("Name.Entity", "n"), ("Name.Entity", "\\}"), - ("Name.Entity", "}"), ("Text.Whitespace", "\n"), ], pygments.lex("\\CLASS {\\n\\}}", self.lexer)) @@ -237,11 +223,9 @@ self.assertTokenStreamEqualComplete( [("Keyword", "CLASSE"), ("Text.Whitespace", " "), - ("Name.Entity", "{"), ("Name.Entity", "\\"), ("Name.Entity", "n"), ("Name.Entity", "\\}"), - ("Name.Entity", "}"), ("Text.Whitespace", "\n"), ], pygments.lex("\\CLASS {\\n\\}}", lexer)) @@ -252,16 +236,28 @@ self.assertTokenStreamEqualComplete( [("Keyword", "KLASSE"), ("Text.Whitespace", " "), - ("Name.Entity", "{"), ("Name.Entity", "\\"), ("Name.Entity", "n"), ("Name.Entity", "\\}"), - ("Name.Entity", "}"), ("Text.Whitespace", "\n"), ("Text.Whitespace", "\n"), ], pygments.lex("\\CLASS {\\n\\}}\n\\ENDCLASS", lexer)) + def test_class_de_with_noend_option_and_name(self): + lexer = pygments.lexers.load_lexer_from_file( + ALGLEXERFILENAME, "AlgPseudocodeLexer_DE", no_end="True") + self.assertTokenStreamEqualComplete( + [("Keyword", "KLASSE"), + ("Text.Whitespace", " "), + ("Name.Entity", "\\"), + ("Name.Entity", "n"), + ("Name.Entity", "\\}"), + ("Text.Whitespace", "\n"), + ("Text.Whitespace", "\n"), + ], + pygments.lex("\\CLASS {\\n\\}}\n\\ENDCLASS {end class}", lexer)) + def test_remark_1(self): self.assertTokenStreamEqualComplete( [("Comment.Single", "▷"),