Mercurial > hgrepos > Python > libs > pygments-lexer-pseudocode2

comparison pygments_lexer_pseudocode2/lexers/bases.py @ 164:a4317957148b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Move all lexers into a subpackage pygments_lexer_pseudocode2.lexers. This is to prepare for a new subpackage with filters.
author Franz Glasner <fzglas.hg@dom66.de>
date Fri, 08 May 2026 21:19:54 +0200
parents pygments_lexer_pseudocode2/bases.py@e1663ac707b0
children afbca50b7dc1
comparison
equal deleted inserted replaced
163:315cfe0a836f 164:a4317957148b
1 # -*- coding: utf-8 -*-
2 # :-
3 # SPDX-FileCopyrightText: © 2026 Franz Glasner
4 # SPDX-License-Identifier: MIT
5 # :-
6 r"""Some common bases for the lexers."""
7
8 __all__ = ["LexBase", "uni_name", "py_innerstring_rules", "py_name_rules"]
9
10
11 import sys
12
13 from pygments import unistring
14 from pygments.lexer import RegexLexer, combined, bygroups, include
15 from pygments.token import (Comment, Error, Name, Number, Other, String)
16
17
18 PY2 = sys.version_info[0] <= 2
19
20
21 #
22 # SPDX-SnippetBegin
23 # SPDX-License-Identifier: BSD-2-Clause
24 # SPDX-SnippetCopyrightText: Copyright 2006-2023 by the Pygments team
25 # SPDX-SnippetCopyrightText: Copyright 2026 by Franz Glasner
26 #
27
28 uni_name = "[%s][%s]*" % (unistring.xid_start, unistring.xid_continue)
29
30
31 """PY3 allows no @staticmethod but PY2 needs it."""
32 if PY2:
33 _staticmethod = staticmethod
34 else:
35 def _staticmethod(fn):
36 return fn
37
38
39 def py_innerstring_rules(ttype):
40 return [
41 # the old style '%s' % (...) string formatting (still valid in Py3)
42 (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
43 '[hlL]?[E-GXc-giorsaux%]', String.Interpol),
44 # the new style '{}'.format(...) string formatting
45 (r'\{'
46 r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name
47 r'(\![sra])?' # conversion
48 r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?'
49 r'\}', String.Interpol),
50 #
51 # backslashes, quotes and formatting signs must be parsed
52 # one at a time
53 #
54 (r'[^\\\'"%{\n]+', ttype),
55 (r'[\'"\\]', ttype),
56 # unhandled string formatting sign
57 (r'%|(\{{1,2})', ttype)
58 # newlines are an error (use "nl" state)
59 ]
60
61
62 def py_name_rules(ttype, deco_ttype=Name.Decorator):
63 return [
64 # We recognize decorator syntax here
65 (r'@' + uni_name, deco_ttype),
66 #
67 # Python's new matrix multiplication operator:
68 # not used here in pseudocode
69 # (r'@', Operator),
70 (uni_name, ttype),
71 ]
72
73 # SPDX-SnippetEnd
74
75
76 class LexBase(RegexLexer):
77
78 """A base that defines some common lexer states.
79
80 Default flags are not important.
81
82 """
83
84 def op_ignore(lexer, match, ctx=None):
85 """Unconditionally ignore the match."""
86 if False:
87 yield match.start(), Other, ""
88 if ctx:
89 ctx.pos = match.end()
90
91 @_staticmethod
92 def op_fixed(toktype, value):
93 """Unconditionally yield a given token type and value."""
94
95 def _op_fixed(lexer, match, ctx=None):
96 yield match.start(), toktype, value
97 if ctx:
98 ctx.pos = match.end()
99
100 return _op_fixed
101
102 tokens = {
103 #
104 # These states are borrowed from Pygment's Python lexer.
105 # Their names have been prefixed with `py-'.
106 #
107 # SPDX-SnippetBegin
108 # SPDX-License-Identifier: BSD-2-Clause
109 # SPDX-SnippetCopyrightText: Copyright 2006-2023 by the Pygments team
110 # SPDX-SnippetCopyrightText: Copyright 2026 by Franz Glasner
111 #
112 'py-numbers': [
113 (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)'
114 r'([eE][+-]?\d(?:_?\d)*)?', Number.Float),
115 (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float),
116 (r'0[oO](?:_?[0-7])+', Number.Oct),
117 (r'0[bB](?:_?[01])+', Number.Bin),
118 (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex),
119 (r'\d(?:_?\d)*', Number.Integer),
120 ],
121 'py-strings': [
122 # non-raw strings
123 ('([uU]?)(""")', bygroups(String.Affix, String.Double),
124 combined('py-stringescape', 'py-tdqs')),
125 ("([uU]?)(''')", bygroups(String.Affix, String.Single),
126 combined('py-stringescape', 'py-tsqs')),
127 ('([uU]?)(")', bygroups(String.Affix, String.Double),
128 combined('py-stringescape', 'py-dqs')),
129 ("([uU]?)(')", bygroups(String.Affix, String.Single),
130 combined('py-stringescape', 'py-sqs')),
131 # non-raw bytes
132 ('([bB])(""")', bygroups(String.Affix, String.Double),
133 combined('py-bytesescape', 'py-tdqs')),
134 ("([bB])(''')", bygroups(String.Affix, String.Single),
135 combined('py-bytesescape', 'py-tsqs')),
136 ('([bB])(")', bygroups(String.Affix, String.Double),
137 combined('py-bytesescape', 'py-dqs')),
138 ("([bB])(')", bygroups(String.Affix, String.Single),
139 combined('py-bytesescape', 'py-sqs')),
140 ],
141 'py-stringescape': [
142 (r'\\(N\{.*?\}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8})', String.Escape),
143 include('py-bytesescape')
144 ],
145 'py-bytesescape': [
146 (r'\\([\\abfnrtv"\']|\n|x[a-fA-F0-9]{2}|[0-7]{1,3})',
147 String.Escape)
148 ],
149 'py-dqs': [
150 (r'"', String.Double, '#pop'),
151 (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings
152 include('py-strings-double'),
153 (r'\n', Error), # added by fag
154 ],
155 'py-sqs': [
156 (r"'", String.Single, '#pop'),
157 (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings
158 include('py-strings-single'),
159 (r'\n', Error), # added by fag
160 ],
161 'py-tdqs': [
162 (r'"""', String.Double, '#pop'),
163 include('py-strings-double'),
164 (r'\n', String.Double)
165 ],
166 'py-tsqs': [
167 (r"'''", String.Single, '#pop'),
168 include('py-strings-single'),
169 (r'\n', String.Single)
170 ],
171 'py-strings-single': py_innerstring_rules(String.Single),
172 'py-strings-double': py_innerstring_rules(String.Double),
173 'py-name': py_name_rules(Name.Entity),
174 # SPDX-SnippetEnd
175 # This snippet is from the Pygments' documentation "Write your own lexer"
176 'multiline-nested-comment': [
177 (r'[^*/]+', Comment.Multiline),
178 (r'/\*', Comment.Multiline, '#push'),
179 (r'\*/', Comment.Multiline, '#pop'),
180 (r'[*/]', Comment.Multiline),
181 ],
182 'multiline-nested-comment-alt': [
183 (r'[^*()]+', Comment.Multiline),
184 (r'\(\*', Comment.Multiline, '#push'),
185 (r'\*\)', Comment.Multiline, '#pop'),
186 (r'[*()]', Comment.Multiline),
187 ]
188 }