python.py 53 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204
  1. """
  2. pygments.lexers.python
  3. ~~~~~~~~~~~~~~~~~~~~~~
  4. Lexers for Python and related languages.
  5. :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
  6. :license: BSD, see LICENSE for details.
  7. """
  8. import keyword
  9. from pygments.lexer import DelegatingLexer, RegexLexer, include, \
  10. bygroups, using, default, words, combined, this
  11. from pygments.util import get_bool_opt, shebang_matches
  12. from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
  13. Number, Punctuation, Generic, Other, Error, Whitespace
  14. from pygments import unistring as uni
  15. __all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer',
  16. 'Python2Lexer', 'Python2TracebackLexer',
  17. 'CythonLexer', 'DgLexer', 'NumPyLexer']
  18. class PythonLexer(RegexLexer):
  19. """
  20. For Python source code (version 3.x).
  21. .. versionchanged:: 2.5
  22. This is now the default ``PythonLexer``. It is still available as the
  23. alias ``Python3Lexer``.
  24. """
  25. name = 'Python'
  26. url = 'https://www.python.org'
  27. aliases = ['python', 'py', 'sage', 'python3', 'py3', 'bazel', 'starlark', 'pyi']
  28. filenames = [
  29. '*.py',
  30. '*.pyw',
  31. # Type stubs
  32. '*.pyi',
  33. # Jython
  34. '*.jy',
  35. # Sage
  36. '*.sage',
  37. # SCons
  38. '*.sc',
  39. 'SConstruct',
  40. 'SConscript',
  41. # Skylark/Starlark (used by Bazel, Buck, and Pants)
  42. '*.bzl',
  43. 'BUCK',
  44. 'BUILD',
  45. 'BUILD.bazel',
  46. 'WORKSPACE',
  47. # Twisted Application infrastructure
  48. '*.tac',
  49. # Execubot level format
  50. '*.pye',
  51. ]
  52. mimetypes = ['text/x-python', 'application/x-python',
  53. 'text/x-python3', 'application/x-python3']
  54. version_added = '0.10'
  55. uni_name = f"[{uni.xid_start}][{uni.xid_continue}]*"
  56. def innerstring_rules(ttype):
  57. return [
  58. # the old style '%s' % (...) string formatting (still valid in Py3)
  59. (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
  60. '[hlL]?[E-GXc-giorsaux%]', String.Interpol),
  61. # the new style '{}'.format(...) string formatting
  62. (r'\{'
  63. r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name
  64. r'(\![sra])?' # conversion
  65. r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?'
  66. r'\}', String.Interpol),
  67. # backslashes, quotes and formatting signs must be parsed one at a time
  68. (r'[^\\\'"%{\n]+', ttype),
  69. (r'[\'"\\]', ttype),
  70. # unhandled string formatting sign
  71. (r'%|(\{{1,2})', ttype)
  72. # newlines are an error (use "nl" state)
  73. ]
  74. def fstring_rules(ttype):
  75. return [
  76. # Assuming that a '}' is the closing brace after format specifier.
  77. # Sadly, this means that we won't detect syntax error. But it's
  78. # more important to parse correct syntax correctly, than to
  79. # highlight invalid syntax.
  80. (r'\}', String.Interpol),
  81. (r'\{', String.Interpol, 'expr-inside-fstring'),
  82. # backslashes, quotes and formatting signs must be parsed one at a time
  83. (r'[^\\\'"{}\n]+', ttype),
  84. (r'[\'"\\]', ttype),
  85. # newlines are an error (use "nl" state)
  86. ]
  87. tokens = {
  88. 'root': [
  89. (r'\n', Whitespace),
  90. (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")',
  91. bygroups(Whitespace, String.Affix, String.Doc)),
  92. (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')",
  93. bygroups(Whitespace, String.Affix, String.Doc)),
  94. (r'\A#!.+$', Comment.Hashbang),
  95. (r'#.*$', Comment.Single),
  96. (r'\\\n', Text),
  97. (r'\\', Text),
  98. include('keywords'),
  99. include('soft-keywords'),
  100. (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Whitespace), 'funcname'),
  101. (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Whitespace), 'classname'),
  102. (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Whitespace),
  103. 'fromimport'),
  104. (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Whitespace),
  105. 'import'),
  106. include('expr'),
  107. ],
  108. 'expr': [
  109. # raw f-strings and t-strings
  110. ('(?i)(r[ft]|[ft]r)(""")',
  111. bygroups(String.Affix, String.Double),
  112. combined('rfstringescape', 'tdqf')),
  113. ("(?i)(r[ft]|[ft]r)(''')",
  114. bygroups(String.Affix, String.Single),
  115. combined('rfstringescape', 'tsqf')),
  116. ('(?i)(r[ft]|[ft]r)(")',
  117. bygroups(String.Affix, String.Double),
  118. combined('rfstringescape', 'dqf')),
  119. ("(?i)(r[ft]|[ft]r)(')",
  120. bygroups(String.Affix, String.Single),
  121. combined('rfstringescape', 'sqf')),
  122. # non-raw f-strings and t-strings
  123. ('([fFtT])(""")', bygroups(String.Affix, String.Double),
  124. combined('fstringescape', 'tdqf')),
  125. ("([fFtT])(''')", bygroups(String.Affix, String.Single),
  126. combined('fstringescape', 'tsqf')),
  127. ('([fFtT])(")', bygroups(String.Affix, String.Double),
  128. combined('fstringescape', 'dqf')),
  129. ("([fFtT])(')", bygroups(String.Affix, String.Single),
  130. combined('fstringescape', 'sqf')),
  131. # raw bytes and strings
  132. ('(?i)(rb|br|r)(""")',
  133. bygroups(String.Affix, String.Double), 'tdqs'),
  134. ("(?i)(rb|br|r)(''')",
  135. bygroups(String.Affix, String.Single), 'tsqs'),
  136. ('(?i)(rb|br|r)(")',
  137. bygroups(String.Affix, String.Double), 'dqs'),
  138. ("(?i)(rb|br|r)(')",
  139. bygroups(String.Affix, String.Single), 'sqs'),
  140. # non-raw strings
  141. ('([uU]?)(""")', bygroups(String.Affix, String.Double),
  142. combined('stringescape', 'tdqs')),
  143. ("([uU]?)(''')", bygroups(String.Affix, String.Single),
  144. combined('stringescape', 'tsqs')),
  145. ('([uU]?)(")', bygroups(String.Affix, String.Double),
  146. combined('stringescape', 'dqs')),
  147. ("([uU]?)(')", bygroups(String.Affix, String.Single),
  148. combined('stringescape', 'sqs')),
  149. # non-raw bytes
  150. ('([bB])(""")', bygroups(String.Affix, String.Double),
  151. combined('bytesescape', 'tdqs')),
  152. ("([bB])(''')", bygroups(String.Affix, String.Single),
  153. combined('bytesescape', 'tsqs')),
  154. ('([bB])(")', bygroups(String.Affix, String.Double),
  155. combined('bytesescape', 'dqs')),
  156. ("([bB])(')", bygroups(String.Affix, String.Single),
  157. combined('bytesescape', 'sqs')),
  158. (r'[^\S\n]+', Text),
  159. include('numbers'),
  160. (r'!=|==|<<|>>|:=|[-~+/*%=<>&^|.]', Operator),
  161. (r'[]{}:(),;[]', Punctuation),
  162. (r'(in|is|and|or|not)\b', Operator.Word),
  163. include('expr-keywords'),
  164. include('builtins'),
  165. include('magicfuncs'),
  166. include('magicvars'),
  167. include('name'),
  168. ],
  169. 'expr-inside-fstring': [
  170. (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
  171. # without format specifier
  172. (r'(=\s*)?' # debug (https://bugs.python.org/issue36817)
  173. r'(\![sraf])?' # conversion
  174. r'\}', String.Interpol, '#pop'),
  175. # with format specifier
  176. # we'll catch the remaining '}' in the outer scope
  177. (r'(=\s*)?' # debug (https://bugs.python.org/issue36817)
  178. r'(\![sraf])?' # conversion
  179. r':', String.Interpol, '#pop'),
  180. (r'\s+', Whitespace), # allow new lines
  181. include('expr'),
  182. ],
  183. 'expr-inside-fstring-inner': [
  184. (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
  185. (r'[])}]', Punctuation, '#pop'),
  186. (r'\s+', Whitespace), # allow new lines
  187. include('expr'),
  188. ],
  189. 'expr-keywords': [
  190. # Based on https://docs.python.org/3/reference/expressions.html
  191. (words((
  192. 'async for', 'await', 'else', 'for', 'if', 'lambda',
  193. 'yield', 'yield from'), suffix=r'\b'),
  194. Keyword),
  195. (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant),
  196. ],
  197. 'keywords': [
  198. (words((
  199. 'assert', 'async', 'await', 'break', 'continue', 'del', 'elif',
  200. 'else', 'except', 'finally', 'for', 'global', 'if', 'lambda',
  201. 'pass', 'raise', 'nonlocal', 'return', 'try', 'while', 'yield',
  202. 'yield from', 'as', 'with'), suffix=r'\b'),
  203. Keyword),
  204. (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant),
  205. ],
  206. 'soft-keywords': [
  207. # `match`, `case` and `_` soft keywords
  208. (r'(^[ \t]*)' # at beginning of line + possible indentation
  209. r'(match|case)\b' # a possible keyword
  210. r'(?![ \t]*(?:' # not followed by...
  211. r'[:,;=^&|@~)\]}]|(?:' + # characters and keywords that mean this isn't
  212. # pattern matching (but None/True/False is ok)
  213. r'|'.join(k for k in keyword.kwlist if k[0].islower()) + r')\b))',
  214. bygroups(Text, Keyword), 'soft-keywords-inner'),
  215. ],
  216. 'soft-keywords-inner': [
  217. # optional `_` keyword
  218. (r'(\s+)([^\n_]*)(_\b)', bygroups(Whitespace, using(this), Keyword)),
  219. default('#pop')
  220. ],
  221. 'builtins': [
  222. (words((
  223. '__import__', 'abs', 'aiter', 'all', 'any', 'bin', 'bool', 'bytearray',
  224. 'breakpoint', 'bytes', 'callable', 'chr', 'classmethod', 'compile',
  225. 'complex', 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval',
  226. 'filter', 'float', 'format', 'frozenset', 'getattr', 'globals',
  227. 'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'isinstance',
  228. 'issubclass', 'iter', 'len', 'list', 'locals', 'map', 'max',
  229. 'memoryview', 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow',
  230. 'print', 'property', 'range', 'repr', 'reversed', 'round', 'set',
  231. 'setattr', 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super',
  232. 'tuple', 'type', 'vars', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'),
  233. Name.Builtin),
  234. (r'(?<!\.)(self|Ellipsis|NotImplemented|cls)\b', Name.Builtin.Pseudo),
  235. (words((
  236. 'ArithmeticError', 'AssertionError', 'AttributeError',
  237. 'BaseException', 'BufferError', 'BytesWarning', 'DeprecationWarning',
  238. 'EOFError', 'EnvironmentError', 'Exception', 'FloatingPointError',
  239. 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError',
  240. 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError',
  241. 'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError',
  242. 'NotImplementedError', 'OSError', 'OverflowError',
  243. 'PendingDeprecationWarning', 'ReferenceError', 'ResourceWarning',
  244. 'RuntimeError', 'RuntimeWarning', 'StopIteration',
  245. 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit',
  246. 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
  247. 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
  248. 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError',
  249. 'Warning', 'WindowsError', 'ZeroDivisionError',
  250. # new builtin exceptions from PEP 3151
  251. 'BlockingIOError', 'ChildProcessError', 'ConnectionError',
  252. 'BrokenPipeError', 'ConnectionAbortedError', 'ConnectionRefusedError',
  253. 'ConnectionResetError', 'FileExistsError', 'FileNotFoundError',
  254. 'InterruptedError', 'IsADirectoryError', 'NotADirectoryError',
  255. 'PermissionError', 'ProcessLookupError', 'TimeoutError',
  256. # others new in Python 3
  257. 'StopAsyncIteration', 'ModuleNotFoundError', 'RecursionError',
  258. 'EncodingWarning'),
  259. prefix=r'(?<!\.)', suffix=r'\b'),
  260. Name.Exception),
  261. ],
  262. 'magicfuncs': [
  263. (words((
  264. '__abs__', '__add__', '__aenter__', '__aexit__', '__aiter__',
  265. '__and__', '__anext__', '__await__', '__bool__', '__bytes__',
  266. '__call__', '__complex__', '__contains__', '__del__', '__delattr__',
  267. '__delete__', '__delitem__', '__dir__', '__divmod__', '__enter__',
  268. '__eq__', '__exit__', '__float__', '__floordiv__', '__format__',
  269. '__ge__', '__get__', '__getattr__', '__getattribute__',
  270. '__getitem__', '__gt__', '__hash__', '__iadd__', '__iand__',
  271. '__ifloordiv__', '__ilshift__', '__imatmul__', '__imod__',
  272. '__imul__', '__index__', '__init__', '__instancecheck__',
  273. '__int__', '__invert__', '__ior__', '__ipow__', '__irshift__',
  274. '__isub__', '__iter__', '__itruediv__', '__ixor__', '__le__',
  275. '__len__', '__length_hint__', '__lshift__', '__lt__', '__matmul__',
  276. '__missing__', '__mod__', '__mul__', '__ne__', '__neg__',
  277. '__new__', '__next__', '__or__', '__pos__', '__pow__',
  278. '__prepare__', '__radd__', '__rand__', '__rdivmod__', '__repr__',
  279. '__reversed__', '__rfloordiv__', '__rlshift__', '__rmatmul__',
  280. '__rmod__', '__rmul__', '__ror__', '__round__', '__rpow__',
  281. '__rrshift__', '__rshift__', '__rsub__', '__rtruediv__',
  282. '__rxor__', '__set__', '__setattr__', '__setitem__', '__str__',
  283. '__sub__', '__subclasscheck__', '__truediv__',
  284. '__xor__'), suffix=r'\b'),
  285. Name.Function.Magic),
  286. ],
  287. 'magicvars': [
  288. (words((
  289. '__annotations__', '__bases__', '__class__', '__closure__',
  290. '__code__', '__defaults__', '__dict__', '__doc__', '__file__',
  291. '__func__', '__globals__', '__kwdefaults__', '__module__',
  292. '__mro__', '__name__', '__objclass__', '__qualname__',
  293. '__self__', '__slots__', '__weakref__'), suffix=r'\b'),
  294. Name.Variable.Magic),
  295. ],
  296. 'numbers': [
  297. (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)'
  298. r'([eE][+-]?\d(?:_?\d)*)?', Number.Float),
  299. (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float),
  300. (r'0[oO](?:_?[0-7])+', Number.Oct),
  301. (r'0[bB](?:_?[01])+', Number.Bin),
  302. (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex),
  303. (r'\d(?:_?\d)*', Number.Integer),
  304. ],
  305. 'name': [
  306. (r'@' + uni_name, Name.Decorator),
  307. (r'@', Operator), # new matrix multiplication operator
  308. (uni_name, Name),
  309. ],
  310. 'funcname': [
  311. include('magicfuncs'),
  312. (uni_name, Name.Function, '#pop'),
  313. default('#pop'),
  314. ],
  315. 'classname': [
  316. (uni_name, Name.Class, '#pop'),
  317. ],
  318. 'import': [
  319. (r'(\s+)(as)(\s+)', bygroups(Whitespace, Keyword, Whitespace)),
  320. (r'\.', Name.Namespace),
  321. (uni_name, Name.Namespace),
  322. (r'(\s*)(,)(\s*)', bygroups(Whitespace, Operator, Whitespace)),
  323. default('#pop') # all else: go back
  324. ],
  325. 'fromimport': [
  326. (r'(\s+)(import)\b', bygroups(Whitespace, Keyword.Namespace), '#pop'),
  327. (r'\.', Name.Namespace),
  328. # if None occurs here, it's "raise x from None", since None can
  329. # never be a module name
  330. (r'None\b', Keyword.Constant, '#pop'),
  331. (uni_name, Name.Namespace),
  332. default('#pop'),
  333. ],
  334. 'rfstringescape': [
  335. (r'\{\{', String.Escape),
  336. (r'\}\}', String.Escape),
  337. ],
  338. 'fstringescape': [
  339. include('rfstringescape'),
  340. include('stringescape'),
  341. ],
  342. 'bytesescape': [
  343. (r'\\([\\abfnrtv"\']|\n|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
  344. ],
  345. 'stringescape': [
  346. (r'\\(N\{.*?\}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8})', String.Escape),
  347. include('bytesescape')
  348. ],
  349. 'fstrings-single': fstring_rules(String.Single),
  350. 'fstrings-double': fstring_rules(String.Double),
  351. 'strings-single': innerstring_rules(String.Single),
  352. 'strings-double': innerstring_rules(String.Double),
  353. 'dqf': [
  354. (r'"', String.Double, '#pop'),
  355. (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings
  356. include('fstrings-double')
  357. ],
  358. 'sqf': [
  359. (r"'", String.Single, '#pop'),
  360. (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings
  361. include('fstrings-single')
  362. ],
  363. 'dqs': [
  364. (r'"', String.Double, '#pop'),
  365. (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings
  366. include('strings-double')
  367. ],
  368. 'sqs': [
  369. (r"'", String.Single, '#pop'),
  370. (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings
  371. include('strings-single')
  372. ],
  373. 'tdqf': [
  374. (r'"""', String.Double, '#pop'),
  375. include('fstrings-double'),
  376. (r'\n', String.Double)
  377. ],
  378. 'tsqf': [
  379. (r"'''", String.Single, '#pop'),
  380. include('fstrings-single'),
  381. (r'\n', String.Single)
  382. ],
  383. 'tdqs': [
  384. (r'"""', String.Double, '#pop'),
  385. include('strings-double'),
  386. (r'\n', String.Double)
  387. ],
  388. 'tsqs': [
  389. (r"'''", String.Single, '#pop'),
  390. include('strings-single'),
  391. (r'\n', String.Single)
  392. ],
  393. }
  394. def analyse_text(text):
  395. return shebang_matches(text, r'pythonw?(3(\.\d)?)?') or \
  396. 'import ' in text[:1000]
  397. Python3Lexer = PythonLexer
  398. class Python2Lexer(RegexLexer):
  399. """
  400. For Python 2.x source code.
  401. .. versionchanged:: 2.5
  402. This class has been renamed from ``PythonLexer``. ``PythonLexer`` now
  403. refers to the Python 3 variant. File name patterns like ``*.py`` have
  404. been moved to Python 3 as well.
  405. """
  406. name = 'Python 2.x'
  407. url = 'https://www.python.org'
  408. aliases = ['python2', 'py2']
  409. filenames = [] # now taken over by PythonLexer (3.x)
  410. mimetypes = ['text/x-python2', 'application/x-python2']
  411. version_added = ''
  412. def innerstring_rules(ttype):
  413. return [
  414. # the old style '%s' % (...) string formatting
  415. (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
  416. '[hlL]?[E-GXc-giorsux%]', String.Interpol),
  417. # backslashes, quotes and formatting signs must be parsed one at a time
  418. (r'[^\\\'"%\n]+', ttype),
  419. (r'[\'"\\]', ttype),
  420. # unhandled string formatting sign
  421. (r'%', ttype),
  422. # newlines are an error (use "nl" state)
  423. ]
  424. tokens = {
  425. 'root': [
  426. (r'\n', Whitespace),
  427. (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")',
  428. bygroups(Whitespace, String.Affix, String.Doc)),
  429. (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')",
  430. bygroups(Whitespace, String.Affix, String.Doc)),
  431. (r'[^\S\n]+', Text),
  432. (r'\A#!.+$', Comment.Hashbang),
  433. (r'#.*$', Comment.Single),
  434. (r'[]{}:(),;[]', Punctuation),
  435. (r'\\\n', Text),
  436. (r'\\', Text),
  437. (r'(in|is|and|or|not)\b', Operator.Word),
  438. (r'!=|==|<<|>>|[-~+/*%=<>&^|.]', Operator),
  439. include('keywords'),
  440. (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Whitespace), 'funcname'),
  441. (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Whitespace), 'classname'),
  442. (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Whitespace),
  443. 'fromimport'),
  444. (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Whitespace),
  445. 'import'),
  446. include('builtins'),
  447. include('magicfuncs'),
  448. include('magicvars'),
  449. include('backtick'),
  450. ('([rR]|[uUbB][rR]|[rR][uUbB])(""")',
  451. bygroups(String.Affix, String.Double), 'tdqs'),
  452. ("([rR]|[uUbB][rR]|[rR][uUbB])(''')",
  453. bygroups(String.Affix, String.Single), 'tsqs'),
  454. ('([rR]|[uUbB][rR]|[rR][uUbB])(")',
  455. bygroups(String.Affix, String.Double), 'dqs'),
  456. ("([rR]|[uUbB][rR]|[rR][uUbB])(')",
  457. bygroups(String.Affix, String.Single), 'sqs'),
  458. ('([uUbB]?)(""")', bygroups(String.Affix, String.Double),
  459. combined('stringescape', 'tdqs')),
  460. ("([uUbB]?)(''')", bygroups(String.Affix, String.Single),
  461. combined('stringescape', 'tsqs')),
  462. ('([uUbB]?)(")', bygroups(String.Affix, String.Double),
  463. combined('stringescape', 'dqs')),
  464. ("([uUbB]?)(')", bygroups(String.Affix, String.Single),
  465. combined('stringescape', 'sqs')),
  466. include('name'),
  467. include('numbers'),
  468. ],
  469. 'keywords': [
  470. (words((
  471. 'assert', 'break', 'continue', 'del', 'elif', 'else', 'except',
  472. 'exec', 'finally', 'for', 'global', 'if', 'lambda', 'pass',
  473. 'print', 'raise', 'return', 'try', 'while', 'yield',
  474. 'yield from', 'as', 'with'), suffix=r'\b'),
  475. Keyword),
  476. ],
  477. 'builtins': [
  478. (words((
  479. '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin',
  480. 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr', 'classmethod',
  481. 'cmp', 'coerce', 'compile', 'complex', 'delattr', 'dict', 'dir', 'divmod',
  482. 'enumerate', 'eval', 'execfile', 'exit', 'file', 'filter', 'float',
  483. 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'hex', 'id',
  484. 'input', 'int', 'intern', 'isinstance', 'issubclass', 'iter', 'len',
  485. 'list', 'locals', 'long', 'map', 'max', 'min', 'next', 'object',
  486. 'oct', 'open', 'ord', 'pow', 'property', 'range', 'raw_input', 'reduce',
  487. 'reload', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice',
  488. 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type',
  489. 'unichr', 'unicode', 'vars', 'xrange', 'zip'),
  490. prefix=r'(?<!\.)', suffix=r'\b'),
  491. Name.Builtin),
  492. (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|cls'
  493. r')\b', Name.Builtin.Pseudo),
  494. (words((
  495. 'ArithmeticError', 'AssertionError', 'AttributeError',
  496. 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError',
  497. 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit',
  498. 'IOError', 'ImportError', 'ImportWarning', 'IndentationError',
  499. 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError',
  500. 'MemoryError', 'NameError',
  501. 'NotImplementedError', 'OSError', 'OverflowError', 'OverflowWarning',
  502. 'PendingDeprecationWarning', 'ReferenceError',
  503. 'RuntimeError', 'RuntimeWarning', 'StandardError', 'StopIteration',
  504. 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit',
  505. 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
  506. 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
  507. 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError', 'Warning',
  508. 'WindowsError', 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'),
  509. Name.Exception),
  510. ],
  511. 'magicfuncs': [
  512. (words((
  513. '__abs__', '__add__', '__and__', '__call__', '__cmp__', '__coerce__',
  514. '__complex__', '__contains__', '__del__', '__delattr__', '__delete__',
  515. '__delitem__', '__delslice__', '__div__', '__divmod__', '__enter__',
  516. '__eq__', '__exit__', '__float__', '__floordiv__', '__ge__', '__get__',
  517. '__getattr__', '__getattribute__', '__getitem__', '__getslice__', '__gt__',
  518. '__hash__', '__hex__', '__iadd__', '__iand__', '__idiv__', '__ifloordiv__',
  519. '__ilshift__', '__imod__', '__imul__', '__index__', '__init__',
  520. '__instancecheck__', '__int__', '__invert__', '__iop__', '__ior__',
  521. '__ipow__', '__irshift__', '__isub__', '__iter__', '__itruediv__',
  522. '__ixor__', '__le__', '__len__', '__long__', '__lshift__', '__lt__',
  523. '__missing__', '__mod__', '__mul__', '__ne__', '__neg__', '__new__',
  524. '__nonzero__', '__oct__', '__op__', '__or__', '__pos__', '__pow__',
  525. '__radd__', '__rand__', '__rcmp__', '__rdiv__', '__rdivmod__', '__repr__',
  526. '__reversed__', '__rfloordiv__', '__rlshift__', '__rmod__', '__rmul__',
  527. '__rop__', '__ror__', '__rpow__', '__rrshift__', '__rshift__', '__rsub__',
  528. '__rtruediv__', '__rxor__', '__set__', '__setattr__', '__setitem__',
  529. '__setslice__', '__str__', '__sub__', '__subclasscheck__', '__truediv__',
  530. '__unicode__', '__xor__'), suffix=r'\b'),
  531. Name.Function.Magic),
  532. ],
  533. 'magicvars': [
  534. (words((
  535. '__bases__', '__class__', '__closure__', '__code__', '__defaults__',
  536. '__dict__', '__doc__', '__file__', '__func__', '__globals__',
  537. '__metaclass__', '__module__', '__mro__', '__name__', '__self__',
  538. '__slots__', '__weakref__'),
  539. suffix=r'\b'),
  540. Name.Variable.Magic),
  541. ],
  542. 'numbers': [
  543. (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
  544. (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
  545. (r'0[0-7]+j?', Number.Oct),
  546. (r'0[bB][01]+', Number.Bin),
  547. (r'0[xX][a-fA-F0-9]+', Number.Hex),
  548. (r'\d+L', Number.Integer.Long),
  549. (r'\d+j?', Number.Integer)
  550. ],
  551. 'backtick': [
  552. ('`.*?`', String.Backtick),
  553. ],
  554. 'name': [
  555. (r'@[\w.]+', Name.Decorator),
  556. (r'[a-zA-Z_]\w*', Name),
  557. ],
  558. 'funcname': [
  559. include('magicfuncs'),
  560. (r'[a-zA-Z_]\w*', Name.Function, '#pop'),
  561. default('#pop'),
  562. ],
  563. 'classname': [
  564. (r'[a-zA-Z_]\w*', Name.Class, '#pop')
  565. ],
  566. 'import': [
  567. (r'(?:[ \t]|\\\n)+', Text),
  568. (r'as\b', Keyword.Namespace),
  569. (r',', Operator),
  570. (r'[a-zA-Z_][\w.]*', Name.Namespace),
  571. default('#pop') # all else: go back
  572. ],
  573. 'fromimport': [
  574. (r'(?:[ \t]|\\\n)+', Text),
  575. (r'import\b', Keyword.Namespace, '#pop'),
  576. # if None occurs here, it's "raise x from None", since None can
  577. # never be a module name
  578. (r'None\b', Name.Builtin.Pseudo, '#pop'),
  579. # sadly, in "raise x from y" y will be highlighted as namespace too
  580. (r'[a-zA-Z_.][\w.]*', Name.Namespace),
  581. # anything else here also means "raise x from y" and is therefore
  582. # not an error
  583. default('#pop'),
  584. ],
  585. 'stringescape': [
  586. (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
  587. r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
  588. ],
  589. 'strings-single': innerstring_rules(String.Single),
  590. 'strings-double': innerstring_rules(String.Double),
  591. 'dqs': [
  592. (r'"', String.Double, '#pop'),
  593. (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings
  594. include('strings-double')
  595. ],
  596. 'sqs': [
  597. (r"'", String.Single, '#pop'),
  598. (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings
  599. include('strings-single')
  600. ],
  601. 'tdqs': [
  602. (r'"""', String.Double, '#pop'),
  603. include('strings-double'),
  604. (r'\n', String.Double)
  605. ],
  606. 'tsqs': [
  607. (r"'''", String.Single, '#pop'),
  608. include('strings-single'),
  609. (r'\n', String.Single)
  610. ],
  611. }
  612. def analyse_text(text):
  613. return shebang_matches(text, r'pythonw?2(\.\d)?')
  614. class _PythonConsoleLexerBase(RegexLexer):
  615. name = 'Python console session'
  616. aliases = ['pycon', 'python-console']
  617. mimetypes = ['text/x-python-doctest']
  618. """Auxiliary lexer for `PythonConsoleLexer`.
  619. Code tokens are output as ``Token.Other.Code``, traceback tokens as
  620. ``Token.Other.Traceback``.
  621. """
  622. tokens = {
  623. 'root': [
  624. (r'(>>> )(.*\n)', bygroups(Generic.Prompt, Other.Code), 'continuations'),
  625. # This happens, e.g., when tracebacks are embedded in documentation;
  626. # trailing whitespaces are often stripped in such contexts.
  627. (r'(>>>)(\n)', bygroups(Generic.Prompt, Whitespace)),
  628. (r'(\^C)?Traceback \(most recent call last\):\n', Other.Traceback, 'traceback'),
  629. # SyntaxError starts with this
  630. (r' File "[^"]+", line \d+', Other.Traceback, 'traceback'),
  631. (r'.*\n', Generic.Output),
  632. ],
  633. 'continuations': [
  634. (r'(\.\.\. )(.*\n)', bygroups(Generic.Prompt, Other.Code)),
  635. # See above.
  636. (r'(\.\.\.)(\n)', bygroups(Generic.Prompt, Whitespace)),
  637. default('#pop'),
  638. ],
  639. 'traceback': [
  640. # As soon as we see a traceback, consume everything until the next
  641. # >>> prompt.
  642. (r'(?=>>>( |$))', Text, '#pop'),
  643. (r'(KeyboardInterrupt)(\n)', bygroups(Name.Class, Whitespace)),
  644. (r'.*\n', Other.Traceback),
  645. ],
  646. }
  647. class PythonConsoleLexer(DelegatingLexer):
  648. """
  649. For Python console output or doctests, such as:
  650. .. sourcecode:: pycon
  651. >>> a = 'foo'
  652. >>> print(a)
  653. foo
  654. >>> 1 / 0
  655. Traceback (most recent call last):
  656. File "<stdin>", line 1, in <module>
  657. ZeroDivisionError: integer division or modulo by zero
  658. Additional options:
  659. `python3`
  660. Use Python 3 lexer for code. Default is ``True``.
  661. .. versionadded:: 1.0
  662. .. versionchanged:: 2.5
  663. Now defaults to ``True``.
  664. """
  665. name = 'Python console session'
  666. aliases = ['pycon', 'python-console']
  667. mimetypes = ['text/x-python-doctest']
  668. url = 'https://python.org'
  669. version_added = ''
  670. def __init__(self, **options):
  671. python3 = get_bool_opt(options, 'python3', True)
  672. if python3:
  673. pylexer = PythonLexer
  674. tblexer = PythonTracebackLexer
  675. else:
  676. pylexer = Python2Lexer
  677. tblexer = Python2TracebackLexer
  678. # We have two auxiliary lexers. Use DelegatingLexer twice with
  679. # different tokens. TODO: DelegatingLexer should support this
  680. # directly, by accepting a tuplet of auxiliary lexers and a tuple of
  681. # distinguishing tokens. Then we wouldn't need this intermediary
  682. # class.
  683. class _ReplaceInnerCode(DelegatingLexer):
  684. def __init__(self, **options):
  685. super().__init__(pylexer, _PythonConsoleLexerBase, Other.Code, **options)
  686. super().__init__(tblexer, _ReplaceInnerCode, Other.Traceback, **options)
  687. class PythonTracebackLexer(RegexLexer):
  688. """
  689. For Python 3.x tracebacks, with support for chained exceptions.
  690. .. versionchanged:: 2.5
  691. This is now the default ``PythonTracebackLexer``. It is still available
  692. as the alias ``Python3TracebackLexer``.
  693. """
  694. name = 'Python Traceback'
  695. aliases = ['pytb', 'py3tb']
  696. filenames = ['*.pytb', '*.py3tb']
  697. mimetypes = ['text/x-python-traceback', 'text/x-python3-traceback']
  698. url = 'https://python.org'
  699. version_added = '1.0'
  700. tokens = {
  701. 'root': [
  702. (r'\n', Whitespace),
  703. (r'^(\^C)?Traceback \(most recent call last\):\n', Generic.Traceback, 'intb'),
  704. (r'^During handling of the above exception, another '
  705. r'exception occurred:\n\n', Generic.Traceback),
  706. (r'^The above exception was the direct cause of the '
  707. r'following exception:\n\n', Generic.Traceback),
  708. (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
  709. (r'^.*\n', Other),
  710. ],
  711. 'intb': [
  712. (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
  713. bygroups(Text, Name.Builtin, Text, Number, Text, Name, Whitespace)),
  714. (r'^( File )("[^"]+")(, line )(\d+)(\n)',
  715. bygroups(Text, Name.Builtin, Text, Number, Whitespace)),
  716. (r'^( )(.+)(\n)',
  717. bygroups(Whitespace, using(PythonLexer), Whitespace), 'markers'),
  718. (r'^([ \t]*)(\.\.\.)(\n)',
  719. bygroups(Whitespace, Comment, Whitespace)), # for doctests...
  720. (r'^([^:]+)(: )(.+)(\n)',
  721. bygroups(Generic.Error, Text, Name, Whitespace), '#pop'),
  722. (r'^([a-zA-Z_][\w.]*)(:?\n)',
  723. bygroups(Generic.Error, Whitespace), '#pop'),
  724. default('#pop'),
  725. ],
  726. 'markers': [
  727. # Either `PEP 657 <https://www.python.org/dev/peps/pep-0657/>`
  728. # error locations in Python 3.11+, or single-caret markers
  729. # for syntax errors before that.
  730. (r'^( {4,})([~^]+)(\n)',
  731. bygroups(Whitespace, Punctuation.Marker, Whitespace),
  732. '#pop'),
  733. default('#pop'),
  734. ],
  735. }
  736. Python3TracebackLexer = PythonTracebackLexer
  737. class Python2TracebackLexer(RegexLexer):
  738. """
  739. For Python tracebacks.
  740. .. versionchanged:: 2.5
  741. This class has been renamed from ``PythonTracebackLexer``.
  742. ``PythonTracebackLexer`` now refers to the Python 3 variant.
  743. """
  744. name = 'Python 2.x Traceback'
  745. aliases = ['py2tb']
  746. filenames = ['*.py2tb']
  747. mimetypes = ['text/x-python2-traceback']
  748. url = 'https://python.org'
  749. version_added = '0.7'
  750. tokens = {
  751. 'root': [
  752. # Cover both (most recent call last) and (innermost last)
  753. # The optional ^C allows us to catch keyboard interrupt signals.
  754. (r'^(\^C)?(Traceback.*\n)',
  755. bygroups(Text, Generic.Traceback), 'intb'),
  756. # SyntaxError starts with this.
  757. (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
  758. (r'^.*\n', Other),
  759. ],
  760. 'intb': [
  761. (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
  762. bygroups(Text, Name.Builtin, Text, Number, Text, Name, Whitespace)),
  763. (r'^( File )("[^"]+")(, line )(\d+)(\n)',
  764. bygroups(Text, Name.Builtin, Text, Number, Whitespace)),
  765. (r'^( )(.+)(\n)',
  766. bygroups(Text, using(Python2Lexer), Whitespace), 'marker'),
  767. (r'^([ \t]*)(\.\.\.)(\n)',
  768. bygroups(Text, Comment, Whitespace)), # for doctests...
  769. (r'^([^:]+)(: )(.+)(\n)',
  770. bygroups(Generic.Error, Text, Name, Whitespace), '#pop'),
  771. (r'^([a-zA-Z_]\w*)(:?\n)',
  772. bygroups(Generic.Error, Whitespace), '#pop')
  773. ],
  774. 'marker': [
  775. # For syntax errors.
  776. (r'( {4,})(\^)', bygroups(Text, Punctuation.Marker), '#pop'),
  777. default('#pop'),
  778. ],
  779. }
  780. class CythonLexer(RegexLexer):
  781. """
  782. For Pyrex and Cython source code.
  783. """
  784. name = 'Cython'
  785. url = 'https://cython.org'
  786. aliases = ['cython', 'pyx', 'pyrex']
  787. filenames = ['*.pyx', '*.pxd', '*.pxi']
  788. mimetypes = ['text/x-cython', 'application/x-cython']
  789. version_added = '1.1'
  790. tokens = {
  791. 'root': [
  792. (r'\n', Whitespace),
  793. (r'^(\s*)("""(?:.|\n)*?""")', bygroups(Whitespace, String.Doc)),
  794. (r"^(\s*)('''(?:.|\n)*?''')", bygroups(Whitespace, String.Doc)),
  795. (r'[^\S\n]+', Text),
  796. (r'#.*$', Comment),
  797. (r'[]{}:(),;[]', Punctuation),
  798. (r'\\\n', Whitespace),
  799. (r'\\', Text),
  800. (r'(in|is|and|or|not)\b', Operator.Word),
  801. (r'(<)([a-zA-Z0-9.?]+)(>)',
  802. bygroups(Punctuation, Keyword.Type, Punctuation)),
  803. (r'!=|==|<<|>>|[-~+/*%=<>&^|.?]', Operator),
  804. (r'(from)(\d+)(<=)(\s+)(<)(\d+)(:)',
  805. bygroups(Keyword, Number.Integer, Operator, Whitespace, Operator,
  806. Name, Punctuation)),
  807. include('keywords'),
  808. (r'(def|property)(\s+)', bygroups(Keyword, Whitespace), 'funcname'),
  809. (r'(cp?def)(\s+)', bygroups(Keyword, Whitespace), 'cdef'),
  810. # (should actually start a block with only cdefs)
  811. (r'(cdef)(:)', bygroups(Keyword, Punctuation)),
  812. (r'(class|cppclass|struct)(\s+)', bygroups(Keyword, Whitespace), 'classname'),
  813. (r'(from)(\s+)', bygroups(Keyword, Whitespace), 'fromimport'),
  814. (r'(c?import)(\s+)', bygroups(Keyword, Whitespace), 'import'),
  815. include('builtins'),
  816. include('backtick'),
  817. ('(?:[rR]|[uU][rR]|[rR][uU])"""', String, 'tdqs'),
  818. ("(?:[rR]|[uU][rR]|[rR][uU])'''", String, 'tsqs'),
  819. ('(?:[rR]|[uU][rR]|[rR][uU])"', String, 'dqs'),
  820. ("(?:[rR]|[uU][rR]|[rR][uU])'", String, 'sqs'),
  821. ('[uU]?"""', String, combined('stringescape', 'tdqs')),
  822. ("[uU]?'''", String, combined('stringescape', 'tsqs')),
  823. ('[uU]?"', String, combined('stringescape', 'dqs')),
  824. ("[uU]?'", String, combined('stringescape', 'sqs')),
  825. include('name'),
  826. include('numbers'),
  827. ],
  828. 'keywords': [
  829. (words((
  830. 'assert', 'async', 'await', 'break', 'by', 'continue', 'ctypedef', 'del',
  831. 'elif', 'else', 'except', 'except?', 'exec', 'finally', 'for', 'fused', 'gil',
  832. 'global', 'if', 'include', 'lambda', 'namespace', 'new', 'noexcept','nogil',
  833. 'pass', 'print', 'raise', 'return', 'try', 'while', 'yield', 'as', 'with'),
  834. suffix=r'\b'),
  835. Keyword),
  836. (words(('True', 'False', 'None', 'NULL'), suffix=r'\b'), Keyword.Constant),
  837. (r'(DEF|IF|ELIF|ELSE)\b', Comment.Preproc),
  838. ],
  839. 'builtins': [
  840. (words((
  841. '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin', 'bint',
  842. 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'char', 'chr',
  843. 'classmethod', 'cmp', 'coerce', 'compile', 'complex', 'delattr',
  844. 'dict', 'dir', 'divmod', 'double', 'enumerate', 'eval', 'execfile', 'exit',
  845. 'file', 'filter', 'float', 'frozenset', 'getattr', 'globals',
  846. 'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'intern', 'isinstance',
  847. 'issubclass', 'iter', 'len', 'list', 'locals', 'long', 'map', 'max',
  848. 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'property',
  849. 'Py_ssize_t', 'range', 'raw_input', 'reduce', 'reload', 'repr', 'reversed',
  850. 'round', 'set', 'setattr', 'size_t', 'slice', 'sorted', 'staticmethod',
  851. 'ssize_t', 'str', 'sum', 'super', 'tuple', 'type', 'unichr', 'unicode',
  852. 'unsigned', 'vars', 'xrange', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'),
  853. Name.Builtin),
  854. (r'(?<!\.)(self|cls|Ellipsis|NotImplemented)\b', Name.Builtin.Pseudo),
  855. (words((
  856. 'ArithmeticError', 'AssertionError', 'AttributeError',
  857. 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError',
  858. 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit',
  859. 'IOError', 'ImportError', 'ImportWarning', 'IndentationError',
  860. 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError',
  861. 'MemoryError', 'NameError', 'NotImplementedError',
  862. 'OSError', 'OverflowError', 'OverflowWarning',
  863. 'PendingDeprecationWarning', 'ReferenceError', 'RuntimeError',
  864. 'RuntimeWarning', 'StandardError', 'StopIteration', 'SyntaxError',
  865. 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError',
  866. 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
  867. 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
  868. 'UnicodeWarning', 'UserWarning', 'ValueError', 'Warning',
  869. 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'),
  870. Name.Exception),
  871. ],
  872. 'numbers': [
  873. (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
  874. (r'0\d+', Number.Oct),
  875. (r'0[xX][a-fA-F0-9]+', Number.Hex),
  876. (r'\d+L', Number.Integer.Long),
  877. (r'\d+', Number.Integer)
  878. ],
  879. 'backtick': [
  880. ('`.*?`', String.Backtick),
  881. ],
  882. 'name': [
  883. (r'@\w+', Name.Decorator),
  884. (r'[a-zA-Z_]\w*', Name),
  885. ],
  886. 'funcname': [
  887. (r'[a-zA-Z_]\w*', Name.Function, '#pop')
  888. ],
  889. 'cdef': [
  890. (r"(public|readonly|extern|api|inline|packed)\b", Keyword.Reserved),
  891. (r"(struct|enum|union|class|cppclass)\b(\s+)([a-zA-Z_]\w*)",
  892. bygroups(Keyword, Whitespace, Name.Class), "#pop",),
  893. (r"([a-zA-Z_]\w*)(\s*)(?=\()", bygroups(Name.Function, Whitespace), "#pop"),
  894. (r"([a-zA-Z_]\w*)(\s*)(?=[:,=#\n]|$)", bygroups(Name.Variable, Whitespace), "#pop"),
  895. (r"([a-zA-Z_]\w*)(\s*)(,)", bygroups(Name.Variable, Whitespace, Punctuation)),
  896. (r'from\b', Keyword, '#pop'),
  897. (r'as\b', Keyword),
  898. (r':', Punctuation, '#pop'),
  899. (r'(?=["\'])', Text, '#pop'),
  900. (r'[a-zA-Z_]\w*', Keyword.Type),
  901. (r'.', Text),
  902. ],
  903. 'classname': [
  904. (r'[a-zA-Z_]\w*', Name.Class, '#pop')
  905. ],
  906. 'import': [
  907. (r'(\s+)(as)(\s+)', bygroups(Whitespace, Keyword, Whitespace)),
  908. (r'[a-zA-Z_][\w.]*', Name.Namespace),
  909. (r'(\s*)(,)(\s*)', bygroups(Whitespace, Operator, Whitespace)),
  910. default('#pop') # all else: go back
  911. ],
  912. 'fromimport': [
  913. (r'(\s+)(c?import)\b', bygroups(Whitespace, Keyword), '#pop'),
  914. (r'[a-zA-Z_.][\w.]*', Name.Namespace),
  915. # ``cdef foo from "header"``, or ``for foo from 0 < i < 10``
  916. default('#pop'),
  917. ],
  918. 'stringescape': [
  919. (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
  920. r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
  921. ],
  922. 'strings': [
  923. (r'%(\([a-zA-Z0-9]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
  924. '[hlL]?[E-GXc-giorsux%]', String.Interpol),
  925. (r'[^\\\'"%\n]+', String),
  926. # quotes, percents and backslashes must be parsed one at a time
  927. (r'[\'"\\]', String),
  928. # unhandled string formatting sign
  929. (r'%', String)
  930. # newlines are an error (use "nl" state)
  931. ],
  932. 'nl': [
  933. (r'\n', String)
  934. ],
  935. 'dqs': [
  936. (r'"', String, '#pop'),
  937. (r'\\\\|\\"|\\\n', String.Escape), # included here again for raw strings
  938. include('strings')
  939. ],
  940. 'sqs': [
  941. (r"'", String, '#pop'),
  942. (r"\\\\|\\'|\\\n", String.Escape), # included here again for raw strings
  943. include('strings')
  944. ],
  945. 'tdqs': [
  946. (r'"""', String, '#pop'),
  947. include('strings'),
  948. include('nl')
  949. ],
  950. 'tsqs': [
  951. (r"'''", String, '#pop'),
  952. include('strings'),
  953. include('nl')
  954. ],
  955. }
  956. class DgLexer(RegexLexer):
  957. """
  958. Lexer for dg,
  959. a functional and object-oriented programming language
  960. running on the CPython 3 VM.
  961. """
  962. name = 'dg'
  963. aliases = ['dg']
  964. filenames = ['*.dg']
  965. mimetypes = ['text/x-dg']
  966. url = 'http://pyos.github.io/dg'
  967. version_added = '1.6'
  968. tokens = {
  969. 'root': [
  970. (r'\s+', Text),
  971. (r'#.*?$', Comment.Single),
  972. (r'(?i)0b[01]+', Number.Bin),
  973. (r'(?i)0o[0-7]+', Number.Oct),
  974. (r'(?i)0x[0-9a-f]+', Number.Hex),
  975. (r'(?i)[+-]?[0-9]+\.[0-9]+(e[+-]?[0-9]+)?j?', Number.Float),
  976. (r'(?i)[+-]?[0-9]+e[+-]?\d+j?', Number.Float),
  977. (r'(?i)[+-]?[0-9]+j?', Number.Integer),
  978. (r"(?i)(br|r?b?)'''", String, combined('stringescape', 'tsqs', 'string')),
  979. (r'(?i)(br|r?b?)"""', String, combined('stringescape', 'tdqs', 'string')),
  980. (r"(?i)(br|r?b?)'", String, combined('stringescape', 'sqs', 'string')),
  981. (r'(?i)(br|r?b?)"', String, combined('stringescape', 'dqs', 'string')),
  982. (r"`\w+'*`", Operator),
  983. (r'\b(and|in|is|or|where)\b', Operator.Word),
  984. (r'[!$%&*+\-./:<-@\\^|~;,]+', Operator),
  985. (words((
  986. 'bool', 'bytearray', 'bytes', 'classmethod', 'complex', 'dict', 'dict\'',
  987. 'float', 'frozenset', 'int', 'list', 'list\'', 'memoryview', 'object',
  988. 'property', 'range', 'set', 'set\'', 'slice', 'staticmethod', 'str',
  989. 'super', 'tuple', 'tuple\'', 'type'),
  990. prefix=r'(?<!\.)', suffix=r'(?![\'\w])'),
  991. Name.Builtin),
  992. (words((
  993. '__import__', 'abs', 'all', 'any', 'bin', 'bind', 'chr', 'cmp', 'compile',
  994. 'complex', 'delattr', 'dir', 'divmod', 'drop', 'dropwhile', 'enumerate',
  995. 'eval', 'exhaust', 'filter', 'flip', 'foldl1?', 'format', 'fst',
  996. 'getattr', 'globals', 'hasattr', 'hash', 'head', 'hex', 'id', 'init',
  997. 'input', 'isinstance', 'issubclass', 'iter', 'iterate', 'last', 'len',
  998. 'locals', 'map', 'max', 'min', 'next', 'oct', 'open', 'ord', 'pow',
  999. 'print', 'repr', 'reversed', 'round', 'setattr', 'scanl1?', 'snd',
  1000. 'sorted', 'sum', 'tail', 'take', 'takewhile', 'vars', 'zip'),
  1001. prefix=r'(?<!\.)', suffix=r'(?![\'\w])'),
  1002. Name.Builtin),
  1003. (r"(?<!\.)(self|Ellipsis|NotImplemented|None|True|False)(?!['\w])",
  1004. Name.Builtin.Pseudo),
  1005. (r"(?<!\.)[A-Z]\w*(Error|Exception|Warning)'*(?!['\w])",
  1006. Name.Exception),
  1007. (r"(?<!\.)(Exception|GeneratorExit|KeyboardInterrupt|StopIteration|"
  1008. r"SystemExit)(?!['\w])", Name.Exception),
  1009. (r"(?<![\w.])(except|finally|for|if|import|not|otherwise|raise|"
  1010. r"subclass|while|with|yield)(?!['\w])", Keyword.Reserved),
  1011. (r"[A-Z_]+'*(?!['\w])", Name),
  1012. (r"[A-Z]\w+'*(?!['\w])", Keyword.Type),
  1013. (r"\w+'*", Name),
  1014. (r'[()]', Punctuation),
  1015. (r'.', Error),
  1016. ],
  1017. 'stringescape': [
  1018. (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
  1019. r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
  1020. ],
  1021. 'string': [
  1022. (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
  1023. '[hlL]?[E-GXc-giorsux%]', String.Interpol),
  1024. (r'[^\\\'"%\n]+', String),
  1025. # quotes, percents and backslashes must be parsed one at a time
  1026. (r'[\'"\\]', String),
  1027. # unhandled string formatting sign
  1028. (r'%', String),
  1029. (r'\n', String)
  1030. ],
  1031. 'dqs': [
  1032. (r'"', String, '#pop')
  1033. ],
  1034. 'sqs': [
  1035. (r"'", String, '#pop')
  1036. ],
  1037. 'tdqs': [
  1038. (r'"""', String, '#pop')
  1039. ],
  1040. 'tsqs': [
  1041. (r"'''", String, '#pop')
  1042. ],
  1043. }
  1044. class NumPyLexer(PythonLexer):
  1045. """
  1046. A Python lexer recognizing Numerical Python builtins.
  1047. """
  1048. name = 'NumPy'
  1049. url = 'https://numpy.org/'
  1050. aliases = ['numpy']
  1051. version_added = '0.10'
  1052. # override the mimetypes to not inherit them from python
  1053. mimetypes = []
  1054. filenames = []
  1055. EXTRA_KEYWORDS = {
  1056. 'abs', 'absolute', 'accumulate', 'add', 'alen', 'all', 'allclose',
  1057. 'alltrue', 'alterdot', 'amax', 'amin', 'angle', 'any', 'append',
  1058. 'apply_along_axis', 'apply_over_axes', 'arange', 'arccos', 'arccosh',
  1059. 'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'argmax', 'argmin',
  1060. 'argsort', 'argwhere', 'around', 'array', 'array2string', 'array_equal',
  1061. 'array_equiv', 'array_repr', 'array_split', 'array_str', 'arrayrange',
  1062. 'asanyarray', 'asarray', 'asarray_chkfinite', 'ascontiguousarray',
  1063. 'asfarray', 'asfortranarray', 'asmatrix', 'asscalar', 'astype',
  1064. 'atleast_1d', 'atleast_2d', 'atleast_3d', 'average', 'bartlett',
  1065. 'base_repr', 'beta', 'binary_repr', 'bincount', 'binomial',
  1066. 'bitwise_and', 'bitwise_not', 'bitwise_or', 'bitwise_xor', 'blackman',
  1067. 'bmat', 'broadcast', 'byte_bounds', 'bytes', 'byteswap', 'c_',
  1068. 'can_cast', 'ceil', 'choose', 'clip', 'column_stack', 'common_type',
  1069. 'compare_chararrays', 'compress', 'concatenate', 'conj', 'conjugate',
  1070. 'convolve', 'copy', 'corrcoef', 'correlate', 'cos', 'cosh', 'cov',
  1071. 'cross', 'cumprod', 'cumproduct', 'cumsum', 'delete', 'deprecate',
  1072. 'diag', 'diagflat', 'diagonal', 'diff', 'digitize', 'disp', 'divide',
  1073. 'dot', 'dsplit', 'dstack', 'dtype', 'dump', 'dumps', 'ediff1d', 'empty',
  1074. 'empty_like', 'equal', 'exp', 'expand_dims', 'expm1', 'extract', 'eye',
  1075. 'fabs', 'fastCopyAndTranspose', 'fft', 'fftfreq', 'fftshift', 'fill',
  1076. 'finfo', 'fix', 'flat', 'flatnonzero', 'flatten', 'fliplr', 'flipud',
  1077. 'floor', 'floor_divide', 'fmod', 'frexp', 'fromarrays', 'frombuffer',
  1078. 'fromfile', 'fromfunction', 'fromiter', 'frompyfunc', 'fromstring',
  1079. 'generic', 'get_array_wrap', 'get_include', 'get_numarray_include',
  1080. 'get_numpy_include', 'get_printoptions', 'getbuffer', 'getbufsize',
  1081. 'geterr', 'geterrcall', 'geterrobj', 'getfield', 'gradient', 'greater',
  1082. 'greater_equal', 'gumbel', 'hamming', 'hanning', 'histogram',
  1083. 'histogram2d', 'histogramdd', 'hsplit', 'hstack', 'hypot', 'i0',
  1084. 'identity', 'ifft', 'imag', 'index_exp', 'indices', 'inf', 'info',
  1085. 'inner', 'insert', 'int_asbuffer', 'interp', 'intersect1d',
  1086. 'intersect1d_nu', 'inv', 'invert', 'iscomplex', 'iscomplexobj',
  1087. 'isfinite', 'isfortran', 'isinf', 'isnan', 'isneginf', 'isposinf',
  1088. 'isreal', 'isrealobj', 'isscalar', 'issctype', 'issubclass_',
  1089. 'issubdtype', 'issubsctype', 'item', 'itemset', 'iterable', 'ix_',
  1090. 'kaiser', 'kron', 'ldexp', 'left_shift', 'less', 'less_equal', 'lexsort',
  1091. 'linspace', 'load', 'loads', 'loadtxt', 'log', 'log10', 'log1p', 'log2',
  1092. 'logical_and', 'logical_not', 'logical_or', 'logical_xor', 'logspace',
  1093. 'lstsq', 'mat', 'matrix', 'max', 'maximum', 'maximum_sctype',
  1094. 'may_share_memory', 'mean', 'median', 'meshgrid', 'mgrid', 'min',
  1095. 'minimum', 'mintypecode', 'mod', 'modf', 'msort', 'multiply', 'nan',
  1096. 'nan_to_num', 'nanargmax', 'nanargmin', 'nanmax', 'nanmin', 'nansum',
  1097. 'ndenumerate', 'ndim', 'ndindex', 'negative', 'newaxis', 'newbuffer',
  1098. 'newbyteorder', 'nonzero', 'not_equal', 'obj2sctype', 'ogrid', 'ones',
  1099. 'ones_like', 'outer', 'permutation', 'piecewise', 'pinv', 'pkgload',
  1100. 'place', 'poisson', 'poly', 'poly1d', 'polyadd', 'polyder', 'polydiv',
  1101. 'polyfit', 'polyint', 'polymul', 'polysub', 'polyval', 'power', 'prod',
  1102. 'product', 'ptp', 'put', 'putmask', 'r_', 'randint', 'random_integers',
  1103. 'random_sample', 'ranf', 'rank', 'ravel', 'real', 'real_if_close',
  1104. 'recarray', 'reciprocal', 'reduce', 'remainder', 'repeat', 'require',
  1105. 'reshape', 'resize', 'restoredot', 'right_shift', 'rint', 'roll',
  1106. 'rollaxis', 'roots', 'rot90', 'round', 'round_', 'row_stack', 's_',
  1107. 'sample', 'savetxt', 'sctype2char', 'searchsorted', 'seed', 'select',
  1108. 'set_numeric_ops', 'set_printoptions', 'set_string_function',
  1109. 'setbufsize', 'setdiff1d', 'seterr', 'seterrcall', 'seterrobj',
  1110. 'setfield', 'setflags', 'setmember1d', 'setxor1d', 'shape',
  1111. 'show_config', 'shuffle', 'sign', 'signbit', 'sin', 'sinc', 'sinh',
  1112. 'size', 'slice', 'solve', 'sometrue', 'sort', 'sort_complex', 'source',
  1113. 'split', 'sqrt', 'square', 'squeeze', 'standard_normal', 'std',
  1114. 'subtract', 'sum', 'svd', 'swapaxes', 'take', 'tan', 'tanh', 'tensordot',
  1115. 'test', 'tile', 'tofile', 'tolist', 'tostring', 'trace', 'transpose',
  1116. 'trapz', 'tri', 'tril', 'trim_zeros', 'triu', 'true_divide', 'typeDict',
  1117. 'typename', 'uniform', 'union1d', 'unique', 'unique1d', 'unravel_index',
  1118. 'unwrap', 'vander', 'var', 'vdot', 'vectorize', 'view', 'vonmises',
  1119. 'vsplit', 'vstack', 'weibull', 'where', 'who', 'zeros', 'zeros_like'
  1120. }
  1121. def get_tokens_unprocessed(self, text):
  1122. for index, token, value in \
  1123. PythonLexer.get_tokens_unprocessed(self, text):
  1124. if token is Name and value in self.EXTRA_KEYWORDS:
  1125. yield index, Keyword.Pseudo, value
  1126. else:
  1127. yield index, token, value
  1128. def analyse_text(text):
  1129. ltext = text[:1000]
  1130. return (shebang_matches(text, r'pythonw?(3(\.\d)?)?') or
  1131. 'import ' in ltext) \
  1132. and ('import numpy' in ltext or 'from numpy import' in ltext)