sql.py 41 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111
  1. """
  2. pygments.lexers.sql
  3. ~~~~~~~~~~~~~~~~~~~
  4. Lexers for various SQL dialects and related interactive sessions.
  5. Postgres specific lexers:
  6. `PostgresLexer`
  7. A SQL lexer for the PostgreSQL dialect. Differences w.r.t. the SQL
  8. lexer are:
  9. - keywords and data types list parsed from the PG docs (run the
  10. `_postgres_builtins` module to update them);
  11. - Content of $-strings parsed using a specific lexer, e.g. the content
  12. of a PL/Python function is parsed using the Python lexer;
  13. - parse PG specific constructs: E-strings, $-strings, U&-strings,
  14. different operators and punctuation.
  15. `PlPgsqlLexer`
  16. A lexer for the PL/pgSQL language. Adds a few specific construct on
  17. top of the PG SQL lexer (such as <<label>>).
  18. `PostgresConsoleLexer`
  19. A lexer to highlight an interactive psql session:
  20. - identifies the prompt and does its best to detect the end of command
  21. in multiline statement where not all the lines are prefixed by a
  22. prompt, telling them apart from the output;
  23. - highlights errors in the output and notification levels;
  24. - handles psql backslash commands.
  25. `PostgresExplainLexer`
  26. A lexer to highlight Postgres execution plan.
  27. The ``tests/examplefiles`` contains a few test files with data to be
  28. parsed by these lexers.
  29. :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
  30. :license: BSD, see LICENSE for details.
  31. """
  32. import collections
  33. import re
  34. from pygments.lexer import Lexer, RegexLexer, do_insertions, bygroups, words
  35. from pygments.lexers import _googlesql_builtins
  36. from pygments.lexers import _mysql_builtins
  37. from pygments.lexers import _postgres_builtins
  38. from pygments.lexers import _sql_builtins
  39. from pygments.lexers import _tsql_builtins
  40. from pygments.lexers import get_lexer_by_name, ClassNotFound
  41. from pygments.token import Punctuation, Whitespace, Text, Comment, Operator, \
  42. Keyword, Name, String, Number, Generic, Literal
  43. __all__ = ['GoogleSqlLexer', 'PostgresLexer', 'PlPgsqlLexer',
  44. 'PostgresConsoleLexer', 'PostgresExplainLexer', 'SqlLexer',
  45. 'TransactSqlLexer', 'MySqlLexer', 'SqliteConsoleLexer', 'RqlLexer']
  46. line_re = re.compile('.*?\n')
  47. sqlite_prompt_re = re.compile(r'^(?:sqlite| ...)>(?= )')
  48. language_re = re.compile(r"\s+LANGUAGE\s+'?(\w+)'?", re.IGNORECASE)
  49. do_re = re.compile(r'\bDO\b', re.IGNORECASE)
  50. # Regular expressions for analyse_text()
  51. name_between_bracket_re = re.compile(r'\[[a-zA-Z_]\w*\]')
  52. name_between_backtick_re = re.compile(r'`[a-zA-Z_]\w*`')
  53. tsql_go_re = re.compile(r'\bgo\b', re.IGNORECASE)
  54. tsql_declare_re = re.compile(r'\bdeclare\s+@', re.IGNORECASE)
  55. tsql_variable_re = re.compile(r'@[a-zA-Z_]\w*\b')
  56. # Identifiers for analyse_text()
  57. googlesql_identifiers = (
  58. _googlesql_builtins.functionnames
  59. + _googlesql_builtins.keywords
  60. + _googlesql_builtins.types)
  61. def language_callback(lexer, match):
  62. """Parse the content of a $-string using a lexer
  63. The lexer is chosen looking for a nearby LANGUAGE or assumed as
  64. plpgsql if inside a DO statement and no LANGUAGE has been found.
  65. """
  66. lx = None
  67. m = language_re.match(lexer.text[match.end():match.end()+100])
  68. if m is not None:
  69. lx = lexer._get_lexer(m.group(1))
  70. else:
  71. m = list(language_re.finditer(
  72. lexer.text[max(0, match.start()-100):match.start()]))
  73. if m:
  74. lx = lexer._get_lexer(m[-1].group(1))
  75. else:
  76. m = list(do_re.finditer(
  77. lexer.text[max(0, match.start()-25):match.start()]))
  78. if m:
  79. lx = lexer._get_lexer('plpgsql')
  80. # 1 = $, 2 = delimiter, 3 = $
  81. yield (match.start(1), String, match.group(1))
  82. yield (match.start(2), String.Delimiter, match.group(2))
  83. yield (match.start(3), String, match.group(3))
  84. # 4 = string contents
  85. if lx:
  86. yield from lx.get_tokens_unprocessed(match.group(4))
  87. else:
  88. yield (match.start(4), String, match.group(4))
  89. # 5 = $, 6 = delimiter, 7 = $
  90. yield (match.start(5), String, match.group(5))
  91. yield (match.start(6), String.Delimiter, match.group(6))
  92. yield (match.start(7), String, match.group(7))
  93. class PostgresBase:
  94. """Base class for Postgres-related lexers.
  95. This is implemented as a mixin to avoid the Lexer metaclass kicking in.
  96. this way the different lexer don't have a common Lexer ancestor. If they
  97. had, _tokens could be created on this ancestor and not updated for the
  98. other classes, resulting e.g. in PL/pgSQL parsed as SQL. This shortcoming
  99. seem to suggest that regexp lexers are not really subclassable.
  100. """
  101. def get_tokens_unprocessed(self, text, *args):
  102. # Have a copy of the entire text to be used by `language_callback`.
  103. self.text = text
  104. yield from super().get_tokens_unprocessed(text, *args)
  105. def _get_lexer(self, lang):
  106. if lang.lower() == 'sql':
  107. return get_lexer_by_name('postgresql', **self.options)
  108. tries = [lang]
  109. if lang.startswith('pl'):
  110. tries.append(lang[2:])
  111. if lang.endswith('u'):
  112. tries.append(lang[:-1])
  113. if lang.startswith('pl') and lang.endswith('u'):
  114. tries.append(lang[2:-1])
  115. for lx in tries:
  116. try:
  117. return get_lexer_by_name(lx, **self.options)
  118. except ClassNotFound:
  119. pass
  120. else:
  121. # TODO: better logging
  122. # print >>sys.stderr, "language not found:", lang
  123. return None
  124. class PostgresLexer(PostgresBase, RegexLexer):
  125. """
  126. Lexer for the PostgreSQL dialect of SQL.
  127. """
  128. name = 'PostgreSQL SQL dialect'
  129. aliases = ['postgresql', 'postgres']
  130. mimetypes = ['text/x-postgresql']
  131. url = 'https://www.postgresql.org'
  132. version_added = '1.5'
  133. flags = re.IGNORECASE
  134. tokens = {
  135. 'root': [
  136. (r'\s+', Whitespace),
  137. (r'--.*\n?', Comment.Single),
  138. (r'/\*', Comment.Multiline, 'multiline-comments'),
  139. (r'(' + '|'.join(s.replace(" ", r"\s+")
  140. for s in _postgres_builtins.DATATYPES +
  141. _postgres_builtins.PSEUDO_TYPES) + r')\b',
  142. Name.Builtin),
  143. (words(_postgres_builtins.KEYWORDS, suffix=r'\b'), Keyword),
  144. (r'[+*/<>=~!@#%^&|`?-]+', Operator),
  145. (r'::', Operator), # cast
  146. (r'\$\d+', Name.Variable),
  147. (r'([0-9]*\.[0-9]*|[0-9]+)(e[+-]?[0-9]+)?', Number.Float),
  148. (r'[0-9]+', Number.Integer),
  149. (r"((?:E|U&)?)(')", bygroups(String.Affix, String.Single), 'string'),
  150. # quoted identifier
  151. (r'((?:U&)?)(")', bygroups(String.Affix, String.Name), 'quoted-ident'),
  152. (r'(?s)(\$)([^$]*)(\$)(.*?)(\$)(\2)(\$)', language_callback),
  153. (r'[a-z_]\w*', Name),
  154. # psql variable in SQL
  155. (r""":(['"]?)[a-z]\w*\b\1""", Name.Variable),
  156. (r'[;:()\[\]{},.]', Punctuation),
  157. ],
  158. 'multiline-comments': [
  159. (r'/\*', Comment.Multiline, 'multiline-comments'),
  160. (r'\*/', Comment.Multiline, '#pop'),
  161. (r'[^/*]+', Comment.Multiline),
  162. (r'[/*]', Comment.Multiline)
  163. ],
  164. 'string': [
  165. (r"[^']+", String.Single),
  166. (r"''", String.Single),
  167. (r"'", String.Single, '#pop'),
  168. ],
  169. 'quoted-ident': [
  170. (r'[^"]+', String.Name),
  171. (r'""', String.Name),
  172. (r'"', String.Name, '#pop'),
  173. ],
  174. }
  175. class PlPgsqlLexer(PostgresBase, RegexLexer):
  176. """
  177. Handle the extra syntax in Pl/pgSQL language.
  178. """
  179. name = 'PL/pgSQL'
  180. aliases = ['plpgsql']
  181. mimetypes = ['text/x-plpgsql']
  182. url = 'https://www.postgresql.org/docs/current/plpgsql.html'
  183. version_added = '1.5'
  184. flags = re.IGNORECASE
  185. # FIXME: use inheritance
  186. tokens = {name: state[:] for (name, state) in PostgresLexer.tokens.items()}
  187. # extend the keywords list
  188. for i, pattern in enumerate(tokens['root']):
  189. if pattern[1] == Keyword:
  190. tokens['root'][i] = (
  191. words(_postgres_builtins.KEYWORDS +
  192. _postgres_builtins.PLPGSQL_KEYWORDS, suffix=r'\b'),
  193. Keyword)
  194. del i
  195. break
  196. else:
  197. assert 0, "SQL keywords not found"
  198. # Add specific PL/pgSQL rules (before the SQL ones)
  199. tokens['root'][:0] = [
  200. (r'\%[a-z]\w*\b', Name.Builtin), # actually, a datatype
  201. (r':=', Operator),
  202. (r'\<\<[a-z]\w*\>\>', Name.Label),
  203. (r'\#[a-z]\w*\b', Keyword.Pseudo), # #variable_conflict
  204. ]
  205. class PsqlRegexLexer(PostgresBase, RegexLexer):
  206. """
  207. Extend the PostgresLexer adding support specific for psql commands.
  208. This is not a complete psql lexer yet as it lacks prompt support
  209. and output rendering.
  210. """
  211. name = 'PostgreSQL console - regexp based lexer'
  212. aliases = [] # not public
  213. flags = re.IGNORECASE
  214. tokens = {name: state[:] for (name, state) in PostgresLexer.tokens.items()}
  215. tokens['root'].append(
  216. (r'\\[^\s]+', Keyword.Pseudo, 'psql-command'))
  217. tokens['psql-command'] = [
  218. (r'\n', Text, 'root'),
  219. (r'\s+', Whitespace),
  220. (r'\\[^\s]+', Keyword.Pseudo),
  221. (r""":(['"]?)[a-z]\w*\b\1""", Name.Variable),
  222. (r"'(''|[^'])*'", String.Single),
  223. (r"`([^`])*`", String.Backtick),
  224. (r"[^\s]+", String.Symbol),
  225. ]
  226. re_prompt = re.compile(r'^(\S.*?)??[=\-\(\$\'\"][#>]')
  227. re_psql_command = re.compile(r'\s*\\')
  228. re_end_command = re.compile(r';\s*(--.*?)?$')
  229. re_psql_command = re.compile(r'(\s*)(\\.+?)(\s+)$')
  230. re_error = re.compile(r'(ERROR|FATAL):')
  231. re_message = re.compile(
  232. r'((?:DEBUG|INFO|NOTICE|WARNING|ERROR|'
  233. r'FATAL|HINT|DETAIL|CONTEXT|LINE [0-9]+):)(.*?\n)')
  234. class lookahead:
  235. """Wrap an iterator and allow pushing back an item."""
  236. def __init__(self, x):
  237. self.iter = iter(x)
  238. self._nextitem = None
  239. def __iter__(self):
  240. return self
  241. def send(self, i):
  242. self._nextitem = i
  243. return i
  244. def __next__(self):
  245. if self._nextitem is not None:
  246. ni = self._nextitem
  247. self._nextitem = None
  248. return ni
  249. return next(self.iter)
  250. next = __next__
  251. class PostgresConsoleLexer(Lexer):
  252. """
  253. Lexer for psql sessions.
  254. """
  255. name = 'PostgreSQL console (psql)'
  256. aliases = ['psql', 'postgresql-console', 'postgres-console']
  257. mimetypes = ['text/x-postgresql-psql']
  258. url = 'https://www.postgresql.org'
  259. version_added = '1.5'
  260. _example = "psql/psql_session.txt"
  261. def get_tokens_unprocessed(self, data):
  262. sql = PsqlRegexLexer(**self.options)
  263. lines = lookahead(line_re.findall(data))
  264. # prompt-output cycle
  265. while 1:
  266. # consume the lines of the command: start with an optional prompt
  267. # and continue until the end of command is detected
  268. curcode = ''
  269. insertions = []
  270. for line in lines:
  271. # Identify a shell prompt in case of psql commandline example
  272. if line.startswith('$') and not curcode:
  273. lexer = get_lexer_by_name('console', **self.options)
  274. yield from lexer.get_tokens_unprocessed(line)
  275. break
  276. # Identify a psql prompt
  277. mprompt = re_prompt.match(line)
  278. if mprompt is not None:
  279. insertions.append((len(curcode),
  280. [(0, Generic.Prompt, mprompt.group())]))
  281. curcode += line[len(mprompt.group()):]
  282. else:
  283. curcode += line
  284. # Check if this is the end of the command
  285. # TODO: better handle multiline comments at the end with
  286. # a lexer with an external state?
  287. if re_psql_command.match(curcode) \
  288. or re_end_command.search(curcode):
  289. break
  290. # Emit the combined stream of command and prompt(s)
  291. yield from do_insertions(insertions,
  292. sql.get_tokens_unprocessed(curcode))
  293. # Emit the output lines
  294. out_token = Generic.Output
  295. for line in lines:
  296. mprompt = re_prompt.match(line)
  297. if mprompt is not None:
  298. # push the line back to have it processed by the prompt
  299. lines.send(line)
  300. break
  301. mmsg = re_message.match(line)
  302. if mmsg is not None:
  303. if mmsg.group(1).startswith("ERROR") \
  304. or mmsg.group(1).startswith("FATAL"):
  305. out_token = Generic.Error
  306. yield (mmsg.start(1), Generic.Strong, mmsg.group(1))
  307. yield (mmsg.start(2), out_token, mmsg.group(2))
  308. else:
  309. yield (0, out_token, line)
  310. else:
  311. return
  312. class PostgresExplainLexer(RegexLexer):
  313. """
  314. Handle PostgreSQL EXPLAIN output
  315. """
  316. name = 'PostgreSQL EXPLAIN dialect'
  317. aliases = ['postgres-explain']
  318. filenames = ['*.explain']
  319. mimetypes = ['text/x-postgresql-explain']
  320. url = 'https://www.postgresql.org/docs/current/using-explain.html'
  321. version_added = '2.15'
  322. tokens = {
  323. 'root': [
  324. (r'(:|\(|\)|ms|kB|->|\.\.|\,|\/|=|%)', Punctuation),
  325. (r'(\s+)', Whitespace),
  326. # This match estimated cost and effectively measured counters with ANALYZE
  327. # Then, we move to instrumentation state
  328. (r'(cost)(=?)', bygroups(Name.Class, Punctuation), 'instrumentation'),
  329. (r'(actual)( )(=?)', bygroups(Name.Class, Whitespace, Punctuation), 'instrumentation'),
  330. # Misc keywords
  331. (words(('actual', 'Memory Usage', 'Disk Usage', 'Memory', 'Buckets',
  332. 'Batches', 'originally', 'row', 'rows', 'Hits', 'Misses',
  333. 'Evictions', 'Overflows', 'Planned Partitions', 'Estimates',
  334. 'capacity', 'distinct keys', 'lookups', 'hit percent',
  335. 'Index Searches', 'Storage', 'Disk Maximum Storage'), suffix=r'\b'),
  336. Comment.Single),
  337. (r'(hit|read|dirtied|written|write|time|calls)(=)', bygroups(Comment.Single, Operator)),
  338. (r'(shared|temp|local)', Keyword.Pseudo),
  339. # We move to sort state in order to emphasize specific keywords (especially disk access)
  340. (r'(Sort Method)(: )', bygroups(Comment.Preproc, Punctuation), 'sort'),
  341. # These keywords can be followed by an object, like a table
  342. (r'(Sort Key|Group Key|Presorted Key|Hash Key)(:)( )',
  343. bygroups(Comment.Preproc, Punctuation, Whitespace), 'object_name'),
  344. (r'(Cache Key|Cache Mode)(:)( )', bygroups(Comment, Punctuation, Whitespace), 'object_name'),
  345. # These keywords can be followed by a predicate
  346. (words(('Join Filter', 'Subplans Removed', 'Filter', 'Merge Cond',
  347. 'Hash Cond', 'Index Cond', 'Recheck Cond', 'Heap Blocks',
  348. 'TID Cond', 'Run Condition', 'Order By', 'Function Call',
  349. 'Table Function Call', 'Inner Unique', 'Params Evaluated',
  350. 'Single Copy', 'Sampling', 'One-Time Filter', 'Output',
  351. 'Relations', 'Remote SQL', 'Disabled'), suffix=r'\b'),
  352. Comment.Preproc, 'predicate'),
  353. # Special keyword to handle ON CONFLICT
  354. (r'Conflict ', Comment.Preproc, 'conflict'),
  355. # Special keyword for InitPlan or SubPlan
  356. (r'(InitPlan|SubPlan)( )(\d+)( )',
  357. bygroups(Keyword, Whitespace, Number.Integer, Whitespace),
  358. 'init_plan'),
  359. (words(('Sort Method', 'Join Filter', 'Planning time',
  360. 'Planning Time', 'Execution time', 'Execution Time',
  361. 'Workers Planned', 'Workers Launched', 'Buffers',
  362. 'Planning', 'Worker', 'Query Identifier', 'Time',
  363. 'Full-sort Groups', 'Pre-sorted Groups'), suffix=r'\b'), Comment.Preproc),
  364. # Emphasize these keywords
  365. (words(('Rows Removed by Join Filter', 'Rows Removed by Filter',
  366. 'Rows Removed by Index Recheck',
  367. 'Heap Fetches', 'never executed'),
  368. suffix=r'\b'), Name.Exception),
  369. (r'(I/O Timings)(:)( )', bygroups(Name.Exception, Punctuation, Whitespace)),
  370. (words(_postgres_builtins.EXPLAIN_KEYWORDS, suffix=r'\b'), Keyword),
  371. # join keywords
  372. (r'((Right|Left|Full|Semi|Anti) Join)', Keyword.Type),
  373. (r'(Parallel |Async |Finalize |Partial )', Comment.Preproc),
  374. (r'Backward', Comment.Preproc),
  375. (r'(Intersect|Except|Hash)', Comment.Preproc),
  376. (r'(CTE)( )(\w*)?', bygroups(Comment, Whitespace, Name.Variable)),
  377. # Treat "on" and "using" as a punctuation
  378. (r'(on|using)', Punctuation, 'object_name'),
  379. # strings
  380. (r"'(''|[^'])*'", String.Single),
  381. # numbers
  382. (r'-?\d+\.\d+', Number.Float),
  383. (r'(-?\d+)', Number.Integer),
  384. # boolean
  385. (r'(true|false)', Name.Constant),
  386. # explain header
  387. (r'\s*QUERY PLAN\s*\n\s*-+', Comment.Single),
  388. # Settings
  389. (r'(Settings)(:)( )', bygroups(Comment.Preproc, Punctuation, Whitespace), 'setting'),
  390. # Handle JIT counters
  391. (r'(JIT|Functions|Options|Timing)(:)', bygroups(Comment.Preproc, Punctuation)),
  392. (r'(Inlining|Optimization|Expressions|Deforming|Generation|Emission|Total)', Keyword.Pseudo),
  393. # Handle Triggers counters
  394. (r'(Trigger)( )(\S*)(:)( )',
  395. bygroups(Comment.Preproc, Whitespace, Name.Variable, Punctuation, Whitespace)),
  396. ],
  397. 'expression': [
  398. # matches any kind of parenthesized expression
  399. # the first opening paren is matched by the 'caller'
  400. (r'\(', Punctuation, '#push'),
  401. (r'\)', Punctuation, '#pop'),
  402. (r'(never executed)', Name.Exception),
  403. (r'[^)(]+', Comment),
  404. ],
  405. 'object_name': [
  406. # This is a cost or analyze measure
  407. (r'(\(cost)(=?)', bygroups(Name.Class, Punctuation), 'instrumentation'),
  408. (r'(\(actual)( )(=?)', bygroups(Name.Class, Whitespace, Punctuation), 'instrumentation'),
  409. # if object_name is parenthesized, mark opening paren as
  410. # punctuation, call 'expression', and exit state
  411. (r'\(', Punctuation, 'expression'),
  412. (r'(on)', Punctuation),
  413. # matches possibly schema-qualified table and column names
  414. (r'\w+(\.\w+)*( USING \S+| \w+ USING \S+)', Name.Variable),
  415. (r'\"?\w+\"?(?:\.\"?\w+\"?)?', Name.Variable),
  416. (r'\'\S*\'', Name.Variable),
  417. # if we encounter a comma, another object is listed
  418. (r',\n', Punctuation, 'object_name'),
  419. (r',', Punctuation, 'object_name'),
  420. # special case: "*SELECT*"
  421. (r'"\*SELECT\*( \d+)?"(.\w+)?', Name.Variable),
  422. (r'"\*VALUES\*(_\d+)?"(.\w+)?', Name.Variable),
  423. (r'"ANY_subquery"', Name.Variable),
  424. # Variable $1 ...
  425. (r'\$\d+', Name.Variable),
  426. # cast
  427. (r'::\w+', Name.Variable),
  428. (r' +', Whitespace),
  429. (r'"', Punctuation),
  430. (r'\[\.\.\.\]', Punctuation),
  431. (r'\)', Punctuation, '#pop'),
  432. ],
  433. 'predicate': [
  434. # if predicate is parenthesized, mark paren as punctuation
  435. (r'(\()([^\n]*)(\))', bygroups(Punctuation, Name.Variable, Punctuation), '#pop'),
  436. # otherwise color until newline
  437. (r'[^\n]*', Name.Variable, '#pop'),
  438. ],
  439. 'instrumentation': [
  440. (r'=|\.\.', Punctuation),
  441. (r' +', Whitespace),
  442. (r'(rows|width|time|loops)', Name.Class),
  443. (r'\d+\.\d+', Number.Float),
  444. (r'(\d+)', Number.Integer),
  445. (r'\)', Punctuation, '#pop'),
  446. ],
  447. 'conflict': [
  448. (r'(Resolution: )(\w+)', bygroups(Comment.Preproc, Name.Variable)),
  449. (r'(Arbiter \w+:)', Comment.Preproc, 'object_name'),
  450. (r'(Filter: )', Comment.Preproc, 'predicate'),
  451. ],
  452. 'setting': [
  453. (r'([a-z_]*?)(\s*)(=)(\s*)(\'.*?\')', bygroups(Name.Attribute, Whitespace, Operator, Whitespace, String)),
  454. (r'\, ', Punctuation),
  455. ],
  456. 'init_plan': [
  457. (r'\(', Punctuation),
  458. (r'returns \$\d+(,\$\d+)?', Name.Variable),
  459. (r'\)', Punctuation, '#pop'),
  460. ],
  461. 'sort': [
  462. (r':|kB', Punctuation),
  463. (r'(quicksort|top-N|heapsort|Average|Memory|Peak)', Comment.Prepoc),
  464. (r'(external|merge|Disk|sort)', Name.Exception),
  465. (r'(\d+)', Number.Integer),
  466. (r' +', Whitespace),
  467. ],
  468. }
  469. class SqlLexer(RegexLexer):
  470. """
  471. Lexer for Structured Query Language. Currently, this lexer does
  472. not recognize any special syntax except ANSI SQL.
  473. """
  474. name = 'SQL'
  475. aliases = ['sql']
  476. filenames = ['*.sql']
  477. mimetypes = ['text/x-sql']
  478. url = 'https://en.wikipedia.org/wiki/SQL'
  479. version_added = ''
  480. flags = re.IGNORECASE
  481. tokens = {
  482. 'root': [
  483. (r'\s+', Whitespace),
  484. (r'--.*\n?', Comment.Single),
  485. (r'/\*', Comment.Multiline, 'multiline-comments'),
  486. (words(_sql_builtins.KEYWORDS, suffix=r'\b'), Keyword),
  487. (words(_sql_builtins.DATATYPES, suffix=r'\b'), Name.Builtin),
  488. (r'[+*/<>=~!@#%^&|`?-]', Operator),
  489. (r'[0-9]+', Number.Integer),
  490. # TODO: Backslash escapes?
  491. (r"'(''|[^'])*'", String.Single),
  492. (r'"(""|[^"])*"', String.Symbol), # not a real string literal in ANSI SQL
  493. (r'[a-z_][\w$]*', Name), # allow $s in strings for Oracle
  494. (r'[;:()\[\],.]', Punctuation)
  495. ],
  496. 'multiline-comments': [
  497. (r'/\*', Comment.Multiline, 'multiline-comments'),
  498. (r'\*/', Comment.Multiline, '#pop'),
  499. (r'[^/*]+', Comment.Multiline),
  500. (r'[/*]', Comment.Multiline)
  501. ]
  502. }
  503. def analyse_text(self, text):
  504. return
  505. class TransactSqlLexer(RegexLexer):
  506. """
  507. Transact-SQL (T-SQL) is Microsoft's and Sybase's proprietary extension to
  508. SQL.
  509. The list of keywords includes ODBC and keywords reserved for future use.
  510. """
  511. name = 'Transact-SQL'
  512. aliases = ['tsql', 't-sql']
  513. filenames = ['*.sql']
  514. mimetypes = ['text/x-tsql']
  515. url = 'https://www.tsql.info'
  516. version_added = ''
  517. flags = re.IGNORECASE
  518. tokens = {
  519. 'root': [
  520. (r'\s+', Whitespace),
  521. (r'--.*[$|\n]?', Comment.Single),
  522. (r'/\*', Comment.Multiline, 'multiline-comments'),
  523. (words(_tsql_builtins.OPERATORS), Operator),
  524. (words(_tsql_builtins.OPERATOR_WORDS, suffix=r'\b'), Operator.Word),
  525. (words(_tsql_builtins.TYPES, suffix=r'\b'), Name.Class),
  526. (words(_tsql_builtins.FUNCTIONS, suffix=r'\b'), Name.Function),
  527. (r'(goto)(\s+)(\w+\b)', bygroups(Keyword, Whitespace, Name.Label)),
  528. (words(_tsql_builtins.KEYWORDS, suffix=r'\b'), Keyword),
  529. (r'(\[)([^]]+)(\])', bygroups(Operator, Name, Operator)),
  530. (r'0x[0-9a-f]+', Number.Hex),
  531. # Float variant 1, for example: 1., 1.e2, 1.2e3
  532. (r'[0-9]+\.[0-9]*(e[+-]?[0-9]+)?', Number.Float),
  533. # Float variant 2, for example: .1, .1e2
  534. (r'\.[0-9]+(e[+-]?[0-9]+)?', Number.Float),
  535. # Float variant 3, for example: 123e45
  536. (r'[0-9]+e[+-]?[0-9]+', Number.Float),
  537. (r'[0-9]+', Number.Integer),
  538. (r"'(''|[^'])*'", String.Single),
  539. (r'"(""|[^"])*"', String.Symbol),
  540. (r'[;(),.]', Punctuation),
  541. # Below we use \w even for the first "real" character because
  542. # tokens starting with a digit have already been recognized
  543. # as Number above.
  544. (r'@@\w+', Name.Builtin),
  545. (r'@\w+', Name.Variable),
  546. (r'(\w+)(:)', bygroups(Name.Label, Punctuation)),
  547. (r'#?#?\w+', Name), # names for temp tables and anything else
  548. (r'\?', Name.Variable.Magic), # parameter for prepared statements
  549. ],
  550. 'multiline-comments': [
  551. (r'/\*', Comment.Multiline, 'multiline-comments'),
  552. (r'\*/', Comment.Multiline, '#pop'),
  553. (r'[^/*]+', Comment.Multiline),
  554. (r'[/*]', Comment.Multiline)
  555. ]
  556. }
  557. def analyse_text(text):
  558. rating = 0
  559. if tsql_declare_re.search(text):
  560. # Found T-SQL variable declaration.
  561. rating = 1.0
  562. else:
  563. name_between_backtick_count = len(
  564. name_between_backtick_re.findall(text))
  565. name_between_bracket_count = len(
  566. name_between_bracket_re.findall(text))
  567. # We need to check if there are any names using
  568. # backticks or brackets, as otherwise both are 0
  569. # and 0 >= 2 * 0, so we would always assume it's true
  570. dialect_name_count = name_between_backtick_count + name_between_bracket_count
  571. if dialect_name_count >= 1 and \
  572. name_between_bracket_count >= 2 * name_between_backtick_count:
  573. # Found at least twice as many [name] as `name`.
  574. rating += 0.5
  575. elif name_between_bracket_count > name_between_backtick_count:
  576. rating += 0.2
  577. elif name_between_bracket_count > 0:
  578. rating += 0.1
  579. if tsql_variable_re.search(text) is not None:
  580. rating += 0.1
  581. if tsql_go_re.search(text) is not None:
  582. rating += 0.1
  583. return rating
  584. class MySqlLexer(RegexLexer):
  585. """The Oracle MySQL lexer.
  586. This lexer does not attempt to maintain strict compatibility with
  587. MariaDB syntax or keywords. Although MySQL and MariaDB's common code
  588. history suggests there may be significant overlap between the two,
  589. compatibility between the two is not a target for this lexer.
  590. """
  591. name = 'MySQL'
  592. aliases = ['mysql']
  593. mimetypes = ['text/x-mysql']
  594. url = 'https://www.mysql.com'
  595. version_added = ''
  596. flags = re.IGNORECASE
  597. tokens = {
  598. 'root': [
  599. (r'\s+', Whitespace),
  600. # Comments
  601. (r'(?:#|--\s+).*', Comment.Single),
  602. (r'/\*\+', Comment.Special, 'optimizer-hints'),
  603. (r'/\*', Comment.Multiline, 'multiline-comment'),
  604. # Hexadecimal literals
  605. (r"x'([0-9a-f]{2})+'", Number.Hex), # MySQL requires paired hex characters in this form.
  606. (r'0x[0-9a-f]+', Number.Hex),
  607. # Binary literals
  608. (r"b'[01]+'", Number.Bin),
  609. (r'0b[01]+', Number.Bin),
  610. # Numeric literals
  611. (r'[0-9]+\.[0-9]*(e[+-]?[0-9]+)?', Number.Float), # Mandatory integer, optional fraction and exponent
  612. (r'[0-9]*\.[0-9]+(e[+-]?[0-9]+)?', Number.Float), # Mandatory fraction, optional integer and exponent
  613. (r'[0-9]+e[+-]?[0-9]+', Number.Float), # Exponents with integer significands are still floats
  614. (r'[0-9]+(?=[^0-9a-z$_\u0080-\uffff])', Number.Integer), # Integers that are not in a schema object name
  615. # Date literals
  616. (r"\{\s*d\s*(?P<quote>['\"])\s*\d{2}(\d{2})?.?\d{2}.?\d{2}\s*(?P=quote)\s*\}",
  617. Literal.Date),
  618. # Time literals
  619. (r"\{\s*t\s*(?P<quote>['\"])\s*(?:\d+\s+)?\d{1,2}.?\d{1,2}.?\d{1,2}(\.\d*)?\s*(?P=quote)\s*\}",
  620. Literal.Date),
  621. # Timestamp literals
  622. (
  623. r"\{\s*ts\s*(?P<quote>['\"])\s*"
  624. r"\d{2}(?:\d{2})?.?\d{2}.?\d{2}" # Date part
  625. r"\s+" # Whitespace between date and time
  626. r"\d{1,2}.?\d{1,2}.?\d{1,2}(\.\d*)?" # Time part
  627. r"\s*(?P=quote)\s*\}",
  628. Literal.Date
  629. ),
  630. # String literals
  631. (r"'", String.Single, 'single-quoted-string'),
  632. (r'"', String.Double, 'double-quoted-string'),
  633. # Variables
  634. (r'@@(?:global\.|persist\.|persist_only\.|session\.)?[a-z_]+', Name.Variable),
  635. (r'@[a-z0-9_$.]+', Name.Variable),
  636. (r"@'", Name.Variable, 'single-quoted-variable'),
  637. (r'@"', Name.Variable, 'double-quoted-variable'),
  638. (r"@`", Name.Variable, 'backtick-quoted-variable'),
  639. (r'\?', Name.Variable), # For demonstrating prepared statements
  640. # Operators
  641. (r'[!%&*+/:<=>^|~-]+', Operator),
  642. # Exceptions; these words tokenize differently in different contexts.
  643. (r'\b(set)(?!\s*\()', Keyword),
  644. (r'\b(character)(\s+)(set)\b', bygroups(Keyword, Whitespace, Keyword)),
  645. # In all other known cases, "SET" is tokenized by MYSQL_DATATYPES.
  646. (words(_mysql_builtins.MYSQL_CONSTANTS, prefix=r'\b', suffix=r'\b'),
  647. Name.Constant),
  648. (words(_mysql_builtins.MYSQL_DATATYPES, prefix=r'\b', suffix=r'\b'),
  649. Keyword.Type),
  650. (words(_mysql_builtins.MYSQL_KEYWORDS, prefix=r'\b', suffix=r'\b'),
  651. Keyword),
  652. (words(_mysql_builtins.MYSQL_FUNCTIONS, prefix=r'\b', suffix=r'\b(\s*)(\()'),
  653. bygroups(Name.Function, Whitespace, Punctuation)),
  654. # Schema object names
  655. #
  656. # Note: Although the first regex supports unquoted all-numeric
  657. # identifiers, this will not be a problem in practice because
  658. # numeric literals have already been handled above.
  659. #
  660. ('[0-9a-z$_\u0080-\uffff]+', Name),
  661. (r'`', Name.Quoted, 'schema-object-name'),
  662. # Punctuation
  663. (r'[(),.;]', Punctuation),
  664. ],
  665. # Multiline comment substates
  666. # ---------------------------
  667. 'optimizer-hints': [
  668. (r'[^*a-z]+', Comment.Special),
  669. (r'\*/', Comment.Special, '#pop'),
  670. (words(_mysql_builtins.MYSQL_OPTIMIZER_HINTS, suffix=r'\b'),
  671. Comment.Preproc),
  672. ('[a-z]+', Comment.Special),
  673. (r'\*', Comment.Special),
  674. ],
  675. 'multiline-comment': [
  676. (r'[^*]+', Comment.Multiline),
  677. (r'\*/', Comment.Multiline, '#pop'),
  678. (r'\*', Comment.Multiline),
  679. ],
  680. # String substates
  681. # ----------------
  682. 'single-quoted-string': [
  683. (r"[^'\\]+", String.Single),
  684. (r"''", String.Escape),
  685. (r"""\\[0'"bnrtZ\\%_]""", String.Escape),
  686. (r"'", String.Single, '#pop'),
  687. ],
  688. 'double-quoted-string': [
  689. (r'[^"\\]+', String.Double),
  690. (r'""', String.Escape),
  691. (r"""\\[0'"bnrtZ\\%_]""", String.Escape),
  692. (r'"', String.Double, '#pop'),
  693. ],
  694. # Variable substates
  695. # ------------------
  696. 'single-quoted-variable': [
  697. (r"[^']+", Name.Variable),
  698. (r"''", Name.Variable),
  699. (r"'", Name.Variable, '#pop'),
  700. ],
  701. 'double-quoted-variable': [
  702. (r'[^"]+', Name.Variable),
  703. (r'""', Name.Variable),
  704. (r'"', Name.Variable, '#pop'),
  705. ],
  706. 'backtick-quoted-variable': [
  707. (r'[^`]+', Name.Variable),
  708. (r'``', Name.Variable),
  709. (r'`', Name.Variable, '#pop'),
  710. ],
  711. # Schema object name substates
  712. # ----------------------------
  713. #
  714. # "Name.Quoted" and "Name.Quoted.Escape" are non-standard but
  715. # formatters will style them as "Name" by default but add
  716. # additional styles based on the token name. This gives users
  717. # flexibility to add custom styles as desired.
  718. #
  719. 'schema-object-name': [
  720. (r'[^`]+', Name.Quoted),
  721. (r'``', Name.Quoted.Escape),
  722. (r'`', Name.Quoted, '#pop'),
  723. ],
  724. }
  725. def analyse_text(text):
  726. rating = 0
  727. name_between_backtick_count = len(
  728. name_between_backtick_re.findall(text))
  729. name_between_bracket_count = len(
  730. name_between_bracket_re.findall(text))
  731. # Same logic as above in the TSQL analysis
  732. dialect_name_count = name_between_backtick_count + name_between_bracket_count
  733. if dialect_name_count >= 1 and \
  734. name_between_backtick_count >= 2 * name_between_bracket_count:
  735. # Found at least twice as many `name` as [name].
  736. rating += 0.5
  737. elif name_between_backtick_count > name_between_bracket_count:
  738. rating += 0.2
  739. elif name_between_backtick_count > 0:
  740. rating += 0.1
  741. return rating
  742. class GoogleSqlLexer(RegexLexer):
  743. """
  744. GoogleSQL is Google's standard SQL dialect, formerly known as ZetaSQL.
  745. The list of keywords includes reserved words for future use.
  746. """
  747. name = 'GoogleSQL'
  748. aliases = ['googlesql', 'zetasql']
  749. filenames = ['*.googlesql', '*.googlesql.sql']
  750. mimetypes = ['text/x-google-sql', 'text/x-google-sql-aux']
  751. url = 'https://cloud.google.com/bigquery/googlesql'
  752. version_added = '2.19'
  753. flags = re.IGNORECASE
  754. tokens = {
  755. 'root': [
  756. (r'\s+', Whitespace),
  757. # Comments
  758. (r'(?:#|--\s+).*', Comment.Single),
  759. (r'/\*', Comment.Multiline, 'multiline-comment'),
  760. # Hexadecimal literals
  761. (r"x'([0-9a-f]{2})+'", Number.Hex),
  762. (r'0x[0-9a-f]+', Number.Hex),
  763. # Binary literals
  764. (r"b'[01]+'", Number.Bin),
  765. (r'0b[01]+', Number.Bin),
  766. # Numeric literals
  767. (r'[0-9]+\.[0-9]*(e[+-]?[0-9]+)?', Number.Float), # Mandatory integer, optional fraction and exponent
  768. (r'[0-9]*\.[0-9]+(e[+-]?[0-9]+)?', Number.Float), # Mandatory fraction, optional integer and exponent
  769. (r'[0-9]+e[+-]?[0-9]+', Number.Float), # Exponents with integer significands are still floats
  770. (r'[0-9]+(?=[^0-9a-z$_\u0080-\uffff])', Number.Integer), # Integers that are not in a schema object name
  771. # Date literals
  772. (r"\{\s*d\s*(?P<quote>['\"])\s*\d{2}(\d{2})?.?\d{2}.?\d{2}\s*(?P=quote)\s*\}",
  773. Literal.Date),
  774. # Time literals
  775. (r"\{\s*t\s*(?P<quote>['\"])\s*(?:\d+\s+)?\d{1,2}.?\d{1,2}.?\d{1,2}(\.\d*)?\s*(?P=quote)\s*\}",
  776. Literal.Date),
  777. # Timestamp literals
  778. (
  779. r"\{\s*ts\s*(?P<quote>['\"])\s*"
  780. r"\d{2}(?:\d{2})?.?\d{2}.?\d{2}" # Date part
  781. r"\s+" # Whitespace between date and time
  782. r"\d{1,2}.?\d{1,2}.?\d{1,2}(\.\d*)?" # Time part
  783. r"\s*(?P=quote)\s*\}",
  784. Literal.Date
  785. ),
  786. # String literals
  787. (r"'", String.Single, 'single-quoted-string'),
  788. (r'"', String.Double, 'double-quoted-string'),
  789. # Variables
  790. (r'@@(?:global\.|persist\.|persist_only\.|session\.)?[a-z_]+', Name.Variable),
  791. (r'@[a-z0-9_$.]+', Name.Variable),
  792. (r"@'", Name.Variable, 'single-quoted-variable'),
  793. (r'@"', Name.Variable, 'double-quoted-variable'),
  794. (r"@`", Name.Variable, 'backtick-quoted-variable'),
  795. (r'\?', Name.Variable), # For demonstrating prepared statements
  796. # Exceptions; these words tokenize differently in different contexts.
  797. (r'\b(set)(?!\s*\()', Keyword),
  798. (r'\b(character)(\s+)(set)\b', bygroups(Keyword, Whitespace, Keyword)),
  799. # Constants, types, keywords, functions, operators
  800. (words(_googlesql_builtins.constants, prefix=r'\b', suffix=r'\b'), Name.Constant),
  801. (words(_googlesql_builtins.types, prefix=r'\b', suffix=r'\b'), Keyword.Type),
  802. (words(_googlesql_builtins.keywords, prefix=r'\b', suffix=r'\b'), Keyword),
  803. (words(_googlesql_builtins.functionnames, prefix=r'\b', suffix=r'\b(\s*)(\()'),
  804. bygroups(Name.Function, Whitespace, Punctuation)),
  805. (words(_googlesql_builtins.operators, prefix=r'\b', suffix=r'\b'), Operator),
  806. # Schema object names
  807. #
  808. # Note: Although the first regex supports unquoted all-numeric
  809. # identifiers, this will not be a problem in practice because
  810. # numeric literals have already been handled above.
  811. #
  812. ('[0-9a-z$_\u0080-\uffff]+', Name),
  813. (r'`', Name.Quoted, 'schema-object-name'),
  814. # Punctuation
  815. (r'[(),.;]', Punctuation),
  816. ],
  817. # Multiline comment substates
  818. # ---------------------------
  819. 'multiline-comment': [
  820. (r'[^*]+', Comment.Multiline),
  821. (r'\*/', Comment.Multiline, '#pop'),
  822. (r'\*', Comment.Multiline),
  823. ],
  824. # String substates
  825. # ----------------
  826. 'single-quoted-string': [
  827. (r"[^'\\]+", String.Single),
  828. (r"''", String.Escape),
  829. (r"""\\[0'"bnrtZ\\%_]""", String.Escape),
  830. (r"'", String.Single, '#pop'),
  831. ],
  832. 'double-quoted-string': [
  833. (r'[^"\\]+', String.Double),
  834. (r'""', String.Escape),
  835. (r"""\\[0'"bnrtZ\\%_]""", String.Escape),
  836. (r'"', String.Double, '#pop'),
  837. ],
  838. # Variable substates
  839. # ------------------
  840. 'single-quoted-variable': [
  841. (r"[^']+", Name.Variable),
  842. (r"''", Name.Variable),
  843. (r"'", Name.Variable, '#pop'),
  844. ],
  845. 'double-quoted-variable': [
  846. (r'[^"]+', Name.Variable),
  847. (r'""', Name.Variable),
  848. (r'"', Name.Variable, '#pop'),
  849. ],
  850. 'backtick-quoted-variable': [
  851. (r'[^`]+', Name.Variable),
  852. (r'``', Name.Variable),
  853. (r'`', Name.Variable, '#pop'),
  854. ],
  855. # Schema object name substates
  856. # ----------------------------
  857. #
  858. # "Name.Quoted" and "Name.Quoted.Escape" are non-standard but
  859. # formatters will style them as "Name" by default but add
  860. # additional styles based on the token name. This gives users
  861. # flexibility to add custom styles as desired.
  862. #
  863. 'schema-object-name': [
  864. (r'[^`]+', Name.Quoted),
  865. (r'``', Name.Quoted.Escape),
  866. (r'`', Name.Quoted, '#pop'),
  867. ],
  868. }
  869. def analyse_text(text):
  870. tokens = collections.Counter(text.split())
  871. return 0.001 * sum(count for t, count in tokens.items()
  872. if t in googlesql_identifiers)
  873. class SqliteConsoleLexer(Lexer):
  874. """
  875. Lexer for example sessions using sqlite3.
  876. """
  877. name = 'sqlite3con'
  878. aliases = ['sqlite3']
  879. filenames = ['*.sqlite3-console']
  880. mimetypes = ['text/x-sqlite3-console']
  881. url = 'https://www.sqlite.org'
  882. version_added = '0.11'
  883. _example = "sqlite3/sqlite3.sqlite3-console"
  884. def get_tokens_unprocessed(self, data):
  885. sql = SqlLexer(**self.options)
  886. curcode = ''
  887. insertions = []
  888. for match in line_re.finditer(data):
  889. line = match.group()
  890. prompt_match = sqlite_prompt_re.match(line)
  891. if prompt_match is not None:
  892. insertions.append((len(curcode),
  893. [(0, Generic.Prompt, line[:7])]))
  894. insertions.append((len(curcode),
  895. [(7, Whitespace, ' ')]))
  896. curcode += line[8:]
  897. else:
  898. if curcode:
  899. yield from do_insertions(insertions,
  900. sql.get_tokens_unprocessed(curcode))
  901. curcode = ''
  902. insertions = []
  903. if line.startswith('SQL error: '):
  904. yield (match.start(), Generic.Traceback, line)
  905. else:
  906. yield (match.start(), Generic.Output, line)
  907. if curcode:
  908. yield from do_insertions(insertions,
  909. sql.get_tokens_unprocessed(curcode))
  910. class RqlLexer(RegexLexer):
  911. """
  912. Lexer for Relation Query Language.
  913. """
  914. name = 'RQL'
  915. url = 'http://www.logilab.org/project/rql'
  916. aliases = ['rql']
  917. filenames = ['*.rql']
  918. mimetypes = ['text/x-rql']
  919. version_added = '2.0'
  920. flags = re.IGNORECASE
  921. tokens = {
  922. 'root': [
  923. (r'\s+', Whitespace),
  924. (r'(DELETE|SET|INSERT|UNION|DISTINCT|WITH|WHERE|BEING|OR'
  925. r'|AND|NOT|GROUPBY|HAVING|ORDERBY|ASC|DESC|LIMIT|OFFSET'
  926. r'|TODAY|NOW|TRUE|FALSE|NULL|EXISTS)\b', Keyword),
  927. (r'[+*/<>=%-]', Operator),
  928. (r'(Any|is|instance_of|CWEType|CWRelation)\b', Name.Builtin),
  929. (r'[0-9]+', Number.Integer),
  930. (r'[A-Z_]\w*\??', Name),
  931. (r"'(''|[^'])*'", String.Single),
  932. (r'"(""|[^"])*"', String.Single),
  933. (r'[;:()\[\],.]', Punctuation)
  934. ],
  935. }