markup.py 64 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659
  1. """
  2. pygments.lexers.markup
  3. ~~~~~~~~~~~~~~~~~~~~~~
  4. Lexers for non-HTML markup languages.
  5. :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
  6. :license: BSD, see LICENSE for details.
  7. """
  8. import re
  9. from pygments.lexers.html import XmlLexer
  10. from pygments.lexers.javascript import JavascriptLexer
  11. from pygments.lexers.css import CssLexer
  12. from pygments.lexers.lilypond import LilyPondLexer
  13. from pygments.lexers.data import JsonLexer
  14. from pygments.lexer import RegexLexer, DelegatingLexer, include, bygroups, \
  15. using, this, do_insertions, default, words
  16. from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
  17. Number, Punctuation, Generic, Other, Whitespace, Literal
  18. from pygments.util import get_bool_opt, ClassNotFound
  19. __all__ = ['BBCodeLexer', 'MoinWikiLexer', 'RstLexer', 'TexLexer', 'GroffLexer',
  20. 'MozPreprocHashLexer', 'MozPreprocPercentLexer',
  21. 'MozPreprocXulLexer', 'MozPreprocJavascriptLexer',
  22. 'MozPreprocCssLexer', 'MarkdownLexer', 'OrgLexer', 'TiddlyWiki5Lexer',
  23. 'WikitextLexer']
  24. class BBCodeLexer(RegexLexer):
  25. """
  26. A lexer that highlights BBCode(-like) syntax.
  27. """
  28. name = 'BBCode'
  29. aliases = ['bbcode']
  30. mimetypes = ['text/x-bbcode']
  31. url = 'https://www.bbcode.org/'
  32. version_added = '0.6'
  33. tokens = {
  34. 'root': [
  35. (r'[^[]+', Text),
  36. # tag/end tag begin
  37. (r'\[/?\w+', Keyword, 'tag'),
  38. # stray bracket
  39. (r'\[', Text),
  40. ],
  41. 'tag': [
  42. (r'\s+', Text),
  43. # attribute with value
  44. (r'(\w+)(=)("?[^\s"\]]+"?)',
  45. bygroups(Name.Attribute, Operator, String)),
  46. # tag argument (a la [color=green])
  47. (r'(=)("?[^\s"\]]+"?)',
  48. bygroups(Operator, String)),
  49. # tag end
  50. (r'\]', Keyword, '#pop'),
  51. ],
  52. }
  53. class MoinWikiLexer(RegexLexer):
  54. """
  55. For MoinMoin (and Trac) Wiki markup.
  56. """
  57. name = 'MoinMoin/Trac Wiki markup'
  58. aliases = ['trac-wiki', 'moin']
  59. filenames = []
  60. mimetypes = ['text/x-trac-wiki']
  61. url = 'https://moinmo.in'
  62. version_added = '0.7'
  63. flags = re.MULTILINE | re.IGNORECASE
  64. tokens = {
  65. 'root': [
  66. (r'^#.*$', Comment),
  67. (r'(!)(\S+)', bygroups(Keyword, Text)), # Ignore-next
  68. # Titles
  69. (r'^(=+)([^=]+)(=+)(\s*#.+)?$',
  70. bygroups(Generic.Heading, using(this), Generic.Heading, String)),
  71. # Literal code blocks, with optional shebang
  72. (r'(\{\{\{)(\n#!.+)?', bygroups(Name.Builtin, Name.Namespace), 'codeblock'),
  73. (r'(\'\'\'?|\|\||`|__|~~|\^|,,|::)', Comment), # Formatting
  74. # Lists
  75. (r'^( +)([.*-])( )', bygroups(Text, Name.Builtin, Text)),
  76. (r'^( +)([a-z]{1,5}\.)( )', bygroups(Text, Name.Builtin, Text)),
  77. # Other Formatting
  78. (r'\[\[\w+.*?\]\]', Keyword), # Macro
  79. (r'(\[[^\s\]]+)(\s+[^\]]+?)?(\])',
  80. bygroups(Keyword, String, Keyword)), # Link
  81. (r'^----+$', Keyword), # Horizontal rules
  82. (r'[^\n\'\[{!_~^,|]+', Text),
  83. (r'\n', Text),
  84. (r'.', Text),
  85. ],
  86. 'codeblock': [
  87. (r'\}\}\}', Name.Builtin, '#pop'),
  88. # these blocks are allowed to be nested in Trac, but not MoinMoin
  89. (r'\{\{\{', Text, '#push'),
  90. (r'[^{}]+', Comment.Preproc), # slurp boring text
  91. (r'.', Comment.Preproc), # allow loose { or }
  92. ],
  93. }
  94. class RstLexer(RegexLexer):
  95. """
  96. For reStructuredText markup.
  97. Additional options accepted:
  98. `handlecodeblocks`
  99. Highlight the contents of ``.. sourcecode:: language``,
  100. ``.. code:: language`` and ``.. code-block:: language``
  101. directives with a lexer for the given language (default:
  102. ``True``).
  103. .. versionadded:: 0.8
  104. """
  105. name = 'reStructuredText'
  106. url = 'https://docutils.sourceforge.io/rst.html'
  107. aliases = ['restructuredtext', 'rst', 'rest']
  108. filenames = ['*.rst', '*.rest']
  109. mimetypes = ["text/x-rst", "text/prs.fallenstein.rst"]
  110. version_added = '0.7'
  111. flags = re.MULTILINE
  112. def _handle_sourcecode(self, match):
  113. from pygments.lexers import get_lexer_by_name
  114. # section header
  115. yield match.start(1), Punctuation, match.group(1)
  116. yield match.start(2), Text, match.group(2)
  117. yield match.start(3), Operator.Word, match.group(3)
  118. yield match.start(4), Punctuation, match.group(4)
  119. yield match.start(5), Text, match.group(5)
  120. yield match.start(6), Keyword, match.group(6)
  121. yield match.start(7), Text, match.group(7)
  122. # lookup lexer if wanted and existing
  123. lexer = None
  124. if self.handlecodeblocks:
  125. try:
  126. lexer = get_lexer_by_name(match.group(6).strip())
  127. except ClassNotFound:
  128. pass
  129. indention = match.group(8)
  130. indention_size = len(indention)
  131. code = (indention + match.group(9) + match.group(10) + match.group(11))
  132. # no lexer for this language. handle it like it was a code block
  133. if lexer is None:
  134. yield match.start(8), String, code
  135. return
  136. # highlight the lines with the lexer.
  137. ins = []
  138. codelines = code.splitlines(True)
  139. code = ''
  140. for line in codelines:
  141. if len(line) > indention_size:
  142. ins.append((len(code), [(0, Text, line[:indention_size])]))
  143. code += line[indention_size:]
  144. else:
  145. code += line
  146. yield from do_insertions(ins, lexer.get_tokens_unprocessed(code))
  147. # from docutils.parsers.rst.states
  148. closers = '\'")]}>\u2019\u201d\xbb!?'
  149. unicode_delimiters = '\u2010\u2011\u2012\u2013\u2014\u00a0'
  150. end_string_suffix = (rf'((?=$)|(?=[-/:.,; \n\x00{re.escape(unicode_delimiters)}{re.escape(closers)}]))')
  151. tokens = {
  152. 'root': [
  153. # Heading with overline
  154. (r'^(=+|-+|`+|:+|\.+|\'+|"+|~+|\^+|_+|\*+|\++|#+)([ \t]*\n)'
  155. r'(.+)(\n)(\1)(\n)',
  156. bygroups(Generic.Heading, Text, Generic.Heading,
  157. Text, Generic.Heading, Text)),
  158. # Plain heading
  159. (r'^(\S.*)(\n)(={3,}|-{3,}|`{3,}|:{3,}|\.{3,}|\'{3,}|"{3,}|'
  160. r'~{3,}|\^{3,}|_{3,}|\*{3,}|\+{3,}|#{3,})(\n)',
  161. bygroups(Generic.Heading, Text, Generic.Heading, Text)),
  162. # Bulleted lists
  163. (r'^(\s*)([-*+])( .+\n(?:\1 .+\n)*)',
  164. bygroups(Text, Number, using(this, state='inline'))),
  165. # Numbered lists
  166. (r'^(\s*)([0-9#ivxlcmIVXLCM]+\.)( .+\n(?:\1 .+\n)*)',
  167. bygroups(Text, Number, using(this, state='inline'))),
  168. (r'^(\s*)(\(?[0-9#ivxlcmIVXLCM]+\))( .+\n(?:\1 .+\n)*)',
  169. bygroups(Text, Number, using(this, state='inline'))),
  170. # Numbered, but keep words at BOL from becoming lists
  171. (r'^(\s*)([A-Z]+\.)( .+\n(?:\1 .+\n)+)',
  172. bygroups(Text, Number, using(this, state='inline'))),
  173. (r'^(\s*)(\(?[A-Za-z]+\))( .+\n(?:\1 .+\n)+)',
  174. bygroups(Text, Number, using(this, state='inline'))),
  175. # Line blocks
  176. (r'^(\s*)(\|)( .+\n(?:\| .+\n)*)',
  177. bygroups(Text, Operator, using(this, state='inline'))),
  178. # Sourcecode directives
  179. (r'^( *\.\.)(\s*)((?:source)?code(?:-block)?)(::)([ \t]*)([^\n]+)'
  180. r'(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\8.*)?\n)+)',
  181. _handle_sourcecode),
  182. # A directive
  183. (r'^( *\.\.)(\s*)([\w:-]+?)(::)(?:([ \t]*)(.*))',
  184. bygroups(Punctuation, Text, Operator.Word, Punctuation, Text,
  185. using(this, state='inline'))),
  186. # A reference target
  187. (r'^( *\.\.)(\s*)(_(?:[^:\\]|\\.)+:)(.*?)$',
  188. bygroups(Punctuation, Text, Name.Tag, using(this, state='inline'))),
  189. # A footnote/citation target
  190. (r'^( *\.\.)(\s*)(\[.+\])(.*?)$',
  191. bygroups(Punctuation, Text, Name.Tag, using(this, state='inline'))),
  192. # A substitution def
  193. (r'^( *\.\.)(\s*)(\|.+\|)(\s*)([\w:-]+?)(::)(?:([ \t]*)(.*))',
  194. bygroups(Punctuation, Text, Name.Tag, Text, Operator.Word,
  195. Punctuation, Text, using(this, state='inline'))),
  196. # Comments
  197. (r'^ *\.\..*(\n( +.*\n|\n)+)?', Comment),
  198. # Field list marker
  199. (r'^( *)(:(?:\\\\|\\:|[^:\n])+:(?=\s))([ \t]*)',
  200. bygroups(Text, Name.Class, Text)),
  201. # Definition list
  202. (r'^(\S.*(?<!::)\n)((?:(?: +.*)\n)+)',
  203. bygroups(using(this, state='inline'), using(this, state='inline'))),
  204. # Code blocks
  205. (r'(::)(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\3.*)?\n)+)',
  206. bygroups(String.Escape, Text, String, String, Text, String)),
  207. include('inline'),
  208. ],
  209. 'inline': [
  210. (r'\\.', Text), # escape
  211. (r'``', String, 'literal'), # code
  212. (r'(`.+?)(<.+?>)(`__?)', # reference with inline target
  213. bygroups(String, String.Interpol, String)),
  214. (r'`.+?`__?', String), # reference
  215. (r'(`.+?`)(:[a-zA-Z0-9:-]+?:)?',
  216. bygroups(Name.Variable, Name.Attribute)), # role
  217. (r'(:[a-zA-Z0-9:-]+?:)(`.+?`)',
  218. bygroups(Name.Attribute, Name.Variable)), # role (content first)
  219. (r'\*\*.+?\*\*', Generic.Strong), # Strong emphasis
  220. (r'\*.+?\*', Generic.Emph), # Emphasis
  221. (r'\[.*?\]_', String), # Footnote or citation
  222. (r'<.+?>', Name.Tag), # Hyperlink
  223. (r'[^\\\n\[*`:]+', Text),
  224. (r'.', Text),
  225. ],
  226. 'literal': [
  227. (r'[^`]+', String),
  228. (r'``' + end_string_suffix, String, '#pop'),
  229. (r'`', String),
  230. ]
  231. }
  232. def __init__(self, **options):
  233. self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True)
  234. RegexLexer.__init__(self, **options)
  235. def analyse_text(text):
  236. if text[:2] == '..' and text[2:3] != '.':
  237. return 0.3
  238. p1 = text.find("\n")
  239. p2 = text.find("\n", p1 + 1)
  240. if (p2 > -1 and # has two lines
  241. p1 * 2 + 1 == p2 and # they are the same length
  242. text[p1+1] in '-=' and # the next line both starts and ends with
  243. text[p1+1] == text[p2-1]): # ...a sufficiently high header
  244. return 0.5
  245. class TexLexer(RegexLexer):
  246. """
  247. Lexer for the TeX and LaTeX typesetting languages.
  248. """
  249. name = 'TeX'
  250. aliases = ['tex', 'latex']
  251. filenames = ['*.tex', '*.aux', '*.toc']
  252. mimetypes = ['text/x-tex', 'text/x-latex']
  253. url = 'https://tug.org'
  254. version_added = ''
  255. tokens = {
  256. 'general': [
  257. (r'%.*?\n', Comment),
  258. (r'[{}]', Name.Builtin),
  259. (r'[&_^]', Name.Builtin),
  260. ],
  261. 'root': [
  262. (r'\\\[', String.Backtick, 'displaymath'),
  263. (r'\\\(', String, 'inlinemath'),
  264. (r'\$\$', String.Backtick, 'displaymath'),
  265. (r'\$', String, 'inlinemath'),
  266. (r'\\([a-zA-Z@_:]+|\S?)', Keyword, 'command'),
  267. (r'\\$', Keyword),
  268. include('general'),
  269. (r'[^\\$%&_^{}]+', Text),
  270. ],
  271. 'math': [
  272. (r'\\([a-zA-Z]+|\S?)', Name.Variable),
  273. include('general'),
  274. (r'[0-9]+', Number),
  275. (r'[-=!+*/()\[\]]', Operator),
  276. (r'[^=!+*/()\[\]\\$%&_^{}0-9-]+', Name.Builtin),
  277. ],
  278. 'inlinemath': [
  279. (r'\\\)', String, '#pop'),
  280. (r'\$', String, '#pop'),
  281. include('math'),
  282. ],
  283. 'displaymath': [
  284. (r'\\\]', String, '#pop'),
  285. (r'\$\$', String, '#pop'),
  286. (r'\$', Name.Builtin),
  287. include('math'),
  288. ],
  289. 'command': [
  290. (r'\[.*?\]', Name.Attribute),
  291. (r'\*', Keyword),
  292. default('#pop'),
  293. ],
  294. }
  295. def analyse_text(text):
  296. for start in ("\\documentclass", "\\input", "\\documentstyle",
  297. "\\relax"):
  298. if text[:len(start)] == start:
  299. return True
  300. class GroffLexer(RegexLexer):
  301. """
  302. Lexer for the (g)roff typesetting language, supporting groff
  303. extensions. Mainly useful for highlighting manpage sources.
  304. """
  305. name = 'Groff'
  306. aliases = ['groff', 'nroff', 'man']
  307. filenames = ['*.[1-9]', '*.man', '*.1p', '*.3pm']
  308. mimetypes = ['application/x-troff', 'text/troff']
  309. url = 'https://www.gnu.org/software/groff'
  310. version_added = '0.6'
  311. tokens = {
  312. 'root': [
  313. (r'(\.)(\w+)', bygroups(Text, Keyword), 'request'),
  314. (r'\.', Punctuation, 'request'),
  315. # Regular characters, slurp till we find a backslash or newline
  316. (r'[^\\\n]+', Text, 'textline'),
  317. default('textline'),
  318. ],
  319. 'textline': [
  320. include('escapes'),
  321. (r'[^\\\n]+', Text),
  322. (r'\n', Text, '#pop'),
  323. ],
  324. 'escapes': [
  325. # groff has many ways to write escapes.
  326. (r'\\"[^\n]*', Comment),
  327. (r'\\[fn]\w', String.Escape),
  328. (r'\\\(.{2}', String.Escape),
  329. (r'\\.\[.*\]', String.Escape),
  330. (r'\\.', String.Escape),
  331. (r'\\\n', Text, 'request'),
  332. ],
  333. 'request': [
  334. (r'\n', Text, '#pop'),
  335. include('escapes'),
  336. (r'"[^\n"]+"', String.Double),
  337. (r'\d+', Number),
  338. (r'\S+', String),
  339. (r'\s+', Text),
  340. ],
  341. }
  342. def analyse_text(text):
  343. if text[:1] != '.':
  344. return False
  345. if text[:3] == '.\\"':
  346. return True
  347. if text[:4] == '.TH ':
  348. return True
  349. if text[1:3].isalnum() and text[3].isspace():
  350. return 0.9
  351. class MozPreprocHashLexer(RegexLexer):
  352. """
  353. Lexer for Mozilla Preprocessor files (with '#' as the marker).
  354. Other data is left untouched.
  355. """
  356. name = 'mozhashpreproc'
  357. aliases = [name]
  358. filenames = []
  359. mimetypes = []
  360. url = 'https://firefox-source-docs.mozilla.org/build/buildsystem/preprocessor.html'
  361. version_added = '2.0'
  362. tokens = {
  363. 'root': [
  364. (r'^#', Comment.Preproc, ('expr', 'exprstart')),
  365. (r'.+', Other),
  366. ],
  367. 'exprstart': [
  368. (r'(literal)(.*)', bygroups(Comment.Preproc, Text), '#pop:2'),
  369. (words((
  370. 'define', 'undef', 'if', 'ifdef', 'ifndef', 'else', 'elif',
  371. 'elifdef', 'elifndef', 'endif', 'expand', 'filter', 'unfilter',
  372. 'include', 'includesubst', 'error')),
  373. Comment.Preproc, '#pop'),
  374. ],
  375. 'expr': [
  376. (words(('!', '!=', '==', '&&', '||')), Operator),
  377. (r'(defined)(\()', bygroups(Keyword, Punctuation)),
  378. (r'\)', Punctuation),
  379. (r'[0-9]+', Number.Decimal),
  380. (r'__\w+?__', Name.Variable),
  381. (r'@\w+?@', Name.Class),
  382. (r'\w+', Name),
  383. (r'\n', Text, '#pop'),
  384. (r'\s+', Text),
  385. (r'\S', Punctuation),
  386. ],
  387. }
  388. class MozPreprocPercentLexer(MozPreprocHashLexer):
  389. """
  390. Lexer for Mozilla Preprocessor files (with '%' as the marker).
  391. Other data is left untouched.
  392. """
  393. name = 'mozpercentpreproc'
  394. aliases = [name]
  395. filenames = []
  396. mimetypes = []
  397. url = 'https://firefox-source-docs.mozilla.org/build/buildsystem/preprocessor.html'
  398. version_added = '2.0'
  399. tokens = {
  400. 'root': [
  401. (r'^%', Comment.Preproc, ('expr', 'exprstart')),
  402. (r'.+', Other),
  403. ],
  404. }
  405. class MozPreprocXulLexer(DelegatingLexer):
  406. """
  407. Subclass of the `MozPreprocHashLexer` that highlights unlexed data with the
  408. `XmlLexer`.
  409. """
  410. name = "XUL+mozpreproc"
  411. aliases = ['xul+mozpreproc']
  412. filenames = ['*.xul.in']
  413. mimetypes = []
  414. url = 'https://firefox-source-docs.mozilla.org/build/buildsystem/preprocessor.html'
  415. version_added = '2.0'
  416. def __init__(self, **options):
  417. super().__init__(XmlLexer, MozPreprocHashLexer, **options)
  418. class MozPreprocJavascriptLexer(DelegatingLexer):
  419. """
  420. Subclass of the `MozPreprocHashLexer` that highlights unlexed data with the
  421. `JavascriptLexer`.
  422. """
  423. name = "Javascript+mozpreproc"
  424. aliases = ['javascript+mozpreproc']
  425. filenames = ['*.js.in']
  426. mimetypes = []
  427. url = 'https://firefox-source-docs.mozilla.org/build/buildsystem/preprocessor.html'
  428. version_added = '2.0'
  429. def __init__(self, **options):
  430. super().__init__(JavascriptLexer, MozPreprocHashLexer, **options)
  431. class MozPreprocCssLexer(DelegatingLexer):
  432. """
  433. Subclass of the `MozPreprocHashLexer` that highlights unlexed data with the
  434. `CssLexer`.
  435. """
  436. name = "CSS+mozpreproc"
  437. aliases = ['css+mozpreproc']
  438. filenames = ['*.css.in']
  439. mimetypes = []
  440. url = 'https://firefox-source-docs.mozilla.org/build/buildsystem/preprocessor.html'
  441. version_added = '2.0'
  442. def __init__(self, **options):
  443. super().__init__(CssLexer, MozPreprocPercentLexer, **options)
  444. class MarkdownLexer(RegexLexer):
  445. """
  446. For Markdown markup.
  447. """
  448. name = 'Markdown'
  449. url = 'https://daringfireball.net/projects/markdown/'
  450. aliases = ['markdown', 'md']
  451. filenames = ['*.md', '*.markdown']
  452. mimetypes = ["text/x-markdown"]
  453. version_added = '2.2'
  454. flags = re.MULTILINE
  455. def _handle_codeblock(self, match):
  456. from pygments.lexers import get_lexer_by_name
  457. yield match.start('initial'), String.Backtick, match.group('initial')
  458. yield match.start('lang'), String.Backtick, match.group('lang')
  459. if match.group('afterlang') is not None:
  460. yield match.start('whitespace'), Whitespace, match.group('whitespace')
  461. yield match.start('extra'), Text, match.group('extra')
  462. yield match.start('newline'), Whitespace, match.group('newline')
  463. # lookup lexer if wanted and existing
  464. lexer = None
  465. if self.handlecodeblocks:
  466. try:
  467. lexer = get_lexer_by_name(match.group('lang').strip())
  468. except ClassNotFound:
  469. pass
  470. code = match.group('code')
  471. # no lexer for this language. handle it like it was a code block
  472. if lexer is None:
  473. yield match.start('code'), String, code
  474. else:
  475. # FIXME: aren't the offsets wrong?
  476. yield from do_insertions([], lexer.get_tokens_unprocessed(code))
  477. yield match.start('terminator'), String.Backtick, match.group('terminator')
  478. tokens = {
  479. 'root': [
  480. # heading with '#' prefix (atx-style)
  481. (r'(^#[^#].+)(\n)', bygroups(Generic.Heading, Text)),
  482. # subheading with '#' prefix (atx-style)
  483. (r'(^#{2,6}[^#].+)(\n)', bygroups(Generic.Subheading, Text)),
  484. # heading with '=' underlines (Setext-style)
  485. (r'^(.+)(\n)(=+)(\n)', bygroups(Generic.Heading, Text, Generic.Heading, Text)),
  486. # subheading with '-' underlines (Setext-style)
  487. (r'^(.+)(\n)(-+)(\n)', bygroups(Generic.Subheading, Text, Generic.Subheading, Text)),
  488. # task list
  489. (r'^(\s*)([*-] )(\[[ xX]\])( .+\n)',
  490. bygroups(Whitespace, Keyword, Keyword, using(this, state='inline'))),
  491. # bulleted list
  492. (r'^(\s*)([*-])(\s)(.+\n)',
  493. bygroups(Whitespace, Keyword, Whitespace, using(this, state='inline'))),
  494. # numbered list
  495. (r'^(\s*)([0-9]+\.)( .+\n)',
  496. bygroups(Whitespace, Keyword, using(this, state='inline'))),
  497. # quote
  498. (r'^(\s*>\s)(.+\n)', bygroups(Keyword, Generic.Emph)),
  499. # code block fenced by 3 backticks
  500. (r'^(\s*```\n[\w\W]*?^\s*```$\n)', String.Backtick),
  501. # code block with language
  502. # Some tools include extra stuff after the language name, just
  503. # highlight that as text. For example: https://docs.enola.dev/use/execmd
  504. (r'''(?x)
  505. ^(?P<initial>\s*```)
  506. (?P<lang>[\w\-]+)
  507. (?P<afterlang>
  508. (?P<whitespace>[^\S\n]+)
  509. (?P<extra>.*))?
  510. (?P<newline>\n)
  511. (?P<code>(.|\n)*?)
  512. (?P<terminator>^\s*```$\n)
  513. ''',
  514. _handle_codeblock),
  515. include('inline'),
  516. ],
  517. 'inline': [
  518. # escape
  519. (r'\\.', Text),
  520. # inline code
  521. (r'([^`]?)(`[^`\n]+`)', bygroups(Text, String.Backtick)),
  522. # warning: the following rules eat outer tags.
  523. # eg. **foo _bar_ baz** => foo and baz are not recognized as bold
  524. # bold fenced by '**'
  525. (r'([^\*]?)(\*\*[^* \n][^*\n]*\*\*)', bygroups(Text, Generic.Strong)),
  526. # bold fenced by '__'
  527. (r'([^_]?)(__[^_ \n][^_\n]*__)', bygroups(Text, Generic.Strong)),
  528. # italics fenced by '*'
  529. (r'([^\*]?)(\*[^* \n][^*\n]*\*)', bygroups(Text, Generic.Emph)),
  530. # italics fenced by '_'
  531. (r'([^_]?)(_[^_ \n][^_\n]*_)', bygroups(Text, Generic.Emph)),
  532. # strikethrough
  533. (r'([^~]?)(~~[^~ \n][^~\n]*~~)', bygroups(Text, Generic.Deleted)),
  534. # mentions and topics (twitter and github stuff)
  535. (r'[@#][\w/:]+', Name.Entity),
  536. # (image?) links eg: ![Image of Yaktocat](https://octodex.github.com/images/yaktocat.png)
  537. (r'(!?\[)([^]]+)(\])(\()([^)]+)(\))',
  538. bygroups(Text, Name.Tag, Text, Text, Name.Attribute, Text)),
  539. # reference-style links, e.g.:
  540. # [an example][id]
  541. # [id]: http://example.com/
  542. (r'(\[)([^]]+)(\])(\[)([^]]*)(\])',
  543. bygroups(Text, Name.Tag, Text, Text, Name.Label, Text)),
  544. (r'^(\s*\[)([^]]*)(\]:\s*)(.+)',
  545. bygroups(Text, Name.Label, Text, Name.Attribute)),
  546. # general text, must come last!
  547. (r'[^\\\s]+', Text),
  548. (r'.', Text),
  549. ],
  550. }
  551. def __init__(self, **options):
  552. self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True)
  553. RegexLexer.__init__(self, **options)
  554. class OrgLexer(RegexLexer):
  555. """
  556. For Org Mode markup.
  557. """
  558. name = 'Org Mode'
  559. url = 'https://orgmode.org'
  560. aliases = ['org', 'orgmode', 'org-mode']
  561. filenames = ['*.org']
  562. mimetypes = ["text/org"]
  563. version_added = '2.18'
  564. def _inline(start, end):
  565. return rf'(?<!\w){start}(.|\n(?!\n))+?{end}(?!\w)'
  566. tokens = {
  567. 'root': [
  568. (r'^# .*', Comment.Single),
  569. # Headings
  570. (r'^(\* )(COMMENT)( .*)',
  571. bygroups(Generic.Heading, Comment.Preproc, Generic.Heading)),
  572. (r'^(\*\*+ )(COMMENT)( .*)',
  573. bygroups(Generic.Subheading, Comment.Preproc, Generic.Subheading)),
  574. (r'^(\* )(DONE)( .*)',
  575. bygroups(Generic.Heading, Generic.Deleted, Generic.Heading)),
  576. (r'^(\*\*+ )(DONE)( .*)',
  577. bygroups(Generic.Subheading, Generic.Deleted, Generic.Subheading)),
  578. (r'^(\* )(TODO)( .*)',
  579. bygroups(Generic.Heading, Generic.Error, Generic.Heading)),
  580. (r'^(\*\*+ )(TODO)( .*)',
  581. bygroups(Generic.Subheading, Generic.Error, Generic.Subheading)),
  582. (r'^(\* .+?)( :[a-zA-Z0-9_@:]+:)?$', bygroups(Generic.Heading, Generic.Emph)),
  583. (r'^(\*\*+ .+?)( :[a-zA-Z0-9_@:]+:)?$', bygroups(Generic.Subheading, Generic.Emph)),
  584. # Unordered lists items, including TODO items and description items
  585. (r'^(?:( *)([+-] )|( +)(\* ))(\[[ X-]\])?(.+ ::)?',
  586. bygroups(Whitespace, Keyword, Whitespace, Keyword, Generic.Prompt, Name.Label)),
  587. # Ordered list items
  588. (r'^( *)([0-9]+[.)])( \[@[0-9]+\])?', bygroups(Whitespace, Keyword, Generic.Emph)),
  589. # Dynamic blocks
  590. (r'(?i)^( *#\+begin: *)((?:.|\n)*?)(^ *#\+end: *$)',
  591. bygroups(Operator.Word, using(this), Operator.Word)),
  592. # Comment blocks
  593. (r'(?i)^( *#\+begin_comment *\n)((?:.|\n)*?)(^ *#\+end_comment *$)',
  594. bygroups(Operator.Word, Comment.Multiline, Operator.Word)),
  595. # Source code blocks
  596. # TODO: language-dependent syntax highlighting (see Markdown lexer)
  597. (r'(?i)^( *#\+begin_src .*)((?:.|\n)*?)(^ *#\+end_src *$)',
  598. bygroups(Operator.Word, Text, Operator.Word)),
  599. # Other blocks
  600. (r'(?i)^( *#\+begin_\w+)( *\n)((?:.|\n)*?)(^ *#\+end_\w+)( *$)',
  601. bygroups(Operator.Word, Whitespace, Text, Operator.Word, Whitespace)),
  602. # Keywords
  603. (r'^(#\+\w+:)(.*)$', bygroups(Name.Namespace, Text)),
  604. # Properties and drawers
  605. (r'(?i)^( *:\w+: *\n)((?:.|\n)*?)(^ *:end: *$)',
  606. bygroups(Name.Decorator, Comment.Special, Name.Decorator)),
  607. # Line break operator
  608. (r'\\\\$', Operator),
  609. (r'^\s*CLOSED:\s+', Generic.Deleted, 'dateline'),
  610. (r'^\s*(?:DEADLINE:|SCHEDULED:)\s+', Generic.Error, 'dateline'),
  611. # Bold
  612. (_inline(r'\*', r'\*+'), Generic.Strong),
  613. # Italic
  614. (_inline(r'/', r'/'), Generic.Emph),
  615. # Verbatim
  616. (_inline(r'=', r'='), String), # TODO token
  617. # Code
  618. (_inline(r'~', r'~'), String),
  619. # Strikethrough
  620. (_inline(r'\+', r'\+'), Generic.Deleted),
  621. # Underline
  622. (_inline(r'_', r'_+'), Generic.EmphStrong),
  623. # Dates
  624. (r'<.+?>', Literal.Date),
  625. # Macros
  626. (r'\{\{\{.+?\}\}\}', Comment.Preproc),
  627. # Footnotes
  628. (r'(?<!\[)\[fn:.+?\]', Name.Tag),
  629. # Links
  630. (r'(?s)(\[\[)(.*?)(\]\[)(.*?)(\]\])',
  631. bygroups(Punctuation, Name.Attribute, Punctuation, Name.Tag, Punctuation)),
  632. (r'(?s)(\[\[)(.+?)(\]\])', bygroups(Punctuation, Name.Attribute, Punctuation)),
  633. (r'(<<)(.+?)(>>)', bygroups(Punctuation, Name.Attribute, Punctuation)),
  634. # Tables
  635. (r'^( *)(\|[ -].*?[ -]\|)$', bygroups(Whitespace, String)),
  636. # Any other text
  637. (r'[^#*+\-0-9:\\/=~_<{\[|\n]+', Text),
  638. (r'[#*+\-0-9:\\/=~_<{\[|\n]', Text),
  639. ],
  640. 'dateline': [
  641. (r'\s*CLOSED:\s+', Generic.Deleted),
  642. (r'\s*(?:DEADLINE:|SCHEDULED:)\s+', Generic.Error),
  643. (r'\[.+?\]', Literal.Date),
  644. (r'<[^>]+?>', Literal.Date),
  645. (r'(\s*)$', Text, '#pop'),
  646. (r'.', Text),
  647. ],
  648. }
  649. class TiddlyWiki5Lexer(RegexLexer):
  650. """
  651. For TiddlyWiki5 markup.
  652. """
  653. name = 'tiddler'
  654. url = 'https://tiddlywiki.com/#TiddlerFiles'
  655. aliases = ['tid']
  656. filenames = ['*.tid']
  657. mimetypes = ["text/vnd.tiddlywiki"]
  658. version_added = '2.7'
  659. flags = re.MULTILINE
  660. def _handle_codeblock(self, match):
  661. """
  662. match args: 1:backticks, 2:lang_name, 3:newline, 4:code, 5:backticks
  663. """
  664. from pygments.lexers import get_lexer_by_name
  665. # section header
  666. yield match.start(1), String, match.group(1)
  667. yield match.start(2), String, match.group(2)
  668. yield match.start(3), Text, match.group(3)
  669. # lookup lexer if wanted and existing
  670. lexer = None
  671. if self.handlecodeblocks:
  672. try:
  673. lexer = get_lexer_by_name(match.group(2).strip())
  674. except ClassNotFound:
  675. pass
  676. code = match.group(4)
  677. # no lexer for this language. handle it like it was a code block
  678. if lexer is None:
  679. yield match.start(4), String, code
  680. return
  681. yield from do_insertions([], lexer.get_tokens_unprocessed(code))
  682. yield match.start(5), String, match.group(5)
  683. def _handle_cssblock(self, match):
  684. """
  685. match args: 1:style tag 2:newline, 3:code, 4:closing style tag
  686. """
  687. from pygments.lexers import get_lexer_by_name
  688. # section header
  689. yield match.start(1), String, match.group(1)
  690. yield match.start(2), String, match.group(2)
  691. lexer = None
  692. if self.handlecodeblocks:
  693. try:
  694. lexer = get_lexer_by_name('css')
  695. except ClassNotFound:
  696. pass
  697. code = match.group(3)
  698. # no lexer for this language. handle it like it was a code block
  699. if lexer is None:
  700. yield match.start(3), String, code
  701. return
  702. yield from do_insertions([], lexer.get_tokens_unprocessed(code))
  703. yield match.start(4), String, match.group(4)
  704. tokens = {
  705. 'root': [
  706. # title in metadata section
  707. (r'^(title)(:\s)(.+\n)', bygroups(Keyword, Text, Generic.Heading)),
  708. # headings
  709. (r'^(!)([^!].+\n)', bygroups(Generic.Heading, Text)),
  710. (r'^(!{2,6})(.+\n)', bygroups(Generic.Subheading, Text)),
  711. # bulleted or numbered lists or single-line block quotes
  712. # (can be mixed)
  713. (r'^(\s*)([*#>]+)(\s*)(.+\n)',
  714. bygroups(Text, Keyword, Text, using(this, state='inline'))),
  715. # multi-line block quotes
  716. (r'^(<<<.*\n)([\w\W]*?)(^<<<.*$)', bygroups(String, Text, String)),
  717. # table header
  718. (r'^(\|.*?\|h)$', bygroups(Generic.Strong)),
  719. # table footer or caption
  720. (r'^(\|.*?\|[cf])$', bygroups(Generic.Emph)),
  721. # table class
  722. (r'^(\|.*?\|k)$', bygroups(Name.Tag)),
  723. # definitions
  724. (r'^(;.*)$', bygroups(Generic.Strong)),
  725. # text block
  726. (r'^(```\n)([\w\W]*?)(^```$)', bygroups(String, Text, String)),
  727. # code block with language
  728. (r'^(```)(\w+)(\n)([\w\W]*?)(^```$)', _handle_codeblock),
  729. # CSS style block
  730. (r'^(<style>)(\n)([\w\W]*?)(^</style>$)', _handle_cssblock),
  731. include('keywords'),
  732. include('inline'),
  733. ],
  734. 'keywords': [
  735. (words((
  736. '\\define', '\\end', 'caption', 'created', 'modified', 'tags',
  737. 'title', 'type'), prefix=r'^', suffix=r'\b'),
  738. Keyword),
  739. ],
  740. 'inline': [
  741. # escape
  742. (r'\\.', Text),
  743. # created or modified date
  744. (r'\d{17}', Number.Integer),
  745. # italics
  746. (r'(\s)(//[^/]+//)((?=\W|\n))',
  747. bygroups(Text, Generic.Emph, Text)),
  748. # superscript
  749. (r'(\s)(\^\^[^\^]+\^\^)', bygroups(Text, Generic.Emph)),
  750. # subscript
  751. (r'(\s)(,,[^,]+,,)', bygroups(Text, Generic.Emph)),
  752. # underscore
  753. (r'(\s)(__[^_]+__)', bygroups(Text, Generic.Strong)),
  754. # bold
  755. (r"(\s)(''[^']+'')((?=\W|\n))",
  756. bygroups(Text, Generic.Strong, Text)),
  757. # strikethrough
  758. (r'(\s)(~~[^~]+~~)((?=\W|\n))',
  759. bygroups(Text, Generic.Deleted, Text)),
  760. # TiddlyWiki variables
  761. (r'<<[^>]+>>', Name.Tag),
  762. (r'\$\$[^$]+\$\$', Name.Tag),
  763. (r'\$\([^)]+\)\$', Name.Tag),
  764. # TiddlyWiki style or class
  765. (r'^@@.*$', Name.Tag),
  766. # HTML tags
  767. (r'</?[^>]+>', Name.Tag),
  768. # inline code
  769. (r'`[^`]+`', String.Backtick),
  770. # HTML escaped symbols
  771. (r'&\S*?;', String.Regex),
  772. # Wiki links
  773. (r'(\[{2})([^]\|]+)(\]{2})', bygroups(Text, Name.Tag, Text)),
  774. # External links
  775. (r'(\[{2})([^]\|]+)(\|)([^]\|]+)(\]{2})',
  776. bygroups(Text, Name.Tag, Text, Name.Attribute, Text)),
  777. # Transclusion
  778. (r'(\{{2})([^}]+)(\}{2})', bygroups(Text, Name.Tag, Text)),
  779. # URLs
  780. (r'(\b.?.?tps?://[^\s"]+)', bygroups(Name.Attribute)),
  781. # general text, must come last!
  782. (r'[\w]+', Text),
  783. (r'.', Text)
  784. ],
  785. }
  786. def __init__(self, **options):
  787. self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True)
  788. RegexLexer.__init__(self, **options)
  789. class WikitextLexer(RegexLexer):
  790. """
  791. For MediaWiki Wikitext.
  792. Parsing Wikitext is tricky, and results vary between different MediaWiki
  793. installations, so we only highlight common syntaxes (built-in or from
  794. popular extensions), and also assume templates produce no unbalanced
  795. syntaxes.
  796. """
  797. name = 'Wikitext'
  798. url = 'https://www.mediawiki.org/wiki/Wikitext'
  799. aliases = ['wikitext', 'mediawiki']
  800. filenames = []
  801. mimetypes = ['text/x-wiki']
  802. version_added = '2.15'
  803. flags = re.MULTILINE
  804. def nowiki_tag_rules(tag_name):
  805. return [
  806. (rf'(?i)(</)({tag_name})(\s*)(>)', bygroups(Punctuation,
  807. Name.Tag, Whitespace, Punctuation), '#pop'),
  808. include('entity'),
  809. include('text'),
  810. ]
  811. def plaintext_tag_rules(tag_name):
  812. return [
  813. (rf'(?si)(.*?)(</)({tag_name})(\s*)(>)', bygroups(Text,
  814. Punctuation, Name.Tag, Whitespace, Punctuation), '#pop'),
  815. ]
  816. def delegate_tag_rules(tag_name, lexer, **lexer_kwargs):
  817. return [
  818. (rf'(?i)(</)({tag_name})(\s*)(>)', bygroups(Punctuation,
  819. Name.Tag, Whitespace, Punctuation), '#pop'),
  820. (rf'(?si).+?(?=</{tag_name}\s*>)', using(lexer, **lexer_kwargs)),
  821. ]
  822. def text_rules(token):
  823. return [
  824. (r'\w+', token),
  825. (r'[^\S\n]+', token),
  826. (r'(?s).', token),
  827. ]
  828. def handle_syntaxhighlight(self, match, ctx):
  829. from pygments.lexers import get_lexer_by_name
  830. attr_content = match.group()
  831. start = 0
  832. index = 0
  833. while True:
  834. index = attr_content.find('>', start)
  835. # Exclude comment end (-->)
  836. if attr_content[index-2:index] != '--':
  837. break
  838. start = index + 1
  839. if index == -1:
  840. # No tag end
  841. yield from self.get_tokens_unprocessed(attr_content, stack=['root', 'attr'])
  842. return
  843. attr = attr_content[:index]
  844. yield from self.get_tokens_unprocessed(attr, stack=['root', 'attr'])
  845. yield match.start(3) + index, Punctuation, '>'
  846. lexer = None
  847. content = attr_content[index+1:]
  848. lang_match = re.findall(r'\blang=("|\'|)(\w+)(\1)', attr)
  849. if len(lang_match) >= 1:
  850. # Pick the last match in case of multiple matches
  851. lang = lang_match[-1][1]
  852. try:
  853. lexer = get_lexer_by_name(lang)
  854. except ClassNotFound:
  855. pass
  856. if lexer is None:
  857. yield match.start() + index + 1, Text, content
  858. else:
  859. yield from lexer.get_tokens_unprocessed(content)
  860. def handle_score(self, match, ctx):
  861. attr_content = match.group()
  862. start = 0
  863. index = 0
  864. while True:
  865. index = attr_content.find('>', start)
  866. # Exclude comment end (-->)
  867. if attr_content[index-2:index] != '--':
  868. break
  869. start = index + 1
  870. if index == -1:
  871. # No tag end
  872. yield from self.get_tokens_unprocessed(attr_content, stack=['root', 'attr'])
  873. return
  874. attr = attr_content[:index]
  875. content = attr_content[index+1:]
  876. yield from self.get_tokens_unprocessed(attr, stack=['root', 'attr'])
  877. yield match.start(3) + index, Punctuation, '>'
  878. lang_match = re.findall(r'\blang=("|\'|)(\w+)(\1)', attr)
  879. # Pick the last match in case of multiple matches
  880. lang = lang_match[-1][1] if len(lang_match) >= 1 else 'lilypond'
  881. if lang == 'lilypond': # Case sensitive
  882. yield from LilyPondLexer().get_tokens_unprocessed(content)
  883. else: # ABC
  884. # FIXME: Use ABC lexer in the future
  885. yield match.start() + index + 1, Text, content
  886. # a-z removed to prevent linter from complaining, REMEMBER to use (?i)
  887. title_char = r' %!"$&\'()*,\-./0-9:;=?@A-Z\\\^_`~+\u0080-\uFFFF'
  888. nbsp_char = r'(?:\t|&nbsp;|&\#0*160;|&\#[Xx]0*[Aa]0;|[ \xA0\u1680\u2000-\u200A\u202F\u205F\u3000])'
  889. link_address = r'(?:[0-9.]+|\[[0-9a-f:.]+\]|[^\x00-\x20"<>\[\]\x7F\xA0\u1680\u2000-\u200A\u202F\u205F\u3000\uFFFD])'
  890. link_char_class = r'[^\x00-\x20"<>\[\]\x7F\xA0\u1680\u2000-\u200A\u202F\u205F\u3000\uFFFD]'
  891. double_slashes_i = {
  892. '__FORCETOC__', '__NOCONTENTCONVERT__', '__NOCC__', '__NOEDITSECTION__', '__NOGALLERY__',
  893. '__NOTITLECONVERT__', '__NOTC__', '__NOTOC__', '__TOC__',
  894. }
  895. double_slashes = {
  896. '__EXPECTUNUSEDCATEGORY__', '__HIDDENCAT__', '__INDEX__', '__NEWSECTIONLINK__',
  897. '__NOINDEX__', '__NONEWSECTIONLINK__', '__STATICREDIRECT__', '__NOGLOBAL__',
  898. '__DISAMBIG__', '__EXPECTED_UNCONNECTED_PAGE__',
  899. }
  900. protocols = {
  901. 'bitcoin:', 'ftp://', 'ftps://', 'geo:', 'git://', 'gopher://', 'http://', 'https://',
  902. 'irc://', 'ircs://', 'magnet:', 'mailto:', 'mms://', 'news:', 'nntp://', 'redis://',
  903. 'sftp://', 'sip:', 'sips:', 'sms:', 'ssh://', 'svn://', 'tel:', 'telnet://', 'urn:',
  904. 'worldwind://', 'xmpp:', '//',
  905. }
  906. non_relative_protocols = protocols - {'//'}
  907. html_tags = {
  908. 'abbr', 'b', 'bdi', 'bdo', 'big', 'blockquote', 'br', 'caption', 'center', 'cite', 'code',
  909. 'data', 'dd', 'del', 'dfn', 'div', 'dl', 'dt', 'em', 'font', 'h1', 'h2', 'h3', 'h4', 'h5',
  910. 'h6', 'hr', 'i', 'ins', 'kbd', 'li', 'link', 'mark', 'meta', 'ol', 'p', 'q', 'rb', 'rp',
  911. 'rt', 'rtc', 'ruby', 's', 'samp', 'small', 'span', 'strike', 'strong', 'sub', 'sup',
  912. 'table', 'td', 'th', 'time', 'tr', 'tt', 'u', 'ul', 'var', 'wbr',
  913. }
  914. parser_tags = {
  915. 'graph', 'charinsert', 'rss', 'chem', 'categorytree', 'nowiki', 'inputbox', 'math',
  916. 'hiero', 'score', 'pre', 'ref', 'translate', 'imagemap', 'templatestyles', 'languages',
  917. 'noinclude', 'mapframe', 'section', 'poem', 'syntaxhighlight', 'includeonly', 'tvar',
  918. 'onlyinclude', 'templatedata', 'langconvert', 'timeline', 'dynamicpagelist', 'gallery',
  919. 'maplink', 'ce', 'references',
  920. }
  921. variant_langs = {
  922. # ZhConverter.php
  923. 'zh', 'zh-hans', 'zh-hant', 'zh-cn', 'zh-hk', 'zh-mo', 'zh-my', 'zh-sg', 'zh-tw',
  924. # WuuConverter.php
  925. 'wuu', 'wuu-hans', 'wuu-hant',
  926. # UzConverter.php
  927. 'uz', 'uz-latn', 'uz-cyrl',
  928. # TlyConverter.php
  929. 'tly', 'tly-cyrl',
  930. # TgConverter.php
  931. 'tg', 'tg-latn',
  932. # SrConverter.php
  933. 'sr', 'sr-ec', 'sr-el',
  934. # ShiConverter.php
  935. 'shi', 'shi-tfng', 'shi-latn',
  936. # ShConverter.php
  937. 'sh-latn', 'sh-cyrl',
  938. # KuConverter.php
  939. 'ku', 'ku-arab', 'ku-latn',
  940. # IuConverter.php
  941. 'iu', 'ike-cans', 'ike-latn',
  942. # GanConverter.php
  943. 'gan', 'gan-hans', 'gan-hant',
  944. # EnConverter.php
  945. 'en', 'en-x-piglatin',
  946. # CrhConverter.php
  947. 'crh', 'crh-cyrl', 'crh-latn',
  948. # BanConverter.php
  949. 'ban', 'ban-bali', 'ban-x-dharma', 'ban-x-palmleaf', 'ban-x-pku',
  950. }
  951. magic_vars_i = {
  952. 'ARTICLEPATH', 'INT', 'PAGEID', 'SCRIPTPATH', 'SERVER', 'SERVERNAME', 'STYLEPATH',
  953. }
  954. magic_vars = {
  955. '!', '=', 'BASEPAGENAME', 'BASEPAGENAMEE', 'CASCADINGSOURCES', 'CONTENTLANGUAGE',
  956. 'CONTENTLANG', 'CURRENTDAY', 'CURRENTDAY2', 'CURRENTDAYNAME', 'CURRENTDOW', 'CURRENTHOUR',
  957. 'CURRENTMONTH', 'CURRENTMONTH2', 'CURRENTMONTH1', 'CURRENTMONTHABBREV', 'CURRENTMONTHNAME',
  958. 'CURRENTMONTHNAMEGEN', 'CURRENTTIME', 'CURRENTTIMESTAMP', 'CURRENTVERSION', 'CURRENTWEEK',
  959. 'CURRENTYEAR', 'DIRECTIONMARK', 'DIRMARK', 'FULLPAGENAME', 'FULLPAGENAMEE', 'LOCALDAY',
  960. 'LOCALDAY2', 'LOCALDAYNAME', 'LOCALDOW', 'LOCALHOUR', 'LOCALMONTH', 'LOCALMONTH2',
  961. 'LOCALMONTH1', 'LOCALMONTHABBREV', 'LOCALMONTHNAME', 'LOCALMONTHNAMEGEN', 'LOCALTIME',
  962. 'LOCALTIMESTAMP', 'LOCALWEEK', 'LOCALYEAR', 'NAMESPACE', 'NAMESPACEE', 'NAMESPACENUMBER',
  963. 'NUMBEROFACTIVEUSERS', 'NUMBEROFADMINS', 'NUMBEROFARTICLES', 'NUMBEROFEDITS',
  964. 'NUMBEROFFILES', 'NUMBEROFPAGES', 'NUMBEROFUSERS', 'PAGELANGUAGE', 'PAGENAME', 'PAGENAMEE',
  965. 'REVISIONDAY', 'REVISIONDAY2', 'REVISIONID', 'REVISIONMONTH', 'REVISIONMONTH1',
  966. 'REVISIONSIZE', 'REVISIONTIMESTAMP', 'REVISIONUSER', 'REVISIONYEAR', 'ROOTPAGENAME',
  967. 'ROOTPAGENAMEE', 'SITENAME', 'SUBJECTPAGENAME', 'ARTICLEPAGENAME', 'SUBJECTPAGENAMEE',
  968. 'ARTICLEPAGENAMEE', 'SUBJECTSPACE', 'ARTICLESPACE', 'SUBJECTSPACEE', 'ARTICLESPACEE',
  969. 'SUBPAGENAME', 'SUBPAGENAMEE', 'TALKPAGENAME', 'TALKPAGENAMEE', 'TALKSPACE', 'TALKSPACEE',
  970. }
  971. parser_functions_i = {
  972. 'ANCHORENCODE', 'BIDI', 'CANONICALURL', 'CANONICALURLE', 'FILEPATH', 'FORMATNUM',
  973. 'FULLURL', 'FULLURLE', 'GENDER', 'GRAMMAR', 'INT', r'\#LANGUAGE', 'LC', 'LCFIRST', 'LOCALURL',
  974. 'LOCALURLE', 'NS', 'NSE', 'PADLEFT', 'PADRIGHT', 'PAGEID', 'PLURAL', 'UC', 'UCFIRST',
  975. 'URLENCODE',
  976. }
  977. parser_functions = {
  978. 'BASEPAGENAME', 'BASEPAGENAMEE', 'CASCADINGSOURCES', 'DEFAULTSORT', 'DEFAULTSORTKEY',
  979. 'DEFAULTCATEGORYSORT', 'FULLPAGENAME', 'FULLPAGENAMEE', 'NAMESPACE', 'NAMESPACEE',
  980. 'NAMESPACENUMBER', 'NUMBERINGROUP', 'NUMINGROUP', 'NUMBEROFACTIVEUSERS', 'NUMBEROFADMINS',
  981. 'NUMBEROFARTICLES', 'NUMBEROFEDITS', 'NUMBEROFFILES', 'NUMBEROFPAGES', 'NUMBEROFUSERS',
  982. 'PAGENAME', 'PAGENAMEE', 'PAGESINCATEGORY', 'PAGESINCAT', 'PAGESIZE', 'PROTECTIONEXPIRY',
  983. 'PROTECTIONLEVEL', 'REVISIONDAY', 'REVISIONDAY2', 'REVISIONID', 'REVISIONMONTH',
  984. 'REVISIONMONTH1', 'REVISIONTIMESTAMP', 'REVISIONUSER', 'REVISIONYEAR', 'ROOTPAGENAME',
  985. 'ROOTPAGENAMEE', 'SUBJECTPAGENAME', 'ARTICLEPAGENAME', 'SUBJECTPAGENAMEE',
  986. 'ARTICLEPAGENAMEE', 'SUBJECTSPACE', 'ARTICLESPACE', 'SUBJECTSPACEE', 'ARTICLESPACEE',
  987. 'SUBPAGENAME', 'SUBPAGENAMEE', 'TALKPAGENAME', 'TALKPAGENAMEE', 'TALKSPACE', 'TALKSPACEE',
  988. 'INT', 'DISPLAYTITLE', 'PAGESINNAMESPACE', 'PAGESINNS',
  989. }
  990. tokens = {
  991. 'root': [
  992. # Redirects
  993. (r"""(?xi)
  994. (\A\s*?)(\#REDIRECT:?) # may contain a colon
  995. (\s+)(\[\[) (?=[^\]\n]* \]\]$)
  996. """,
  997. bygroups(Whitespace, Keyword, Whitespace, Punctuation), 'redirect-inner'),
  998. # Subheadings
  999. (r'^(={2,6})(.+?)(\1)(\s*$\n)',
  1000. bygroups(Generic.Subheading, Generic.Subheading, Generic.Subheading, Whitespace)),
  1001. # Headings
  1002. (r'^(=.+?=)(\s*$\n)',
  1003. bygroups(Generic.Heading, Whitespace)),
  1004. # Double-slashed magic words
  1005. (words(double_slashes_i, prefix=r'(?i)'), Name.Function.Magic),
  1006. (words(double_slashes), Name.Function.Magic),
  1007. # Raw URLs
  1008. (r'(?i)\b(?:{}){}{}*'.format('|'.join(protocols),
  1009. link_address, link_char_class), Name.Label),
  1010. # Magic links
  1011. (rf'\b(?:RFC|PMID){nbsp_char}+[0-9]+\b',
  1012. Name.Function.Magic),
  1013. (r"""(?x)
  1014. \bISBN {nbsp_char}
  1015. (?: 97[89] {nbsp_dash}? )?
  1016. (?: [0-9] {nbsp_dash}? ){{9}} # escape format()
  1017. [0-9Xx]\b
  1018. """.format(nbsp_char=nbsp_char, nbsp_dash=f'(?:-|{nbsp_char})'), Name.Function.Magic),
  1019. include('list'),
  1020. include('inline'),
  1021. include('text'),
  1022. ],
  1023. 'redirect-inner': [
  1024. (r'(\]\])(\s*?\n)', bygroups(Punctuation, Whitespace), '#pop'),
  1025. (r'(\#)([^#]*?)', bygroups(Punctuation, Name.Label)),
  1026. (rf'(?i)[{title_char}]+', Name.Tag),
  1027. ],
  1028. 'list': [
  1029. # Description lists
  1030. (r'^;', Keyword, 'dt'),
  1031. # Ordered lists, unordered lists and indents
  1032. (r'^[#:*]+', Keyword),
  1033. # Horizontal rules
  1034. (r'^-{4,}', Keyword),
  1035. ],
  1036. 'inline': [
  1037. # Signatures
  1038. (r'~{3,5}', Keyword),
  1039. # Entities
  1040. include('entity'),
  1041. # Bold & italic
  1042. (r"('')(''')(?!')", bygroups(Generic.Emph,
  1043. Generic.EmphStrong), 'inline-italic-bold'),
  1044. (r"'''(?!')", Generic.Strong, 'inline-bold'),
  1045. (r"''(?!')", Generic.Emph, 'inline-italic'),
  1046. # Comments & parameters & templates
  1047. include('replaceable'),
  1048. # Media links
  1049. (
  1050. r"""(?xi)
  1051. (\[\[)
  1052. (File|Image) (:)
  1053. ((?: [{}] | \{{{{2,3}}[^{{}}]*?\}}{{2,3}} | <!--[\s\S]*?--> )*)
  1054. (?: (\#) ([{}]*?) )?
  1055. """.format(title_char, f'{title_char}#'),
  1056. bygroups(Punctuation, Name.Namespace, Punctuation,
  1057. using(this, state=['wikilink-name']), Punctuation, Name.Label),
  1058. 'medialink-inner'
  1059. ),
  1060. # Wikilinks
  1061. (
  1062. r"""(?xi)
  1063. (\[\[)(?!{}) # Should not contain URLs
  1064. (?: ([{}]*) (:))?
  1065. ((?: [{}] | \{{{{2,3}}[^{{}}]*?\}}{{2,3}} | <!--[\s\S]*?--> )*?)
  1066. (?: (\#) ([{}]*?) )?
  1067. (\]\])
  1068. """.format('|'.join(protocols), title_char.replace('/', ''),
  1069. title_char, f'{title_char}#'),
  1070. bygroups(Punctuation, Name.Namespace, Punctuation,
  1071. using(this, state=['wikilink-name']), Punctuation, Name.Label, Punctuation)
  1072. ),
  1073. (
  1074. r"""(?xi)
  1075. (\[\[)(?!{})
  1076. (?: ([{}]*) (:))?
  1077. ((?: [{}] | \{{{{2,3}}[^{{}}]*?\}}{{2,3}} | <!--[\s\S]*?--> )*?)
  1078. (?: (\#) ([{}]*?) )?
  1079. (\|)
  1080. """.format('|'.join(protocols), title_char.replace('/', ''),
  1081. title_char, f'{title_char}#'),
  1082. bygroups(Punctuation, Name.Namespace, Punctuation,
  1083. using(this, state=['wikilink-name']), Punctuation, Name.Label, Punctuation),
  1084. 'wikilink-inner'
  1085. ),
  1086. # External links
  1087. (
  1088. r"""(?xi)
  1089. (\[)
  1090. ((?:{}) {} {}*)
  1091. (\s*)
  1092. """.format('|'.join(protocols), link_address, link_char_class),
  1093. bygroups(Punctuation, Name.Label, Whitespace),
  1094. 'extlink-inner'
  1095. ),
  1096. # Tables
  1097. (r'^(:*)(\s*?)(\{\|)([^\n]*)$', bygroups(Keyword,
  1098. Whitespace, Punctuation, using(this, state=['root', 'attr'])), 'table'),
  1099. # HTML tags
  1100. (r'(?i)(<)({})\b'.format('|'.join(html_tags)),
  1101. bygroups(Punctuation, Name.Tag), 'tag-inner-ordinary'),
  1102. (r'(?i)(</)({})\b(\s*)(>)'.format('|'.join(html_tags)),
  1103. bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
  1104. # <nowiki>
  1105. (r'(?i)(<)(nowiki)\b', bygroups(Punctuation,
  1106. Name.Tag), ('tag-nowiki', 'tag-inner')),
  1107. # <pre>
  1108. (r'(?i)(<)(pre)\b', bygroups(Punctuation,
  1109. Name.Tag), ('tag-pre', 'tag-inner')),
  1110. # <categorytree>
  1111. (r'(?i)(<)(categorytree)\b', bygroups(
  1112. Punctuation, Name.Tag), ('tag-categorytree', 'tag-inner')),
  1113. # <hiero>
  1114. (r'(?i)(<)(hiero)\b', bygroups(Punctuation,
  1115. Name.Tag), ('tag-hiero', 'tag-inner')),
  1116. # <math>
  1117. (r'(?i)(<)(math)\b', bygroups(Punctuation,
  1118. Name.Tag), ('tag-math', 'tag-inner')),
  1119. # <chem>
  1120. (r'(?i)(<)(chem)\b', bygroups(Punctuation,
  1121. Name.Tag), ('tag-chem', 'tag-inner')),
  1122. # <ce>
  1123. (r'(?i)(<)(ce)\b', bygroups(Punctuation,
  1124. Name.Tag), ('tag-ce', 'tag-inner')),
  1125. # <charinsert>
  1126. (r'(?i)(<)(charinsert)\b', bygroups(
  1127. Punctuation, Name.Tag), ('tag-charinsert', 'tag-inner')),
  1128. # <templatedata>
  1129. (r'(?i)(<)(templatedata)\b', bygroups(
  1130. Punctuation, Name.Tag), ('tag-templatedata', 'tag-inner')),
  1131. # <gallery>
  1132. (r'(?i)(<)(gallery)\b', bygroups(
  1133. Punctuation, Name.Tag), ('tag-gallery', 'tag-inner')),
  1134. # <graph>
  1135. (r'(?i)(<)(gallery)\b', bygroups(
  1136. Punctuation, Name.Tag), ('tag-graph', 'tag-inner')),
  1137. # <dynamicpagelist>
  1138. (r'(?i)(<)(dynamicpagelist)\b', bygroups(
  1139. Punctuation, Name.Tag), ('tag-dynamicpagelist', 'tag-inner')),
  1140. # <inputbox>
  1141. (r'(?i)(<)(inputbox)\b', bygroups(
  1142. Punctuation, Name.Tag), ('tag-inputbox', 'tag-inner')),
  1143. # <rss>
  1144. (r'(?i)(<)(rss)\b', bygroups(
  1145. Punctuation, Name.Tag), ('tag-rss', 'tag-inner')),
  1146. # <imagemap>
  1147. (r'(?i)(<)(imagemap)\b', bygroups(
  1148. Punctuation, Name.Tag), ('tag-imagemap', 'tag-inner')),
  1149. # <syntaxhighlight>
  1150. (r'(?i)(</)(syntaxhighlight)\b(\s*)(>)',
  1151. bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
  1152. (r'(?si)(<)(syntaxhighlight)\b([^>]*?(?<!/)>.*?)(?=</\2\s*>)',
  1153. bygroups(Punctuation, Name.Tag, handle_syntaxhighlight)),
  1154. # <syntaxhighlight>: Fallback case for self-closing tags
  1155. (r'(?i)(<)(syntaxhighlight)\b(\s*?)((?:[^>]|-->)*?)(/\s*?(?<!--)>)', bygroups(
  1156. Punctuation, Name.Tag, Whitespace, using(this, state=['root', 'attr']), Punctuation)),
  1157. # <source>
  1158. (r'(?i)(</)(source)\b(\s*)(>)',
  1159. bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
  1160. (r'(?si)(<)(source)\b([^>]*?(?<!/)>.*?)(?=</\2\s*>)',
  1161. bygroups(Punctuation, Name.Tag, handle_syntaxhighlight)),
  1162. # <source>: Fallback case for self-closing tags
  1163. (r'(?i)(<)(source)\b(\s*?)((?:[^>]|-->)*?)(/\s*?(?<!--)>)', bygroups(
  1164. Punctuation, Name.Tag, Whitespace, using(this, state=['root', 'attr']), Punctuation)),
  1165. # <score>
  1166. (r'(?i)(</)(score)\b(\s*)(>)',
  1167. bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
  1168. (r'(?si)(<)(score)\b([^>]*?(?<!/)>.*?)(?=</\2\s*>)',
  1169. bygroups(Punctuation, Name.Tag, handle_score)),
  1170. # <score>: Fallback case for self-closing tags
  1171. (r'(?i)(<)(score)\b(\s*?)((?:[^>]|-->)*?)(/\s*?(?<!--)>)', bygroups(
  1172. Punctuation, Name.Tag, Whitespace, using(this, state=['root', 'attr']), Punctuation)),
  1173. # Other parser tags
  1174. (r'(?i)(<)({})\b'.format('|'.join(parser_tags)),
  1175. bygroups(Punctuation, Name.Tag), 'tag-inner-ordinary'),
  1176. (r'(?i)(</)({})\b(\s*)(>)'.format('|'.join(parser_tags)),
  1177. bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
  1178. # LanguageConverter markups
  1179. (
  1180. r"""(?xi)
  1181. (-\{{) # Use {{ to escape format()
  1182. ([^|]) (\|)
  1183. (?:
  1184. (?: ([^;]*?) (=>))?
  1185. (\s* (?:{variants}) \s*) (:)
  1186. )?
  1187. """.format(variants='|'.join(variant_langs)),
  1188. bygroups(Punctuation, Keyword, Punctuation,
  1189. using(this, state=['root', 'lc-raw']),
  1190. Operator, Name.Label, Punctuation),
  1191. 'lc-inner'
  1192. ),
  1193. # LanguageConverter markups: composite conversion grammar
  1194. (
  1195. r"""(?xi)
  1196. (-\{)
  1197. ([a-z\s;-]*?) (\|)
  1198. """,
  1199. bygroups(Punctuation,
  1200. using(this, state=['root', 'lc-flag']),
  1201. Punctuation),
  1202. 'lc-raw'
  1203. ),
  1204. # LanguageConverter markups: fallbacks
  1205. (
  1206. r"""(?xi)
  1207. (-\{{) (?!\{{) # Use {{ to escape format()
  1208. (?: (\s* (?:{variants}) \s*) (:))?
  1209. """.format(variants='|'.join(variant_langs)),
  1210. bygroups(Punctuation, Name.Label, Punctuation),
  1211. 'lc-inner'
  1212. ),
  1213. ],
  1214. 'wikilink-name': [
  1215. include('replaceable'),
  1216. (r'[^{<]+', Name.Tag),
  1217. (r'(?s).', Name.Tag),
  1218. ],
  1219. 'wikilink-inner': [
  1220. # Quit in case of another wikilink
  1221. (r'(?=\[\[)', Punctuation, '#pop'),
  1222. (r'\]\]', Punctuation, '#pop'),
  1223. include('inline'),
  1224. include('text'),
  1225. ],
  1226. 'medialink-inner': [
  1227. (r'\]\]', Punctuation, '#pop'),
  1228. (r'(\|)([^\n=|]*)(=)',
  1229. bygroups(Punctuation, Name.Attribute, Operator)),
  1230. (r'\|', Punctuation),
  1231. include('inline'),
  1232. include('text'),
  1233. ],
  1234. 'quote-common': [
  1235. # Quit in case of link/template endings
  1236. (r'(?=\]\]|\{\{|\}\})', Punctuation, '#pop'),
  1237. (r'\n', Text, '#pop'),
  1238. ],
  1239. 'inline-italic': [
  1240. include('quote-common'),
  1241. (r"('')(''')(?!')", bygroups(Generic.Emph,
  1242. Generic.Strong), ('#pop', 'inline-bold')),
  1243. (r"'''(?!')", Generic.EmphStrong, ('#pop', 'inline-italic-bold')),
  1244. (r"''(?!')", Generic.Emph, '#pop'),
  1245. include('inline'),
  1246. include('text-italic'),
  1247. ],
  1248. 'inline-bold': [
  1249. include('quote-common'),
  1250. (r"(''')('')(?!')", bygroups(
  1251. Generic.Strong, Generic.Emph), ('#pop', 'inline-italic')),
  1252. (r"'''(?!')", Generic.Strong, '#pop'),
  1253. (r"''(?!')", Generic.EmphStrong, ('#pop', 'inline-bold-italic')),
  1254. include('inline'),
  1255. include('text-bold'),
  1256. ],
  1257. 'inline-bold-italic': [
  1258. include('quote-common'),
  1259. (r"('')(''')(?!')", bygroups(Generic.EmphStrong,
  1260. Generic.Strong), '#pop'),
  1261. (r"'''(?!')", Generic.EmphStrong, ('#pop', 'inline-italic')),
  1262. (r"''(?!')", Generic.EmphStrong, ('#pop', 'inline-bold')),
  1263. include('inline'),
  1264. include('text-bold-italic'),
  1265. ],
  1266. 'inline-italic-bold': [
  1267. include('quote-common'),
  1268. (r"(''')('')(?!')", bygroups(
  1269. Generic.EmphStrong, Generic.Emph), '#pop'),
  1270. (r"'''(?!')", Generic.EmphStrong, ('#pop', 'inline-italic')),
  1271. (r"''(?!')", Generic.EmphStrong, ('#pop', 'inline-bold')),
  1272. include('inline'),
  1273. include('text-bold-italic'),
  1274. ],
  1275. 'lc-flag': [
  1276. (r'\s+', Whitespace),
  1277. (r';', Punctuation),
  1278. *text_rules(Keyword),
  1279. ],
  1280. 'lc-inner': [
  1281. (
  1282. r"""(?xi)
  1283. (;)
  1284. (?: ([^;]*?) (=>))?
  1285. (\s* (?:{variants}) \s*) (:)
  1286. """.format(variants='|'.join(variant_langs)),
  1287. bygroups(Punctuation, using(this, state=['root', 'lc-raw']),
  1288. Operator, Name.Label, Punctuation)
  1289. ),
  1290. (r';?\s*?\}-', Punctuation, '#pop'),
  1291. include('inline'),
  1292. include('text'),
  1293. ],
  1294. 'lc-raw': [
  1295. (r'\}-', Punctuation, '#pop'),
  1296. include('inline'),
  1297. include('text'),
  1298. ],
  1299. 'replaceable': [
  1300. # Comments
  1301. (r'<!--[\s\S]*?(?:-->|\Z)', Comment.Multiline),
  1302. # Parameters
  1303. (
  1304. r"""(?x)
  1305. (\{{3})
  1306. ([^|]*?)
  1307. (?=\}{3}|\|)
  1308. """,
  1309. bygroups(Punctuation, Name.Variable),
  1310. 'parameter-inner',
  1311. ),
  1312. # Magic variables
  1313. (r'(?i)(\{{\{{)(\s*)({})(\s*)(\}}\}})'.format('|'.join(magic_vars_i)),
  1314. bygroups(Punctuation, Whitespace, Name.Function, Whitespace, Punctuation)),
  1315. (r'(\{{\{{)(\s*)({})(\s*)(\}}\}})'.format('|'.join(magic_vars)),
  1316. bygroups(Punctuation, Whitespace, Name.Function, Whitespace, Punctuation)),
  1317. # Parser functions & templates
  1318. (r'\{\{', Punctuation, 'template-begin-space'),
  1319. # <tvar> legacy syntax
  1320. (r'(?i)(<)(tvar)\b(\|)([^>]*?)(>)', bygroups(Punctuation,
  1321. Name.Tag, Punctuation, String, Punctuation)),
  1322. (r'</>', Punctuation, '#pop'),
  1323. # <tvar>
  1324. (r'(?i)(<)(tvar)\b', bygroups(Punctuation, Name.Tag), 'tag-inner-ordinary'),
  1325. (r'(?i)(</)(tvar)\b(\s*)(>)',
  1326. bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
  1327. ],
  1328. 'parameter-inner': [
  1329. (r'\}{3}', Punctuation, '#pop'),
  1330. (r'\|', Punctuation),
  1331. include('inline'),
  1332. include('text'),
  1333. ],
  1334. 'template-begin-space': [
  1335. # Templates allow line breaks at the beginning, and due to how MediaWiki handles
  1336. # comments, an extra state is required to handle things like {{\n<!---->\n name}}
  1337. (r'<!--[\s\S]*?(?:-->|\Z)', Comment.Multiline),
  1338. (r'\s+', Whitespace),
  1339. # Parser functions
  1340. (
  1341. r'(?i)(\#[{}]*?|{})(:)'.format(title_char,
  1342. '|'.join(parser_functions_i)),
  1343. bygroups(Name.Function, Punctuation), ('#pop', 'template-inner')
  1344. ),
  1345. (
  1346. r'({})(:)'.format('|'.join(parser_functions)),
  1347. bygroups(Name.Function, Punctuation), ('#pop', 'template-inner')
  1348. ),
  1349. # Templates
  1350. (
  1351. rf'(?i)([{title_char}]*?)(:)',
  1352. bygroups(Name.Namespace, Punctuation), ('#pop', 'template-name')
  1353. ),
  1354. default(('#pop', 'template-name'),),
  1355. ],
  1356. 'template-name': [
  1357. (r'(\s*?)(\|)', bygroups(Text, Punctuation), ('#pop', 'template-inner')),
  1358. (r'\}\}', Punctuation, '#pop'),
  1359. (r'\n', Text, '#pop'),
  1360. include('replaceable'),
  1361. *text_rules(Name.Tag),
  1362. ],
  1363. 'template-inner': [
  1364. (r'\}\}', Punctuation, '#pop'),
  1365. (r'\|', Punctuation),
  1366. (
  1367. r"""(?x)
  1368. (?<=\|)
  1369. ( (?: (?! \{\{ | \}\} )[^=\|<])*? ) # Exclude templates and tags
  1370. (=)
  1371. """,
  1372. bygroups(Name.Label, Operator)
  1373. ),
  1374. include('inline'),
  1375. include('text'),
  1376. ],
  1377. 'table': [
  1378. # Use [ \t\n\r\0\x0B] instead of \s to follow PHP trim() behavior
  1379. # Endings
  1380. (r'^([ \t\n\r\0\x0B]*?)(\|\})',
  1381. bygroups(Whitespace, Punctuation), '#pop'),
  1382. # Table rows
  1383. (r'^([ \t\n\r\0\x0B]*?)(\|-+)(.*)$', bygroups(Whitespace, Punctuation,
  1384. using(this, state=['root', 'attr']))),
  1385. # Captions
  1386. (
  1387. r"""(?x)
  1388. ^([ \t\n\r\0\x0B]*?)(\|\+)
  1389. # Exclude links, template and tags
  1390. (?: ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )(\|) )?
  1391. (.*?)$
  1392. """,
  1393. bygroups(Whitespace, Punctuation, using(this, state=[
  1394. 'root', 'attr']), Punctuation, Generic.Heading),
  1395. ),
  1396. # Table data
  1397. (
  1398. r"""(?x)
  1399. ( ^(?:[ \t\n\r\0\x0B]*?)\| | \|\| )
  1400. (?: ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )(\|)(?!\|) )?
  1401. """,
  1402. bygroups(Punctuation, using(this, state=[
  1403. 'root', 'attr']), Punctuation),
  1404. ),
  1405. # Table headers
  1406. (
  1407. r"""(?x)
  1408. ( ^(?:[ \t\n\r\0\x0B]*?)! )
  1409. (?: ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )(\|)(?!\|) )?
  1410. """,
  1411. bygroups(Punctuation, using(this, state=[
  1412. 'root', 'attr']), Punctuation),
  1413. 'table-header',
  1414. ),
  1415. include('list'),
  1416. include('inline'),
  1417. include('text'),
  1418. ],
  1419. 'table-header': [
  1420. # Requires another state for || handling inside headers
  1421. (r'\n', Text, '#pop'),
  1422. (
  1423. r"""(?x)
  1424. (!!|\|\|)
  1425. (?:
  1426. ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )
  1427. (\|)(?!\|)
  1428. )?
  1429. """,
  1430. bygroups(Punctuation, using(this, state=[
  1431. 'root', 'attr']), Punctuation)
  1432. ),
  1433. *text_rules(Generic.Subheading),
  1434. ],
  1435. 'entity': [
  1436. (r'&\S*?;', Name.Entity),
  1437. ],
  1438. 'dt': [
  1439. (r'\n', Text, '#pop'),
  1440. include('inline'),
  1441. (r':', Keyword, '#pop'),
  1442. include('text'),
  1443. ],
  1444. 'extlink-inner': [
  1445. (r'\]', Punctuation, '#pop'),
  1446. include('inline'),
  1447. include('text'),
  1448. ],
  1449. 'nowiki-ish': [
  1450. include('entity'),
  1451. include('text'),
  1452. ],
  1453. 'attr': [
  1454. include('replaceable'),
  1455. (r'\s+', Whitespace),
  1456. (r'(=)(\s*)(")', bygroups(Operator, Whitespace, String.Double), 'attr-val-2'),
  1457. (r"(=)(\s*)(')", bygroups(Operator, Whitespace, String.Single), 'attr-val-1'),
  1458. (r'(=)(\s*)', bygroups(Operator, Whitespace), 'attr-val-0'),
  1459. (r'[\w:-]+', Name.Attribute),
  1460. ],
  1461. 'attr-val-0': [
  1462. (r'\s', Whitespace, '#pop'),
  1463. include('replaceable'),
  1464. *text_rules(String),
  1465. ],
  1466. 'attr-val-1': [
  1467. (r"'", String.Single, '#pop'),
  1468. include('replaceable'),
  1469. *text_rules(String.Single),
  1470. ],
  1471. 'attr-val-2': [
  1472. (r'"', String.Double, '#pop'),
  1473. include('replaceable'),
  1474. *text_rules(String.Double),
  1475. ],
  1476. 'tag-inner-ordinary': [
  1477. (r'/?\s*>', Punctuation, '#pop'),
  1478. include('tag-attr'),
  1479. ],
  1480. 'tag-inner': [
  1481. # Return to root state for self-closing tags
  1482. (r'/\s*>', Punctuation, '#pop:2'),
  1483. (r'\s*>', Punctuation, '#pop'),
  1484. include('tag-attr'),
  1485. ],
  1486. # There states below are just like their non-tag variants, the key difference is
  1487. # they forcibly quit when encountering tag closing markup
  1488. 'tag-attr': [
  1489. include('replaceable'),
  1490. (r'\s+', Whitespace),
  1491. (r'(=)(\s*)(")', bygroups(Operator,
  1492. Whitespace, String.Double), 'tag-attr-val-2'),
  1493. (r"(=)(\s*)(')", bygroups(Operator,
  1494. Whitespace, String.Single), 'tag-attr-val-1'),
  1495. (r'(=)(\s*)', bygroups(Operator, Whitespace), 'tag-attr-val-0'),
  1496. (r'[\w:-]+', Name.Attribute),
  1497. ],
  1498. 'tag-attr-val-0': [
  1499. (r'\s', Whitespace, '#pop'),
  1500. (r'/?>', Punctuation, '#pop:2'),
  1501. include('replaceable'),
  1502. *text_rules(String),
  1503. ],
  1504. 'tag-attr-val-1': [
  1505. (r"'", String.Single, '#pop'),
  1506. (r'/?>', Punctuation, '#pop:2'),
  1507. include('replaceable'),
  1508. *text_rules(String.Single),
  1509. ],
  1510. 'tag-attr-val-2': [
  1511. (r'"', String.Double, '#pop'),
  1512. (r'/?>', Punctuation, '#pop:2'),
  1513. include('replaceable'),
  1514. *text_rules(String.Double),
  1515. ],
  1516. 'tag-nowiki': nowiki_tag_rules('nowiki'),
  1517. 'tag-pre': nowiki_tag_rules('pre'),
  1518. 'tag-categorytree': plaintext_tag_rules('categorytree'),
  1519. 'tag-dynamicpagelist': plaintext_tag_rules('dynamicpagelist'),
  1520. 'tag-hiero': plaintext_tag_rules('hiero'),
  1521. 'tag-inputbox': plaintext_tag_rules('inputbox'),
  1522. 'tag-imagemap': plaintext_tag_rules('imagemap'),
  1523. 'tag-charinsert': plaintext_tag_rules('charinsert'),
  1524. 'tag-timeline': plaintext_tag_rules('timeline'),
  1525. 'tag-gallery': plaintext_tag_rules('gallery'),
  1526. 'tag-graph': plaintext_tag_rules('graph'),
  1527. 'tag-rss': plaintext_tag_rules('rss'),
  1528. 'tag-math': delegate_tag_rules('math', TexLexer, state='math'),
  1529. 'tag-chem': delegate_tag_rules('chem', TexLexer, state='math'),
  1530. 'tag-ce': delegate_tag_rules('ce', TexLexer, state='math'),
  1531. 'tag-templatedata': delegate_tag_rules('templatedata', JsonLexer),
  1532. 'text-italic': text_rules(Generic.Emph),
  1533. 'text-bold': text_rules(Generic.Strong),
  1534. 'text-bold-italic': text_rules(Generic.EmphStrong),
  1535. 'text': text_rules(Text),
  1536. }