haskell.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867
  1. """
  2. pygments.lexers.haskell
  3. ~~~~~~~~~~~~~~~~~~~~~~~
  4. Lexers for Haskell and related languages.
  5. :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
  6. :license: BSD, see LICENSE for details.
  7. """
  8. import re
  9. from pygments.lexer import Lexer, RegexLexer, bygroups, do_insertions, \
  10. default, include, inherit, line_re
  11. from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
  12. Number, Punctuation, Generic, Whitespace
  13. from pygments import unistring as uni
  14. __all__ = ['HaskellLexer', 'HspecLexer', 'IdrisLexer', 'AgdaLexer', 'CryptolLexer',
  15. 'LiterateHaskellLexer', 'LiterateIdrisLexer', 'LiterateAgdaLexer',
  16. 'LiterateCryptolLexer', 'KokaLexer']
  17. class HaskellLexer(RegexLexer):
  18. """
  19. A Haskell lexer based on the lexemes defined in the Haskell 98 Report.
  20. """
  21. name = 'Haskell'
  22. url = 'https://www.haskell.org/'
  23. aliases = ['haskell', 'hs']
  24. filenames = ['*.hs']
  25. mimetypes = ['text/x-haskell']
  26. version_added = '0.8'
  27. reserved = ('case', 'class', 'data', 'default', 'deriving', 'do', 'else',
  28. 'family', 'if', 'in', 'infix[lr]?', 'instance',
  29. 'let', 'newtype', 'of', 'then', 'type', 'where', '_')
  30. ascii = ('NUL', 'SOH', '[SE]TX', 'EOT', 'ENQ', 'ACK',
  31. 'BEL', 'BS', 'HT', 'LF', 'VT', 'FF', 'CR', 'S[OI]', 'DLE',
  32. 'DC[1-4]', 'NAK', 'SYN', 'ETB', 'CAN',
  33. 'EM', 'SUB', 'ESC', '[FGRU]S', 'SP', 'DEL')
  34. tokens = {
  35. 'root': [
  36. # Whitespace:
  37. (r'\s+', Whitespace),
  38. # (r'--\s*|.*$', Comment.Doc),
  39. (r'--(?![!#$%&*+./<=>?@^|_~:\\]).*?$', Comment.Single),
  40. (r'\{-', Comment.Multiline, 'comment'),
  41. # Lexemes:
  42. # Identifiers
  43. (r'\bimport\b', Keyword.Reserved, 'import'),
  44. (r'\bmodule\b', Keyword.Reserved, 'module'),
  45. (r'\berror\b', Name.Exception),
  46. (r'\b({})(?!\')\b'.format('|'.join(reserved)), Keyword.Reserved),
  47. (r"'[^\\]'", String.Char), # character literal
  48. (r"'\\.'", String.Char), # escape character literal (e.g. '\n')
  49. (r'^[_' + uni.Ll + r'][\w\']*', Name.Function),
  50. (r"'?[_" + uni.Ll + r"][\w']*", Name),
  51. (r"('')?[" + uni.Lu + r"][\w\']*", Keyword.Type),
  52. (r"(')[" + uni.Lu + r"][\w\']*", Keyword.Type),
  53. (r"(')\[[^\]]*\]", Keyword.Type), # tuples and lists get special treatment in GHC
  54. (r"(')\([^)]*\)", Keyword.Type), # ..
  55. (r"(')[:!#$%&*+.\\/<=>?@^|~-]+", Keyword.Type), # promoted type operators
  56. # Operators
  57. (r'\\(?![:!#$%&*+.\\/<=>?@^|~-]+)', Name.Function), # lambda operator
  58. (r'(<-|::|->|=>|=)(?![:!#$%&*+.\\/<=>?@^|~-]+)', Operator.Word), # specials
  59. (r':[:!#$%&*+.\\/<=>?@^|~-]*', Keyword.Type), # Constructor operators
  60. (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator), # Other operators
  61. # Numbers
  62. (r'0[xX]_*[\da-fA-F](_*[\da-fA-F])*_*[pP][+-]?\d(_*\d)*', Number.Float),
  63. (r'0[xX]_*[\da-fA-F](_*[\da-fA-F])*\.[\da-fA-F](_*[\da-fA-F])*'
  64. r'(_*[pP][+-]?\d(_*\d)*)?', Number.Float),
  65. (r'\d(_*\d)*_*[eE][+-]?\d(_*\d)*', Number.Float),
  66. (r'\d(_*\d)*\.\d(_*\d)*(_*[eE][+-]?\d(_*\d)*)?', Number.Float),
  67. (r'0[bB]_*[01](_*[01])*', Number.Bin),
  68. (r'0[oO]_*[0-7](_*[0-7])*', Number.Oct),
  69. (r'0[xX]_*[\da-fA-F](_*[\da-fA-F])*', Number.Hex),
  70. (r'\d(_*\d)*', Number.Integer),
  71. # Character/String Literals
  72. (r"'", String.Char, 'character'),
  73. (r'"', String, 'string'),
  74. # Special
  75. (r'\[\]', Keyword.Type),
  76. (r'\(\)', Name.Builtin),
  77. (r'[][(),;`{}]', Punctuation),
  78. ],
  79. 'import': [
  80. # Import statements
  81. (r'\s+', Whitespace),
  82. (r'"', String, 'string'),
  83. # after "funclist" state
  84. (r'\)', Punctuation, '#pop'),
  85. (r'qualified\b', Keyword),
  86. # import X as Y
  87. (r'([' + uni.Lu + r'][\w.]*)(\s+)(as)(\s+)([' + uni.Lu + r'][\w.]*)',
  88. bygroups(Name.Namespace, Whitespace, Keyword, Whitespace, Name), '#pop'),
  89. # import X hiding (functions)
  90. (r'([' + uni.Lu + r'][\w.]*)(\s+)(hiding)(\s+)(\()',
  91. bygroups(Name.Namespace, Whitespace, Keyword, Whitespace, Punctuation), 'funclist'),
  92. # import X (functions)
  93. (r'([' + uni.Lu + r'][\w.]*)(\s+)(\()',
  94. bygroups(Name.Namespace, Whitespace, Punctuation), 'funclist'),
  95. # import X
  96. (r'[\w.]+', Name.Namespace, '#pop'),
  97. ],
  98. 'module': [
  99. (r'\s+', Whitespace),
  100. (r'([' + uni.Lu + r'][\w.]*)(\s+)(\()',
  101. bygroups(Name.Namespace, Whitespace, Punctuation), 'funclist'),
  102. (r'[' + uni.Lu + r'][\w.]*', Name.Namespace, '#pop'),
  103. ],
  104. 'funclist': [
  105. (r'\s+', Whitespace),
  106. (r'[' + uni.Lu + r']\w*', Keyword.Type),
  107. (r'(_[\w\']+|[' + uni.Ll + r'][\w\']*)', Name.Function),
  108. (r'--(?![!#$%&*+./<=>?@^|_~:\\]).*?$', Comment.Single),
  109. (r'\{-', Comment.Multiline, 'comment'),
  110. (r',', Punctuation),
  111. (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator),
  112. # (HACK, but it makes sense to push two instances, believe me)
  113. (r'\(', Punctuation, ('funclist', 'funclist')),
  114. (r'\)', Punctuation, '#pop:2'),
  115. ],
  116. # NOTE: the next four states are shared in the AgdaLexer; make sure
  117. # any change is compatible with Agda as well or copy over and change
  118. 'comment': [
  119. # Multiline Comments
  120. (r'[^-{}]+', Comment.Multiline),
  121. (r'\{-', Comment.Multiline, '#push'),
  122. (r'-\}', Comment.Multiline, '#pop'),
  123. (r'[-{}]', Comment.Multiline),
  124. ],
  125. 'character': [
  126. # Allows multi-chars, incorrectly.
  127. (r"[^\\']'", String.Char, '#pop'),
  128. (r"\\", String.Escape, 'escape'),
  129. ("'", String.Char, '#pop'),
  130. ],
  131. 'string': [
  132. (r'[^\\"]+', String),
  133. (r"\\", String.Escape, 'escape'),
  134. ('"', String, '#pop'),
  135. ],
  136. 'escape': [
  137. (r'[abfnrtv"\'&\\]', String.Escape, '#pop'),
  138. (r'\^[][' + uni.Lu + r'@^_]', String.Escape, '#pop'),
  139. ('|'.join(ascii), String.Escape, '#pop'),
  140. (r'o[0-7]+', String.Escape, '#pop'),
  141. (r'x[\da-fA-F]+', String.Escape, '#pop'),
  142. (r'\d+', String.Escape, '#pop'),
  143. (r'(\s+)(\\)', bygroups(Whitespace, String.Escape), '#pop'),
  144. ],
  145. }
  146. class HspecLexer(HaskellLexer):
  147. """
  148. A Haskell lexer with support for Hspec constructs.
  149. """
  150. name = 'Hspec'
  151. aliases = ['hspec']
  152. filenames = ['*Spec.hs']
  153. mimetypes = []
  154. version_added = '2.4'
  155. tokens = {
  156. 'root': [
  157. (r'(it)(\s*)("[^"]*")', bygroups(Text, Whitespace, String.Doc)),
  158. (r'(describe)(\s*)("[^"]*")', bygroups(Text, Whitespace, String.Doc)),
  159. (r'(context)(\s*)("[^"]*")', bygroups(Text, Whitespace, String.Doc)),
  160. inherit,
  161. ],
  162. }
  163. class IdrisLexer(RegexLexer):
  164. """
  165. A lexer for the dependently typed programming language Idris.
  166. Based on the Haskell and Agda Lexer.
  167. """
  168. name = 'Idris'
  169. url = 'https://www.idris-lang.org/'
  170. aliases = ['idris', 'idr']
  171. filenames = ['*.idr']
  172. mimetypes = ['text/x-idris']
  173. version_added = '2.0'
  174. reserved = ('case', 'class', 'data', 'default', 'using', 'do', 'else',
  175. 'if', 'in', 'infix[lr]?', 'instance', 'rewrite', 'auto',
  176. 'namespace', 'codata', 'mutual', 'private', 'public', 'abstract',
  177. 'total', 'partial',
  178. 'interface', 'implementation', 'export', 'covering', 'constructor',
  179. 'let', 'proof', 'of', 'then', 'static', 'where', '_', 'with',
  180. 'pattern', 'term', 'syntax', 'prefix',
  181. 'postulate', 'parameters', 'record', 'dsl', 'impossible', 'implicit',
  182. 'tactics', 'intros', 'intro', 'compute', 'refine', 'exact', 'trivial')
  183. ascii = ('NUL', 'SOH', '[SE]TX', 'EOT', 'ENQ', 'ACK',
  184. 'BEL', 'BS', 'HT', 'LF', 'VT', 'FF', 'CR', 'S[OI]', 'DLE',
  185. 'DC[1-4]', 'NAK', 'SYN', 'ETB', 'CAN',
  186. 'EM', 'SUB', 'ESC', '[FGRU]S', 'SP', 'DEL')
  187. directives = ('lib', 'link', 'flag', 'include', 'hide', 'freeze', 'access',
  188. 'default', 'logging', 'dynamic', 'name', 'error_handlers', 'language')
  189. tokens = {
  190. 'root': [
  191. # Comments
  192. (r'^(\s*)(%({}))'.format('|'.join(directives)),
  193. bygroups(Whitespace, Keyword.Reserved)),
  194. (r'(\s*)(--(?![!#$%&*+./<=>?@^|_~:\\]).*?)$', bygroups(Whitespace, Comment.Single)),
  195. (r'(\s*)(\|{3}.*?)$', bygroups(Whitespace, Comment.Single)),
  196. (r'(\s*)(\{-)', bygroups(Whitespace, Comment.Multiline), 'comment'),
  197. # Declaration
  198. (r'^(\s*)([^\s(){}]+)(\s*)(:)(\s*)',
  199. bygroups(Whitespace, Name.Function, Whitespace, Operator.Word, Whitespace)),
  200. # Identifiers
  201. (r'\b({})(?!\')\b'.format('|'.join(reserved)), Keyword.Reserved),
  202. (r'(import|module)(\s+)', bygroups(Keyword.Reserved, Whitespace), 'module'),
  203. (r"('')?[A-Z][\w\']*", Keyword.Type),
  204. (r'[a-z][\w\']*', Text),
  205. # Special Symbols
  206. (r'(<-|::|->|=>|=)', Operator.Word), # specials
  207. (r'([(){}\[\]:!#$%&*+.\\/<=>?@^|~-]+)', Operator.Word), # specials
  208. # Numbers
  209. (r'\d+[eE][+-]?\d+', Number.Float),
  210. (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float),
  211. (r'0[xX][\da-fA-F]+', Number.Hex),
  212. (r'\d+', Number.Integer),
  213. # Strings
  214. (r"'", String.Char, 'character'),
  215. (r'"', String, 'string'),
  216. (r'[^\s(){}]+', Text),
  217. (r'\s+?', Whitespace), # Whitespace
  218. ],
  219. 'module': [
  220. (r'\s+', Whitespace),
  221. (r'([A-Z][\w.]*)(\s+)(\()',
  222. bygroups(Name.Namespace, Whitespace, Punctuation), 'funclist'),
  223. (r'[A-Z][\w.]*', Name.Namespace, '#pop'),
  224. ],
  225. 'funclist': [
  226. (r'\s+', Whitespace),
  227. (r'[A-Z]\w*', Keyword.Type),
  228. (r'(_[\w\']+|[a-z][\w\']*)', Name.Function),
  229. (r'--.*$', Comment.Single),
  230. (r'\{-', Comment.Multiline, 'comment'),
  231. (r',', Punctuation),
  232. (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator),
  233. # (HACK, but it makes sense to push two instances, believe me)
  234. (r'\(', Punctuation, ('funclist', 'funclist')),
  235. (r'\)', Punctuation, '#pop:2'),
  236. ],
  237. # NOTE: the next four states are shared in the AgdaLexer; make sure
  238. # any change is compatible with Agda as well or copy over and change
  239. 'comment': [
  240. # Multiline Comments
  241. (r'[^-{}]+', Comment.Multiline),
  242. (r'\{-', Comment.Multiline, '#push'),
  243. (r'-\}', Comment.Multiline, '#pop'),
  244. (r'[-{}]', Comment.Multiline),
  245. ],
  246. 'character': [
  247. # Allows multi-chars, incorrectly.
  248. (r"[^\\']", String.Char),
  249. (r"\\", String.Escape, 'escape'),
  250. ("'", String.Char, '#pop'),
  251. ],
  252. 'string': [
  253. (r'[^\\"]+', String),
  254. (r"\\", String.Escape, 'escape'),
  255. ('"', String, '#pop'),
  256. ],
  257. 'escape': [
  258. (r'[abfnrtv"\'&\\]', String.Escape, '#pop'),
  259. (r'\^[][A-Z@^_]', String.Escape, '#pop'),
  260. ('|'.join(ascii), String.Escape, '#pop'),
  261. (r'o[0-7]+', String.Escape, '#pop'),
  262. (r'x[\da-fA-F]+', String.Escape, '#pop'),
  263. (r'\d+', String.Escape, '#pop'),
  264. (r'(\s+)(\\)', bygroups(Whitespace, String.Escape), '#pop')
  265. ],
  266. }
  267. class AgdaLexer(RegexLexer):
  268. """
  269. For the Agda dependently typed functional programming language and
  270. proof assistant.
  271. """
  272. name = 'Agda'
  273. url = 'http://wiki.portal.chalmers.se/agda/pmwiki.php'
  274. aliases = ['agda']
  275. filenames = ['*.agda']
  276. mimetypes = ['text/x-agda']
  277. version_added = '2.0'
  278. reserved = (
  279. 'abstract', 'codata', 'coinductive', 'constructor', 'data', 'do',
  280. 'eta-equality', 'field', 'forall', 'hiding', 'in', 'inductive', 'infix',
  281. 'infixl', 'infixr', 'instance', 'interleaved', 'let', 'macro', 'mutual',
  282. 'no-eta-equality', 'opaque', 'open', 'overlap', 'pattern', 'postulate', 'primitive',
  283. 'private', 'quote', 'quoteTerm', 'record', 'renaming', 'rewrite',
  284. 'syntax', 'tactic', 'unfolding', 'unquote', 'unquoteDecl', 'unquoteDef', 'using',
  285. 'variable', 'where', 'with',
  286. )
  287. tokens = {
  288. 'root': [
  289. # Declaration
  290. (r'^(\s*)([^\s(){}]+)(\s*)(:)(\s*)',
  291. bygroups(Whitespace, Name.Function, Whitespace,
  292. Operator.Word, Whitespace)),
  293. # Comments
  294. (r'--(?![!#$%&*+./<=>?@^|_~:\\]).*?$', Comment.Single),
  295. (r'\{-', Comment.Multiline, 'comment'),
  296. # Holes
  297. (r'\{!', Comment.Directive, 'hole'),
  298. # Lexemes:
  299. # Identifiers
  300. (r'\b({})(?!\')\b'.format('|'.join(reserved)), Keyword.Reserved),
  301. (r'(import|module)(\s+)', bygroups(Keyword.Reserved, Whitespace),
  302. 'module'),
  303. (r'\b(Set|Prop)[\u2080-\u2089]*\b', Keyword.Type),
  304. # Special Symbols
  305. (r'(\(|\)|\{|\})', Operator),
  306. (r'(\.{1,3}|\||\u03BB|\u2200|\u2192|:|=|->)', Operator.Word),
  307. # Numbers
  308. (r'\d+[eE][+-]?\d+', Number.Float),
  309. (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float),
  310. (r'0[xX][\da-fA-F]+', Number.Hex),
  311. (r'\d+', Number.Integer),
  312. # Strings
  313. (r"'", String.Char, 'character'),
  314. (r'"', String, 'string'),
  315. (r'[^\s(){}]+', Text),
  316. (r'\s+?', Whitespace), # Whitespace
  317. ],
  318. 'hole': [
  319. # Holes
  320. (r'[^!{}]+', Comment.Directive),
  321. (r'\{!', Comment.Directive, '#push'),
  322. (r'!\}', Comment.Directive, '#pop'),
  323. (r'[!{}]', Comment.Directive),
  324. ],
  325. 'module': [
  326. (r'\{-', Comment.Multiline, 'comment'),
  327. (r'[a-zA-Z][\w.\']*', Name, '#pop'),
  328. (r'[\W0-9_]+', Text)
  329. ],
  330. 'comment': HaskellLexer.tokens['comment'],
  331. 'character': HaskellLexer.tokens['character'],
  332. 'string': HaskellLexer.tokens['string'],
  333. 'escape': HaskellLexer.tokens['escape']
  334. }
  335. class CryptolLexer(RegexLexer):
  336. """
  337. FIXME: A Cryptol2 lexer based on the lexemes defined in the Haskell 98 Report.
  338. """
  339. name = 'Cryptol'
  340. aliases = ['cryptol', 'cry']
  341. filenames = ['*.cry']
  342. mimetypes = ['text/x-cryptol']
  343. url = 'https://www.cryptol.net'
  344. version_added = '2.0'
  345. reserved = ('Arith', 'Bit', 'Cmp', 'False', 'Inf', 'True', 'else',
  346. 'export', 'extern', 'fin', 'if', 'import', 'inf', 'lg2',
  347. 'max', 'min', 'module', 'newtype', 'pragma', 'property',
  348. 'then', 'type', 'where', 'width')
  349. ascii = ('NUL', 'SOH', '[SE]TX', 'EOT', 'ENQ', 'ACK',
  350. 'BEL', 'BS', 'HT', 'LF', 'VT', 'FF', 'CR', 'S[OI]', 'DLE',
  351. 'DC[1-4]', 'NAK', 'SYN', 'ETB', 'CAN',
  352. 'EM', 'SUB', 'ESC', '[FGRU]S', 'SP', 'DEL')
  353. tokens = {
  354. 'root': [
  355. # Whitespace:
  356. (r'\s+', Whitespace),
  357. # (r'--\s*|.*$', Comment.Doc),
  358. (r'//.*$', Comment.Single),
  359. (r'/\*', Comment.Multiline, 'comment'),
  360. # Lexemes:
  361. # Identifiers
  362. (r'\bimport\b', Keyword.Reserved, 'import'),
  363. (r'\bmodule\b', Keyword.Reserved, 'module'),
  364. (r'\berror\b', Name.Exception),
  365. (r'\b({})(?!\')\b'.format('|'.join(reserved)), Keyword.Reserved),
  366. (r'^[_a-z][\w\']*', Name.Function),
  367. (r"'?[_a-z][\w']*", Name),
  368. (r"('')?[A-Z][\w\']*", Keyword.Type),
  369. # Operators
  370. (r'\\(?![:!#$%&*+.\\/<=>?@^|~-]+)', Name.Function), # lambda operator
  371. (r'(<-|::|->|=>|=)(?![:!#$%&*+.\\/<=>?@^|~-]+)', Operator.Word), # specials
  372. (r':[:!#$%&*+.\\/<=>?@^|~-]*', Keyword.Type), # Constructor operators
  373. (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator), # Other operators
  374. # Numbers
  375. (r'\d+[eE][+-]?\d+', Number.Float),
  376. (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float),
  377. (r'0[oO][0-7]+', Number.Oct),
  378. (r'0[xX][\da-fA-F]+', Number.Hex),
  379. (r'\d+', Number.Integer),
  380. # Character/String Literals
  381. (r"'", String.Char, 'character'),
  382. (r'"', String, 'string'),
  383. # Special
  384. (r'\[\]', Keyword.Type),
  385. (r'\(\)', Name.Builtin),
  386. (r'[][(),;`{}]', Punctuation),
  387. ],
  388. 'import': [
  389. # Import statements
  390. (r'\s+', Whitespace),
  391. (r'"', String, 'string'),
  392. # after "funclist" state
  393. (r'\)', Punctuation, '#pop'),
  394. (r'qualified\b', Keyword),
  395. # import X as Y
  396. (r'([A-Z][\w.]*)(\s+)(as)(\s+)([A-Z][\w.]*)',
  397. bygroups(Name.Namespace, Whitespace, Keyword, Whitespace, Name), '#pop'),
  398. # import X hiding (functions)
  399. (r'([A-Z][\w.]*)(\s+)(hiding)(\s+)(\()',
  400. bygroups(Name.Namespace, Whitespace, Keyword, Whitespace, Punctuation), 'funclist'),
  401. # import X (functions)
  402. (r'([A-Z][\w.]*)(\s+)(\()',
  403. bygroups(Name.Namespace, Whitespace, Punctuation), 'funclist'),
  404. # import X
  405. (r'[\w.]+', Name.Namespace, '#pop'),
  406. ],
  407. 'module': [
  408. (r'\s+', Whitespace),
  409. (r'([A-Z][\w.]*)(\s+)(\()',
  410. bygroups(Name.Namespace, Whitespace, Punctuation), 'funclist'),
  411. (r'[A-Z][\w.]*', Name.Namespace, '#pop'),
  412. ],
  413. 'funclist': [
  414. (r'\s+', Whitespace),
  415. (r'[A-Z]\w*', Keyword.Type),
  416. (r'(_[\w\']+|[a-z][\w\']*)', Name.Function),
  417. # TODO: these don't match the comments in docs, remove.
  418. # (r'--(?![!#$%&*+./<=>?@^|_~:\\]).*?$', Comment.Single),
  419. # (r'{-', Comment.Multiline, 'comment'),
  420. (r',', Punctuation),
  421. (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator),
  422. # (HACK, but it makes sense to push two instances, believe me)
  423. (r'\(', Punctuation, ('funclist', 'funclist')),
  424. (r'\)', Punctuation, '#pop:2'),
  425. ],
  426. 'comment': [
  427. # Multiline Comments
  428. (r'[^/*]+', Comment.Multiline),
  429. (r'/\*', Comment.Multiline, '#push'),
  430. (r'\*/', Comment.Multiline, '#pop'),
  431. (r'[*/]', Comment.Multiline),
  432. ],
  433. 'character': [
  434. # Allows multi-chars, incorrectly.
  435. (r"[^\\']'", String.Char, '#pop'),
  436. (r"\\", String.Escape, 'escape'),
  437. ("'", String.Char, '#pop'),
  438. ],
  439. 'string': [
  440. (r'[^\\"]+', String),
  441. (r"\\", String.Escape, 'escape'),
  442. ('"', String, '#pop'),
  443. ],
  444. 'escape': [
  445. (r'[abfnrtv"\'&\\]', String.Escape, '#pop'),
  446. (r'\^[][A-Z@^_]', String.Escape, '#pop'),
  447. ('|'.join(ascii), String.Escape, '#pop'),
  448. (r'o[0-7]+', String.Escape, '#pop'),
  449. (r'x[\da-fA-F]+', String.Escape, '#pop'),
  450. (r'\d+', String.Escape, '#pop'),
  451. (r'(\s+)(\\)', bygroups(Whitespace, String.Escape), '#pop'),
  452. ],
  453. }
  454. EXTRA_KEYWORDS = {'join', 'split', 'reverse', 'transpose', 'width',
  455. 'length', 'tail', '<<', '>>', '<<<', '>>>', 'const',
  456. 'reg', 'par', 'seq', 'ASSERT', 'undefined', 'error',
  457. 'trace'}
  458. def get_tokens_unprocessed(self, text):
  459. stack = ['root']
  460. for index, token, value in \
  461. RegexLexer.get_tokens_unprocessed(self, text, stack):
  462. if token is Name and value in self.EXTRA_KEYWORDS:
  463. yield index, Name.Builtin, value
  464. else:
  465. yield index, token, value
  466. class LiterateLexer(Lexer):
  467. """
  468. Base class for lexers of literate file formats based on LaTeX or Bird-style
  469. (prefixing each code line with ">").
  470. Additional options accepted:
  471. `litstyle`
  472. If given, must be ``"bird"`` or ``"latex"``. If not given, the style
  473. is autodetected: if the first non-whitespace character in the source
  474. is a backslash or percent character, LaTeX is assumed, else Bird.
  475. """
  476. bird_re = re.compile(r'(>[ \t]*)(.*\n)')
  477. def __init__(self, baselexer, **options):
  478. self.baselexer = baselexer
  479. Lexer.__init__(self, **options)
  480. def get_tokens_unprocessed(self, text):
  481. style = self.options.get('litstyle')
  482. if style is None:
  483. style = (text.lstrip()[0:1] in '%\\') and 'latex' or 'bird'
  484. code = ''
  485. insertions = []
  486. if style == 'bird':
  487. # bird-style
  488. for match in line_re.finditer(text):
  489. line = match.group()
  490. m = self.bird_re.match(line)
  491. if m:
  492. insertions.append((len(code),
  493. [(0, Comment.Special, m.group(1))]))
  494. code += m.group(2)
  495. else:
  496. insertions.append((len(code), [(0, Text, line)]))
  497. else:
  498. # latex-style
  499. from pygments.lexers.markup import TexLexer
  500. lxlexer = TexLexer(**self.options)
  501. codelines = 0
  502. latex = ''
  503. for match in line_re.finditer(text):
  504. line = match.group()
  505. if codelines:
  506. if line.lstrip().startswith('\\end{code}'):
  507. codelines = 0
  508. latex += line
  509. else:
  510. code += line
  511. elif line.lstrip().startswith('\\begin{code}'):
  512. codelines = 1
  513. latex += line
  514. insertions.append((len(code),
  515. list(lxlexer.get_tokens_unprocessed(latex))))
  516. latex = ''
  517. else:
  518. latex += line
  519. insertions.append((len(code),
  520. list(lxlexer.get_tokens_unprocessed(latex))))
  521. yield from do_insertions(insertions, self.baselexer.get_tokens_unprocessed(code))
  522. class LiterateHaskellLexer(LiterateLexer):
  523. """
  524. For Literate Haskell (Bird-style or LaTeX) source.
  525. Additional options accepted:
  526. `litstyle`
  527. If given, must be ``"bird"`` or ``"latex"``. If not given, the style
  528. is autodetected: if the first non-whitespace character in the source
  529. is a backslash or percent character, LaTeX is assumed, else Bird.
  530. """
  531. name = 'Literate Haskell'
  532. aliases = ['literate-haskell', 'lhaskell', 'lhs']
  533. filenames = ['*.lhs']
  534. mimetypes = ['text/x-literate-haskell']
  535. url = 'https://wiki.haskell.org/Literate_programming'
  536. version_added = '0.9'
  537. def __init__(self, **options):
  538. hslexer = HaskellLexer(**options)
  539. LiterateLexer.__init__(self, hslexer, **options)
  540. class LiterateIdrisLexer(LiterateLexer):
  541. """
  542. For Literate Idris (Bird-style or LaTeX) source.
  543. Additional options accepted:
  544. `litstyle`
  545. If given, must be ``"bird"`` or ``"latex"``. If not given, the style
  546. is autodetected: if the first non-whitespace character in the source
  547. is a backslash or percent character, LaTeX is assumed, else Bird.
  548. """
  549. name = 'Literate Idris'
  550. aliases = ['literate-idris', 'lidris', 'lidr']
  551. filenames = ['*.lidr']
  552. mimetypes = ['text/x-literate-idris']
  553. url = 'https://idris2.readthedocs.io/en/latest/reference/literate.html'
  554. version_added = '2.0'
  555. def __init__(self, **options):
  556. hslexer = IdrisLexer(**options)
  557. LiterateLexer.__init__(self, hslexer, **options)
  558. class LiterateAgdaLexer(LiterateLexer):
  559. """
  560. For Literate Agda source.
  561. Additional options accepted:
  562. `litstyle`
  563. If given, must be ``"bird"`` or ``"latex"``. If not given, the style
  564. is autodetected: if the first non-whitespace character in the source
  565. is a backslash or percent character, LaTeX is assumed, else Bird.
  566. """
  567. name = 'Literate Agda'
  568. aliases = ['literate-agda', 'lagda']
  569. filenames = ['*.lagda']
  570. mimetypes = ['text/x-literate-agda']
  571. url = 'https://agda.readthedocs.io/en/latest/tools/literate-programming.html'
  572. version_added = '2.0'
  573. def __init__(self, **options):
  574. agdalexer = AgdaLexer(**options)
  575. LiterateLexer.__init__(self, agdalexer, litstyle='latex', **options)
  576. class LiterateCryptolLexer(LiterateLexer):
  577. """
  578. For Literate Cryptol (Bird-style or LaTeX) source.
  579. Additional options accepted:
  580. `litstyle`
  581. If given, must be ``"bird"`` or ``"latex"``. If not given, the style
  582. is autodetected: if the first non-whitespace character in the source
  583. is a backslash or percent character, LaTeX is assumed, else Bird.
  584. """
  585. name = 'Literate Cryptol'
  586. aliases = ['literate-cryptol', 'lcryptol', 'lcry']
  587. filenames = ['*.lcry']
  588. mimetypes = ['text/x-literate-cryptol']
  589. url = 'https://www.cryptol.net'
  590. version_added = '2.0'
  591. def __init__(self, **options):
  592. crylexer = CryptolLexer(**options)
  593. LiterateLexer.__init__(self, crylexer, **options)
  594. class KokaLexer(RegexLexer):
  595. """
  596. Lexer for the Koka language.
  597. """
  598. name = 'Koka'
  599. url = 'https://koka-lang.github.io/koka/doc/index.html'
  600. aliases = ['koka']
  601. filenames = ['*.kk', '*.kki']
  602. mimetypes = ['text/x-koka']
  603. version_added = '1.6'
  604. keywords = [
  605. 'infix', 'infixr', 'infixl',
  606. 'type', 'cotype', 'rectype', 'alias',
  607. 'struct', 'con',
  608. 'fun', 'function', 'val', 'var',
  609. 'external',
  610. 'if', 'then', 'else', 'elif', 'return', 'match',
  611. 'private', 'public', 'private',
  612. 'module', 'import', 'as',
  613. 'include', 'inline',
  614. 'rec',
  615. 'try', 'yield', 'enum',
  616. 'interface', 'instance',
  617. ]
  618. # keywords that are followed by a type
  619. typeStartKeywords = [
  620. 'type', 'cotype', 'rectype', 'alias', 'struct', 'enum',
  621. ]
  622. # keywords valid in a type
  623. typekeywords = [
  624. 'forall', 'exists', 'some', 'with',
  625. ]
  626. # builtin names and special names
  627. builtin = [
  628. 'for', 'while', 'repeat',
  629. 'foreach', 'foreach-indexed',
  630. 'error', 'catch', 'finally',
  631. 'cs', 'js', 'file', 'ref', 'assigned',
  632. ]
  633. # symbols that can be in an operator
  634. symbols = r'[$%&*+@!/\\^~=.:\-?|<>]+'
  635. # symbol boundary: an operator keyword should not be followed by any of these
  636. sboundary = '(?!' + symbols + ')'
  637. # name boundary: a keyword should not be followed by any of these
  638. boundary = r'(?![\w/])'
  639. # koka token abstractions
  640. tokenType = Name.Attribute
  641. tokenTypeDef = Name.Class
  642. tokenConstructor = Generic.Emph
  643. # main lexer
  644. tokens = {
  645. 'root': [
  646. include('whitespace'),
  647. # go into type mode
  648. (r'::?' + sboundary, tokenType, 'type'),
  649. (r'(alias)(\s+)([a-z]\w*)?', bygroups(Keyword, Whitespace, tokenTypeDef),
  650. 'alias-type'),
  651. (r'(struct)(\s+)([a-z]\w*)?', bygroups(Keyword, Whitespace, tokenTypeDef),
  652. 'struct-type'),
  653. ((r'({})'.format('|'.join(typeStartKeywords))) +
  654. r'(\s+)([a-z]\w*)?', bygroups(Keyword, Whitespace, tokenTypeDef),
  655. 'type'),
  656. # special sequences of tokens (we use ?: for non-capturing group as
  657. # required by 'bygroups')
  658. (r'(module)(\s+)(interface(?=\s))?(\s+)?((?:[a-z]\w*/)*[a-z]\w*)',
  659. bygroups(Keyword, Whitespace, Keyword, Whitespace, Name.Namespace)),
  660. (r'(import)(\s+)((?:[a-z]\w*/)*[a-z]\w*)'
  661. r'(?:(\s*)(=)(\s*)(qualified)?(\s*)'
  662. r'((?:[a-z]\w*/)*[a-z]\w*))?',
  663. bygroups(Keyword, Whitespace, Name.Namespace, Whitespace, Keyword, Whitespace,
  664. Keyword, Whitespace, Name.Namespace)),
  665. (r'^(public|private)?(\s+)?(function|fun|val)'
  666. r'(\s+)([a-z]\w*|\((?:' + symbols + r'|/)\))',
  667. bygroups(Keyword, Whitespace, Keyword, Whitespace, Name.Function)),
  668. (r'^(?:(public|private)(?=\s+external))?((?<!^)\s+)?(external)(\s+)(inline(?=\s))?(\s+)?'
  669. r'([a-z]\w*|\((?:' + symbols + r'|/)\))',
  670. bygroups(Keyword, Whitespace, Keyword, Whitespace, Keyword, Whitespace, Name.Function)),
  671. # keywords
  672. (r'({})'.format('|'.join(typekeywords)) + boundary, Keyword.Type),
  673. (r'({})'.format('|'.join(keywords)) + boundary, Keyword),
  674. (r'({})'.format('|'.join(builtin)) + boundary, Keyword.Pseudo),
  675. (r'::?|:=|\->|[=.]' + sboundary, Keyword),
  676. # names
  677. (r'((?:[a-z]\w*/)*)([A-Z]\w*)',
  678. bygroups(Name.Namespace, tokenConstructor)),
  679. (r'((?:[a-z]\w*/)*)([a-z]\w*)', bygroups(Name.Namespace, Name)),
  680. (r'((?:[a-z]\w*/)*)(\((?:' + symbols + r'|/)\))',
  681. bygroups(Name.Namespace, Name)),
  682. (r'_\w*', Name.Variable),
  683. # literal string
  684. (r'@"', String.Double, 'litstring'),
  685. # operators
  686. (symbols + "|/(?![*/])", Operator),
  687. (r'`', Operator),
  688. (r'[{}()\[\];,]', Punctuation),
  689. # literals. No check for literal characters with len > 1
  690. (r'[0-9]+\.[0-9]+([eE][\-+]?[0-9]+)?', Number.Float),
  691. (r'0[xX][0-9a-fA-F]+', Number.Hex),
  692. (r'[0-9]+', Number.Integer),
  693. (r"'", String.Char, 'char'),
  694. (r'"', String.Double, 'string'),
  695. ],
  696. # type started by alias
  697. 'alias-type': [
  698. (r'=', Keyword),
  699. include('type')
  700. ],
  701. # type started by struct
  702. 'struct-type': [
  703. (r'(?=\((?!,*\)))', Punctuation, '#pop'),
  704. include('type')
  705. ],
  706. # type started by colon
  707. 'type': [
  708. (r'[(\[<]', tokenType, 'type-nested'),
  709. include('type-content')
  710. ],
  711. # type nested in brackets: can contain parameters, comma etc.
  712. 'type-nested': [
  713. (r'[)\]>]', tokenType, '#pop'),
  714. (r'[(\[<]', tokenType, 'type-nested'),
  715. (r',', tokenType),
  716. (r'([a-z]\w*)(\s*)(:)(?!:)',
  717. bygroups(Name, Whitespace, tokenType)), # parameter name
  718. include('type-content')
  719. ],
  720. # shared contents of a type
  721. 'type-content': [
  722. include('whitespace'),
  723. # keywords
  724. (r'({})'.format('|'.join(typekeywords)) + boundary, Keyword),
  725. (r'(?=(({})'.format('|'.join(keywords)) + boundary + '))',
  726. Keyword, '#pop'), # need to match because names overlap...
  727. # kinds
  728. (r'[EPHVX]' + boundary, tokenType),
  729. # type names
  730. (r'[a-z][0-9]*(?![\w/])', tokenType),
  731. (r'_\w*', tokenType.Variable), # Generic.Emph
  732. (r'((?:[a-z]\w*/)*)([A-Z]\w*)',
  733. bygroups(Name.Namespace, tokenType)),
  734. (r'((?:[a-z]\w*/)*)([a-z]\w+)',
  735. bygroups(Name.Namespace, tokenType)),
  736. # type keyword operators
  737. (r'::|->|[.:|]', tokenType),
  738. # catchall
  739. default('#pop')
  740. ],
  741. # comments and literals
  742. 'whitespace': [
  743. (r'(\n\s*)(#.*)$', bygroups(Whitespace, Comment.Preproc)),
  744. (r'\s+', Whitespace),
  745. (r'/\*', Comment.Multiline, 'comment'),
  746. (r'//.*$', Comment.Single)
  747. ],
  748. 'comment': [
  749. (r'[^/*]+', Comment.Multiline),
  750. (r'/\*', Comment.Multiline, '#push'),
  751. (r'\*/', Comment.Multiline, '#pop'),
  752. (r'[*/]', Comment.Multiline),
  753. ],
  754. 'litstring': [
  755. (r'[^"]+', String.Double),
  756. (r'""', String.Escape),
  757. (r'"', String.Double, '#pop'),
  758. ],
  759. 'string': [
  760. (r'[^\\"\n]+', String.Double),
  761. include('escape-sequence'),
  762. (r'["\n]', String.Double, '#pop'),
  763. ],
  764. 'char': [
  765. (r'[^\\\'\n]+', String.Char),
  766. include('escape-sequence'),
  767. (r'[\'\n]', String.Char, '#pop'),
  768. ],
  769. 'escape-sequence': [
  770. (r'\\[nrt\\"\']', String.Escape),
  771. (r'\\x[0-9a-fA-F]{2}', String.Escape),
  772. (r'\\u[0-9a-fA-F]{4}', String.Escape),
  773. # Yes, \U literals are 6 hex digits.
  774. (r'\\U[0-9a-fA-F]{6}', String.Escape)
  775. ]
  776. }