_tokenizer.py 5.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193
  1. from __future__ import annotations
  2. import contextlib
  3. import re
  4. from dataclasses import dataclass
  5. from typing import Generator, Mapping, NoReturn
  6. from .specifiers import Specifier
  7. @dataclass
  8. class Token:
  9. name: str
  10. text: str
  11. position: int
  12. class ParserSyntaxError(Exception):
  13. """The provided source text could not be parsed correctly."""
  14. def __init__(
  15. self,
  16. message: str,
  17. *,
  18. source: str,
  19. span: tuple[int, int],
  20. ) -> None:
  21. self.span = span
  22. self.message = message
  23. self.source = source
  24. super().__init__()
  25. def __str__(self) -> str:
  26. marker = " " * self.span[0] + "~" * (self.span[1] - self.span[0]) + "^"
  27. return f"{self.message}\n {self.source}\n {marker}"
  28. DEFAULT_RULES: dict[str, re.Pattern[str]] = {
  29. "LEFT_PARENTHESIS": re.compile(r"\("),
  30. "RIGHT_PARENTHESIS": re.compile(r"\)"),
  31. "LEFT_BRACKET": re.compile(r"\["),
  32. "RIGHT_BRACKET": re.compile(r"\]"),
  33. "SEMICOLON": re.compile(r";"),
  34. "COMMA": re.compile(r","),
  35. "QUOTED_STRING": re.compile(
  36. r"""
  37. (
  38. ('[^']*')
  39. |
  40. ("[^"]*")
  41. )
  42. """,
  43. re.VERBOSE,
  44. ),
  45. "OP": re.compile(r"(===|==|~=|!=|<=|>=|<|>)"),
  46. "BOOLOP": re.compile(r"\b(or|and)\b"),
  47. "IN": re.compile(r"\bin\b"),
  48. "NOT": re.compile(r"\bnot\b"),
  49. "VARIABLE": re.compile(
  50. r"""
  51. \b(
  52. python_version
  53. |python_full_version
  54. |os[._]name
  55. |sys[._]platform
  56. |platform_(release|system)
  57. |platform[._](version|machine|python_implementation)
  58. |python_implementation
  59. |implementation_(name|version)
  60. |extras?
  61. |dependency_groups
  62. )\b
  63. """,
  64. re.VERBOSE,
  65. ),
  66. "SPECIFIER": re.compile(
  67. Specifier._operator_regex_str + Specifier._version_regex_str,
  68. re.VERBOSE | re.IGNORECASE,
  69. ),
  70. "AT": re.compile(r"\@"),
  71. "URL": re.compile(r"[^ \t]+"),
  72. "IDENTIFIER": re.compile(r"\b[a-zA-Z0-9][a-zA-Z0-9._-]*\b"),
  73. "VERSION_PREFIX_TRAIL": re.compile(r"\.\*"),
  74. "VERSION_LOCAL_LABEL_TRAIL": re.compile(r"\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*"),
  75. "WS": re.compile(r"[ \t]+"),
  76. "END": re.compile(r"$"),
  77. }
  78. class Tokenizer:
  79. """Context-sensitive token parsing.
  80. Provides methods to examine the input stream to check whether the next token
  81. matches.
  82. """
  83. def __init__(
  84. self,
  85. source: str,
  86. *,
  87. rules: Mapping[str, re.Pattern[str]],
  88. ) -> None:
  89. self.source = source
  90. self.rules = rules
  91. self.next_token: Token | None = None
  92. self.position = 0
  93. def consume(self, name: str) -> None:
  94. """Move beyond provided token name, if at current position."""
  95. if self.check(name):
  96. self.read()
  97. def check(self, name: str, *, peek: bool = False) -> bool:
  98. """Check whether the next token has the provided name.
  99. By default, if the check succeeds, the token *must* be read before
  100. another check. If `peek` is set to `True`, the token is not loaded and
  101. would need to be checked again.
  102. """
  103. assert self.next_token is None, (
  104. f"Cannot check for {name!r}, already have {self.next_token!r}"
  105. )
  106. assert name in self.rules, f"Unknown token name: {name!r}"
  107. expression = self.rules[name]
  108. match = expression.match(self.source, self.position)
  109. if match is None:
  110. return False
  111. if not peek:
  112. self.next_token = Token(name, match[0], self.position)
  113. return True
  114. def expect(self, name: str, *, expected: str) -> Token:
  115. """Expect a certain token name next, failing with a syntax error otherwise.
  116. The token is *not* read.
  117. """
  118. if not self.check(name):
  119. raise self.raise_syntax_error(f"Expected {expected}")
  120. return self.read()
  121. def read(self) -> Token:
  122. """Consume the next token and return it."""
  123. token = self.next_token
  124. assert token is not None
  125. self.position += len(token.text)
  126. self.next_token = None
  127. return token
  128. def raise_syntax_error(
  129. self,
  130. message: str,
  131. *,
  132. span_start: int | None = None,
  133. span_end: int | None = None,
  134. ) -> NoReturn:
  135. """Raise ParserSyntaxError at the given position."""
  136. span = (
  137. self.position if span_start is None else span_start,
  138. self.position if span_end is None else span_end,
  139. )
  140. raise ParserSyntaxError(
  141. message,
  142. source=self.source,
  143. span=span,
  144. )
  145. @contextlib.contextmanager
  146. def enclosing_tokens(
  147. self, open_token: str, close_token: str, *, around: str
  148. ) -> Generator[None, None, None]:
  149. if self.check(open_token):
  150. open_position = self.position
  151. self.read()
  152. else:
  153. open_position = None
  154. yield
  155. if open_position is None:
  156. return
  157. if not self.check(close_token):
  158. self.raise_syntax_error(
  159. f"Expected matching {close_token} for {open_token}, after {around}",
  160. span_start=open_position,
  161. )
  162. self.read()