parser.py 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182
  1. import codecs
  2. import re
  3. from typing import (
  4. IO,
  5. Iterator,
  6. Match,
  7. NamedTuple,
  8. Optional,
  9. Pattern,
  10. Sequence,
  11. )
  12. def make_regex(string: str, extra_flags: int = 0) -> Pattern[str]:
  13. return re.compile(string, re.UNICODE | extra_flags)
  14. _newline = make_regex(r"(\r\n|\n|\r)")
  15. _multiline_whitespace = make_regex(r"\s*", extra_flags=re.MULTILINE)
  16. _whitespace = make_regex(r"[^\S\r\n]*")
  17. _export = make_regex(r"(?:export[^\S\r\n]+)?")
  18. _single_quoted_key = make_regex(r"'([^']+)'")
  19. _unquoted_key = make_regex(r"([^=\#\s]+)")
  20. _equal_sign = make_regex(r"(=[^\S\r\n]*)")
  21. _single_quoted_value = make_regex(r"'((?:\\'|[^'])*)'")
  22. _double_quoted_value = make_regex(r'"((?:\\"|[^"])*)"')
  23. _unquoted_value = make_regex(r"([^\r\n]*)")
  24. _comment = make_regex(r"(?:[^\S\r\n]*#[^\r\n]*)?")
  25. _end_of_line = make_regex(r"[^\S\r\n]*(?:\r\n|\n|\r|$)")
  26. _rest_of_line = make_regex(r"[^\r\n]*(?:\r|\n|\r\n)?")
  27. _double_quote_escapes = make_regex(r"\\[\\'\"abfnrtv]")
  28. _single_quote_escapes = make_regex(r"\\[\\']")
  29. class Original(NamedTuple):
  30. string: str
  31. line: int
  32. class Binding(NamedTuple):
  33. key: Optional[str]
  34. value: Optional[str]
  35. original: Original
  36. error: bool
  37. class Position:
  38. def __init__(self, chars: int, line: int) -> None:
  39. self.chars = chars
  40. self.line = line
  41. @classmethod
  42. def start(cls) -> "Position":
  43. return cls(chars=0, line=1)
  44. def set(self, other: "Position") -> None:
  45. self.chars = other.chars
  46. self.line = other.line
  47. def advance(self, string: str) -> None:
  48. self.chars += len(string)
  49. self.line += len(re.findall(_newline, string))
  50. class Error(Exception):
  51. pass
  52. class Reader:
  53. def __init__(self, stream: IO[str]) -> None:
  54. self.string = stream.read()
  55. self.position = Position.start()
  56. self.mark = Position.start()
  57. def has_next(self) -> bool:
  58. return self.position.chars < len(self.string)
  59. def set_mark(self) -> None:
  60. self.mark.set(self.position)
  61. def get_marked(self) -> Original:
  62. return Original(
  63. string=self.string[self.mark.chars : self.position.chars],
  64. line=self.mark.line,
  65. )
  66. def peek(self, count: int) -> str:
  67. return self.string[self.position.chars : self.position.chars + count]
  68. def read(self, count: int) -> str:
  69. result = self.string[self.position.chars : self.position.chars + count]
  70. if len(result) < count:
  71. raise Error("read: End of string")
  72. self.position.advance(result)
  73. return result
  74. def read_regex(self, regex: Pattern[str]) -> Sequence[str]:
  75. match = regex.match(self.string, self.position.chars)
  76. if match is None:
  77. raise Error("read_regex: Pattern not found")
  78. self.position.advance(self.string[match.start() : match.end()])
  79. return match.groups()
  80. def decode_escapes(regex: Pattern[str], string: str) -> str:
  81. def decode_match(match: Match[str]) -> str:
  82. return codecs.decode(match.group(0), "unicode-escape") # type: ignore
  83. return regex.sub(decode_match, string)
  84. def parse_key(reader: Reader) -> Optional[str]:
  85. char = reader.peek(1)
  86. if char == "#":
  87. return None
  88. elif char == "'":
  89. (key,) = reader.read_regex(_single_quoted_key)
  90. else:
  91. (key,) = reader.read_regex(_unquoted_key)
  92. return key
  93. def parse_unquoted_value(reader: Reader) -> str:
  94. (part,) = reader.read_regex(_unquoted_value)
  95. return re.sub(r"\s+#.*", "", part).rstrip()
  96. def parse_value(reader: Reader) -> str:
  97. char = reader.peek(1)
  98. if char == "'":
  99. (value,) = reader.read_regex(_single_quoted_value)
  100. return decode_escapes(_single_quote_escapes, value)
  101. elif char == '"':
  102. (value,) = reader.read_regex(_double_quoted_value)
  103. return decode_escapes(_double_quote_escapes, value)
  104. elif char in ("", "\n", "\r"):
  105. return ""
  106. else:
  107. return parse_unquoted_value(reader)
  108. def parse_binding(reader: Reader) -> Binding:
  109. reader.set_mark()
  110. try:
  111. reader.read_regex(_multiline_whitespace)
  112. if not reader.has_next():
  113. return Binding(
  114. key=None,
  115. value=None,
  116. original=reader.get_marked(),
  117. error=False,
  118. )
  119. reader.read_regex(_export)
  120. key = parse_key(reader)
  121. reader.read_regex(_whitespace)
  122. if reader.peek(1) == "=":
  123. reader.read_regex(_equal_sign)
  124. value: Optional[str] = parse_value(reader)
  125. else:
  126. value = None
  127. reader.read_regex(_comment)
  128. reader.read_regex(_end_of_line)
  129. return Binding(
  130. key=key,
  131. value=value,
  132. original=reader.get_marked(),
  133. error=False,
  134. )
  135. except Error:
  136. reader.read_regex(_rest_of_line)
  137. return Binding(
  138. key=None,
  139. value=None,
  140. original=reader.get_marked(),
  141. error=True,
  142. )
  143. def parse_stream(stream: IO[str]) -> Iterator[Binding]:
  144. reader = Reader(stream)
  145. while reader.has_next():
  146. yield parse_binding(reader)