_cmap.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358
  1. import binascii
  2. from binascii import Error as BinasciiError
  3. from binascii import unhexlify
  4. from math import ceil
  5. from typing import Any, Union, cast
  6. from ._codecs import adobe_glyphs, charset_encoding
  7. from ._utils import logger_error, logger_warning
  8. from .errors import LimitReachedError
  9. from .generic import (
  10. DecodedStreamObject,
  11. DictionaryObject,
  12. NullObject,
  13. StreamObject,
  14. is_null_or_none,
  15. )
  16. _predefined_cmap: dict[str, str] = {
  17. "/Identity-H": "utf-16-be",
  18. "/Identity-V": "utf-16-be",
  19. "/GB-EUC-H": "gbk",
  20. "/GB-EUC-V": "gbk",
  21. "/GBpc-EUC-H": "gb2312",
  22. "/GBpc-EUC-V": "gb2312",
  23. "/GBK-EUC-H": "gbk",
  24. "/GBK-EUC-V": "gbk",
  25. "/GBK2K-H": "gb18030",
  26. "/GBK2K-V": "gb18030",
  27. "/ETen-B5-H": "cp950",
  28. "/ETen-B5-V": "cp950",
  29. "/ETenms-B5-H": "cp950",
  30. "/ETenms-B5-V": "cp950",
  31. "/UniCNS-UTF16-H": "utf-16-be",
  32. "/UniCNS-UTF16-V": "utf-16-be",
  33. "/UniGB-UTF16-H": "gb18030",
  34. "/UniGB-UTF16-V": "gb18030",
  35. # UCS2 in code
  36. }
  37. def get_encoding(
  38. ft: DictionaryObject
  39. ) -> tuple[Union[str, dict[int, str]], dict[Any, Any]]:
  40. encoding = _parse_encoding(ft)
  41. map_dict, int_entry = _parse_to_unicode(ft)
  42. # Apply rule from PDF ref 1.7 §5.9.1, 1st bullet:
  43. # if cmap not empty encoding should be discarded
  44. # (here transformed into identity for those characters)
  45. # If encoding is a string, it is expected to be an identity translation.
  46. if isinstance(encoding, dict):
  47. for x in int_entry:
  48. if x <= 255:
  49. encoding[x] = chr(x)
  50. return encoding, map_dict
  51. def _parse_encoding(
  52. ft: DictionaryObject
  53. ) -> Union[str, dict[int, str]]:
  54. encoding: Union[str, list[str], dict[int, str]] = []
  55. if "/Encoding" not in ft:
  56. if "/BaseFont" in ft and cast(str, ft["/BaseFont"]) in charset_encoding:
  57. encoding = dict(
  58. zip(range(256), charset_encoding[cast(str, ft["/BaseFont"])])
  59. )
  60. else:
  61. encoding = "charmap"
  62. return encoding
  63. enc: Union[str, DictionaryObject, NullObject] = cast(
  64. Union[str, DictionaryObject, NullObject], ft["/Encoding"].get_object()
  65. )
  66. if isinstance(enc, str):
  67. try:
  68. # already done : enc = NameObject.unnumber(enc.encode()).decode()
  69. # for #xx decoding
  70. if enc in charset_encoding:
  71. encoding = charset_encoding[enc].copy()
  72. elif enc in _predefined_cmap:
  73. encoding = _predefined_cmap[enc]
  74. elif "-UCS2-" in enc:
  75. encoding = "utf-16-be"
  76. else:
  77. raise Exception("not found")
  78. except Exception:
  79. logger_error("Advanced encoding %(encoding)s not implemented yet", source=__name__, encoding=enc)
  80. encoding = enc
  81. elif isinstance(enc, DictionaryObject) and "/BaseEncoding" in enc:
  82. try:
  83. encoding = charset_encoding[cast(str, enc["/BaseEncoding"])].copy()
  84. except Exception:
  85. logger_error(
  86. "Advanced encoding %(encoding)s not implemented yet",
  87. source=__name__, encoding=encoding
  88. )
  89. encoding = charset_encoding["/StandardEncoding"].copy()
  90. else:
  91. encoding = charset_encoding["/StandardEncoding"].copy()
  92. if isinstance(enc, DictionaryObject) and "/Differences" in enc:
  93. x: int = 0
  94. o: Union[int, str]
  95. for o in cast(DictionaryObject, enc["/Differences"]):
  96. if isinstance(o, int):
  97. x = o
  98. else: # isinstance(o, str):
  99. try:
  100. if x < len(encoding):
  101. encoding[x] = adobe_glyphs[o] # type: ignore
  102. except Exception:
  103. encoding[x] = o # type: ignore
  104. x += 1
  105. if isinstance(encoding, list):
  106. encoding = dict(zip(range(256), encoding))
  107. return encoding
  108. def _parse_to_unicode(
  109. ft: DictionaryObject
  110. ) -> tuple[dict[Any, Any], list[int]]:
  111. # will store all translation code
  112. # and map_dict[-1] we will have the number of bytes to convert
  113. map_dict: dict[Any, Any] = {}
  114. # will provide the list of cmap keys as int to correct encoding
  115. int_entry: list[int] = []
  116. if "/ToUnicode" not in ft:
  117. if ft.get("/Subtype", "") == "/Type1":
  118. return _type1_alternative(ft, map_dict, int_entry)
  119. return {}, []
  120. process_rg: bool = False
  121. process_char: bool = False
  122. multiline_rg: Union[
  123. None, tuple[int, int]
  124. ] = None # tuple = (current_char, remaining size) ; cf #1285 for example of file
  125. cm = prepare_cm(ft)
  126. for line in cm.split(b"\n"):
  127. process_rg, process_char, multiline_rg = process_cm_line(
  128. line.strip(b" \t"),
  129. process_rg,
  130. process_char,
  131. multiline_rg,
  132. map_dict,
  133. int_entry,
  134. )
  135. return map_dict, int_entry
  136. def prepare_cm(ft: DictionaryObject) -> bytes:
  137. tu = ft["/ToUnicode"]
  138. cm: bytes
  139. if isinstance(tu, StreamObject):
  140. cm = cast(DecodedStreamObject, ft["/ToUnicode"]).get_data()
  141. else: # if (tu is None) or cast(str, tu).startswith("/Identity"):
  142. # the full range 0000-FFFF will be processed
  143. cm = b"beginbfrange\n<0000> <0001> <0000>\nendbfrange"
  144. if isinstance(cm, str):
  145. cm = cm.encode()
  146. # we need to prepare cm before due to missing return line in pdf printed
  147. # to pdf from word
  148. cm = (
  149. cm.strip()
  150. .replace(b"beginbfchar", b"\nbeginbfchar\n")
  151. .replace(b"endbfchar", b"\nendbfchar\n")
  152. .replace(b"beginbfrange", b"\nbeginbfrange\n")
  153. .replace(b"endbfrange", b"\nendbfrange\n")
  154. .replace(b"<<", b"\n{\n") # text between << and >> not used but
  155. .replace(b">>", b"\n}\n") # some solution to find it back
  156. )
  157. ll = cm.split(b"<")
  158. for i in range(len(ll)):
  159. j = ll[i].find(b">")
  160. if j >= 0:
  161. if j == 0:
  162. # string is empty: stash a placeholder here (see below)
  163. # see https://github.com/py-pdf/pypdf/issues/1111
  164. content = b"."
  165. else:
  166. content = ll[i][:j].replace(b" ", b"")
  167. ll[i] = content + b" " + ll[i][j + 1 :]
  168. cm = (
  169. (b" ".join(ll))
  170. .replace(b"[", b" [ ")
  171. .replace(b"]", b" ]\n ")
  172. .replace(b"\r", b"\n")
  173. )
  174. return cm
  175. def process_cm_line(
  176. line: bytes,
  177. process_rg: bool,
  178. process_char: bool,
  179. multiline_rg: Union[None, tuple[int, int]],
  180. map_dict: dict[Any, Any],
  181. int_entry: list[int],
  182. ) -> tuple[bool, bool, Union[None, tuple[int, int]]]:
  183. if line == b"" or line[0] == 37: # 37 = %
  184. return process_rg, process_char, multiline_rg
  185. line = line.replace(b"\t", b" ")
  186. if b"beginbfrange" in line:
  187. process_rg = True
  188. elif b"endbfrange" in line:
  189. process_rg = False
  190. elif b"beginbfchar" in line:
  191. process_char = True
  192. elif b"endbfchar" in line:
  193. process_char = False
  194. elif process_rg:
  195. try:
  196. multiline_rg = parse_bfrange(line, map_dict, int_entry, multiline_rg)
  197. except binascii.Error as error:
  198. logger_warning(f"Skipping broken line {line!r}: {error}", __name__)
  199. elif process_char:
  200. parse_bfchar(line, map_dict, int_entry)
  201. return process_rg, process_char, multiline_rg
  202. # Usual values should be up to 65_536.
  203. MAPPING_DICTIONARY_SIZE_LIMIT = 100_000
  204. def _check_mapping_size(size: int) -> None:
  205. if size > MAPPING_DICTIONARY_SIZE_LIMIT:
  206. raise LimitReachedError(f"Maximum /ToUnicode size limit reached: {size} > {MAPPING_DICTIONARY_SIZE_LIMIT}.")
  207. def parse_bfrange(
  208. line: bytes,
  209. map_dict: dict[Any, Any],
  210. int_entry: list[int],
  211. multiline_rg: Union[None, tuple[int, int]],
  212. ) -> Union[None, tuple[int, int]]:
  213. lst = [x for x in line.split(b" ") if x]
  214. closure_found = False
  215. entry_count = len(int_entry)
  216. _check_mapping_size(entry_count)
  217. if multiline_rg is not None:
  218. fmt = b"%%0%dX" % (map_dict[-1] * 2)
  219. a = multiline_rg[0] # a, b not in the current line
  220. b = multiline_rg[1]
  221. for sq in lst:
  222. if sq == b"]":
  223. closure_found = True
  224. break
  225. entry_count += 1
  226. _check_mapping_size(entry_count)
  227. map_dict[
  228. unhexlify(fmt % a).decode(
  229. "charmap" if map_dict[-1] == 1 else "utf-16-be",
  230. "surrogatepass",
  231. )
  232. ] = unhexlify(sq).decode("utf-16-be", "surrogatepass")
  233. int_entry.append(a)
  234. a += 1
  235. else:
  236. a = int(lst[0], 16)
  237. b = int(lst[1], 16)
  238. nbi = max(len(lst[0]), len(lst[1]))
  239. map_dict[-1] = ceil(nbi / 2)
  240. fmt = b"%%0%dX" % (map_dict[-1] * 2)
  241. if lst[2] == b"[":
  242. for sq in lst[3:]:
  243. if sq == b"]":
  244. closure_found = True
  245. break
  246. entry_count += 1
  247. _check_mapping_size(entry_count)
  248. map_dict[
  249. unhexlify(fmt % a).decode(
  250. "charmap" if map_dict[-1] == 1 else "utf-16-be",
  251. "surrogatepass",
  252. )
  253. ] = unhexlify(sq).decode("utf-16-be", "surrogatepass")
  254. int_entry.append(a)
  255. a += 1
  256. else: # case without list
  257. c = int(lst[2], 16)
  258. fmt2 = b"%%0%dX" % max(4, len(lst[2]))
  259. closure_found = True
  260. range_size = max(0, b - a + 1)
  261. _check_mapping_size(entry_count + range_size) # This can be checked beforehand.
  262. while a <= b:
  263. map_dict[
  264. unhexlify(fmt % a).decode(
  265. "charmap" if map_dict[-1] == 1 else "utf-16-be",
  266. "surrogatepass",
  267. )
  268. ] = unhexlify(fmt2 % c).decode("utf-16-be", "surrogatepass")
  269. int_entry.append(a)
  270. a += 1
  271. c += 1
  272. return None if closure_found else (a, b)
  273. def parse_bfchar(line: bytes, map_dict: dict[Any, Any], int_entry: list[int]) -> None:
  274. lst = [x for x in line.split(b" ") if x]
  275. new_count = len(lst) // 2
  276. _check_mapping_size(len(int_entry) + new_count) # This can be checked beforehand.
  277. map_dict[-1] = len(lst[0]) // 2
  278. while len(lst) > 1:
  279. map_to = ""
  280. # placeholder (see above) means empty string
  281. if lst[1] != b".":
  282. try:
  283. map_to = unhexlify(lst[1]).decode(
  284. "charmap" if len(lst[1]) < 4 else "utf-16-be", "surrogatepass"
  285. ) # join is here as some cases where the code was split
  286. except BinasciiError as exception:
  287. logger_warning(f"Got invalid hex string: {exception!s} ({lst[1]!r})", __name__)
  288. map_dict[
  289. unhexlify(lst[0]).decode(
  290. "charmap" if map_dict[-1] == 1 else "utf-16-be", "surrogatepass"
  291. )
  292. ] = map_to
  293. int_entry.append(int(lst[0], 16))
  294. lst = lst[2:]
  295. def _type1_alternative(
  296. ft: DictionaryObject,
  297. map_dict: dict[Any, Any],
  298. int_entry: list[int],
  299. ) -> tuple[dict[Any, Any], list[int]]:
  300. if "/FontDescriptor" not in ft:
  301. return map_dict, int_entry
  302. ft_desc = cast(DictionaryObject, ft["/FontDescriptor"]).get("/FontFile")
  303. if is_null_or_none(ft_desc):
  304. return map_dict, int_entry
  305. assert ft_desc is not None, "mypy"
  306. txt = ft_desc.get_object().get_data()
  307. txt = txt.split(b"eexec\n")[0] # only clear part
  308. txt = txt.split(b"/Encoding")[1] # to get the encoding part
  309. lines = txt.replace(b"\r", b"\n").split(b"\n")
  310. for li in lines:
  311. if li.startswith(b"dup"):
  312. words = [_w for _w in li.split(b" ") if _w != b""]
  313. if len(words) > 3 and words[3] != b"put":
  314. continue
  315. try:
  316. i = int(words[1])
  317. except ValueError: # pragma: no cover
  318. continue
  319. try:
  320. v = adobe_glyphs[words[2].decode()]
  321. except KeyError:
  322. if words[2].startswith(b"/uni"):
  323. try:
  324. v = chr(int(words[2][4:], 16))
  325. except ValueError: # pragma: no cover
  326. continue
  327. else:
  328. continue
  329. map_dict[chr(i)] = v
  330. int_entry.append(i)
  331. return map_dict, int_entry