tokenizer_config.json 24 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087
  1. {
  2. "add_bos_token": false,
  3. "add_prefix_space": false,
  4. "added_tokens_decoder": {
  5. "100256": {
  6. "content": "<|pad|>",
  7. "lstrip": false,
  8. "normalized": false,
  9. "rstrip": false,
  10. "single_word": false,
  11. "special": true
  12. },
  13. "100257": {
  14. "content": "<|end_of_text|>",
  15. "lstrip": false,
  16. "normalized": false,
  17. "rstrip": false,
  18. "single_word": false,
  19. "special": true
  20. },
  21. "100258": {
  22. "content": "<|fim_prefix|>",
  23. "lstrip": false,
  24. "normalized": false,
  25. "rstrip": false,
  26. "single_word": false,
  27. "special": true
  28. },
  29. "100259": {
  30. "content": "<|fim_middle|>",
  31. "lstrip": false,
  32. "normalized": false,
  33. "rstrip": false,
  34. "single_word": false,
  35. "special": true
  36. },
  37. "100260": {
  38. "content": "<|fim_suffix|>",
  39. "lstrip": false,
  40. "normalized": false,
  41. "rstrip": false,
  42. "single_word": false,
  43. "special": true
  44. },
  45. "100261": {
  46. "content": "<|fim_pad|>",
  47. "lstrip": false,
  48. "normalized": false,
  49. "rstrip": false,
  50. "single_word": false,
  51. "special": true
  52. },
  53. "100262": {
  54. "content": "<|filename|>",
  55. "lstrip": false,
  56. "normalized": false,
  57. "rstrip": false,
  58. "single_word": false,
  59. "special": true
  60. },
  61. "100263": {
  62. "content": "<|reponame|>",
  63. "lstrip": false,
  64. "normalized": false,
  65. "rstrip": false,
  66. "single_word": false,
  67. "special": true
  68. },
  69. "100264": {
  70. "content": "<|start_of_role|>",
  71. "lstrip": false,
  72. "normalized": false,
  73. "rstrip": false,
  74. "single_word": false,
  75. "special": true
  76. },
  77. "100265": {
  78. "content": "<|end_of_role|>",
  79. "lstrip": false,
  80. "normalized": false,
  81. "rstrip": false,
  82. "single_word": false,
  83. "special": true
  84. },
  85. "100266": {
  86. "content": "<|tool_call|>",
  87. "lstrip": false,
  88. "normalized": false,
  89. "rstrip": false,
  90. "single_word": false,
  91. "special": true
  92. },
  93. "100267": {
  94. "content": "<|start_of_plugin|>",
  95. "lstrip": false,
  96. "normalized": false,
  97. "rstrip": false,
  98. "single_word": false,
  99. "special": true
  100. },
  101. "100268": {
  102. "content": "<|end_of_plugin|>",
  103. "lstrip": false,
  104. "normalized": false,
  105. "rstrip": false,
  106. "single_word": false,
  107. "special": true
  108. },
  109. "100269": {
  110. "content": "<|unk|>",
  111. "lstrip": false,
  112. "normalized": false,
  113. "rstrip": false,
  114. "single_word": false,
  115. "special": true
  116. },
  117. "100270": {
  118. "content": "<image>",
  119. "lstrip": false,
  120. "normalized": false,
  121. "rstrip": false,
  122. "single_word": false,
  123. "special": true
  124. },
  125. "100271": {
  126. "content": "<caption>",
  127. "lstrip": false,
  128. "normalized": false,
  129. "rstrip": false,
  130. "single_word": false,
  131. "special": true
  132. },
  133. "100272": {
  134. "content": "</caption>",
  135. "lstrip": false,
  136. "normalized": false,
  137. "rstrip": false,
  138. "single_word": false,
  139. "special": true
  140. },
  141. "100273": {
  142. "content": "<footnote>",
  143. "lstrip": false,
  144. "normalized": false,
  145. "rstrip": false,
  146. "single_word": false,
  147. "special": true
  148. },
  149. "100274": {
  150. "content": "</footnote>",
  151. "lstrip": false,
  152. "normalized": false,
  153. "rstrip": false,
  154. "single_word": false,
  155. "special": true
  156. },
  157. "100275": {
  158. "content": "<formula>",
  159. "lstrip": false,
  160. "normalized": false,
  161. "rstrip": false,
  162. "single_word": false,
  163. "special": true
  164. },
  165. "100276": {
  166. "content": "</formula>",
  167. "lstrip": false,
  168. "normalized": false,
  169. "rstrip": false,
  170. "single_word": false,
  171. "special": true
  172. },
  173. "100277": {
  174. "content": "<list_item>",
  175. "lstrip": false,
  176. "normalized": false,
  177. "rstrip": false,
  178. "single_word": false,
  179. "special": true
  180. },
  181. "100278": {
  182. "content": "</list_item>",
  183. "lstrip": false,
  184. "normalized": false,
  185. "rstrip": false,
  186. "single_word": false,
  187. "special": true
  188. },
  189. "100279": {
  190. "content": "<page_footer>",
  191. "lstrip": false,
  192. "normalized": false,
  193. "rstrip": false,
  194. "single_word": false,
  195. "special": true
  196. },
  197. "100280": {
  198. "content": "</page_footer>",
  199. "lstrip": false,
  200. "normalized": false,
  201. "rstrip": false,
  202. "single_word": false,
  203. "special": true
  204. },
  205. "100281": {
  206. "content": "<page_header>",
  207. "lstrip": false,
  208. "normalized": false,
  209. "rstrip": false,
  210. "single_word": false,
  211. "special": true
  212. },
  213. "100282": {
  214. "content": "</page_header>",
  215. "lstrip": false,
  216. "normalized": false,
  217. "rstrip": false,
  218. "single_word": false,
  219. "special": true
  220. },
  221. "100283": {
  222. "content": "<picture>",
  223. "lstrip": false,
  224. "normalized": false,
  225. "rstrip": false,
  226. "single_word": false,
  227. "special": true
  228. },
  229. "100284": {
  230. "content": "</picture>",
  231. "lstrip": false,
  232. "normalized": false,
  233. "rstrip": false,
  234. "single_word": false,
  235. "special": true
  236. },
  237. "100285": {
  238. "content": "<section_header_level_1>",
  239. "lstrip": false,
  240. "normalized": false,
  241. "rstrip": false,
  242. "single_word": false,
  243. "special": true
  244. },
  245. "100286": {
  246. "content": "</section_header_level_1>",
  247. "lstrip": false,
  248. "normalized": false,
  249. "rstrip": false,
  250. "single_word": false,
  251. "special": true
  252. },
  253. "100287": {
  254. "content": "<section_header_level_2>",
  255. "lstrip": false,
  256. "normalized": false,
  257. "rstrip": false,
  258. "single_word": false,
  259. "special": true
  260. },
  261. "100288": {
  262. "content": "</section_header_level_2>",
  263. "lstrip": false,
  264. "normalized": false,
  265. "rstrip": false,
  266. "single_word": false,
  267. "special": true
  268. },
  269. "100289": {
  270. "content": "<section_header_level_3>",
  271. "lstrip": false,
  272. "normalized": false,
  273. "rstrip": false,
  274. "single_word": false,
  275. "special": true
  276. },
  277. "100290": {
  278. "content": "</section_header_level_3>",
  279. "lstrip": false,
  280. "normalized": false,
  281. "rstrip": false,
  282. "single_word": false,
  283. "special": true
  284. },
  285. "100291": {
  286. "content": "<section_header_level_4>",
  287. "lstrip": false,
  288. "normalized": false,
  289. "rstrip": false,
  290. "single_word": false,
  291. "special": true
  292. },
  293. "100292": {
  294. "content": "</section_header_level_4>",
  295. "lstrip": false,
  296. "normalized": false,
  297. "rstrip": false,
  298. "single_word": false,
  299. "special": true
  300. },
  301. "100293": {
  302. "content": "<section_header_level_5>",
  303. "lstrip": false,
  304. "normalized": false,
  305. "rstrip": false,
  306. "single_word": false,
  307. "special": true
  308. },
  309. "100294": {
  310. "content": "</section_header_level_5>",
  311. "lstrip": false,
  312. "normalized": false,
  313. "rstrip": false,
  314. "single_word": false,
  315. "special": true
  316. },
  317. "100295": {
  318. "content": "<section_header_level_6>",
  319. "lstrip": false,
  320. "normalized": false,
  321. "rstrip": false,
  322. "single_word": false,
  323. "special": true
  324. },
  325. "100296": {
  326. "content": "</section_header_level_6>",
  327. "lstrip": false,
  328. "normalized": false,
  329. "rstrip": false,
  330. "single_word": false,
  331. "special": true
  332. },
  333. "100297": {
  334. "content": "<otsl>",
  335. "lstrip": false,
  336. "normalized": false,
  337. "rstrip": false,
  338. "single_word": false,
  339. "special": true
  340. },
  341. "100298": {
  342. "content": "</otsl>",
  343. "lstrip": false,
  344. "normalized": false,
  345. "rstrip": false,
  346. "single_word": false,
  347. "special": true
  348. },
  349. "100299": {
  350. "content": "<checkbox_selected>",
  351. "lstrip": false,
  352. "normalized": false,
  353. "rstrip": false,
  354. "single_word": false,
  355. "special": true
  356. },
  357. "100300": {
  358. "content": "</checkbox_selected>",
  359. "lstrip": false,
  360. "normalized": false,
  361. "rstrip": false,
  362. "single_word": false,
  363. "special": true
  364. },
  365. "100301": {
  366. "content": "<checkbox_unselected>",
  367. "lstrip": false,
  368. "normalized": false,
  369. "rstrip": false,
  370. "single_word": false,
  371. "special": true
  372. },
  373. "100302": {
  374. "content": "</checkbox_unselected>",
  375. "lstrip": false,
  376. "normalized": false,
  377. "rstrip": false,
  378. "single_word": false,
  379. "special": true
  380. },
  381. "100303": {
  382. "content": "<form>",
  383. "lstrip": false,
  384. "normalized": false,
  385. "rstrip": false,
  386. "single_word": false,
  387. "special": true
  388. },
  389. "100304": {
  390. "content": "</form>",
  391. "lstrip": false,
  392. "normalized": false,
  393. "rstrip": false,
  394. "single_word": false,
  395. "special": true
  396. },
  397. "100305": {
  398. "content": "<key_value_region>",
  399. "lstrip": false,
  400. "normalized": false,
  401. "rstrip": false,
  402. "single_word": false,
  403. "special": true
  404. },
  405. "100306": {
  406. "content": "</key_value_region>",
  407. "lstrip": false,
  408. "normalized": false,
  409. "rstrip": false,
  410. "single_word": false,
  411. "special": true
  412. },
  413. "100307": {
  414. "content": "<key_",
  415. "lstrip": false,
  416. "normalized": false,
  417. "rstrip": false,
  418. "single_word": false,
  419. "special": true
  420. },
  421. "100308": {
  422. "content": "</key_",
  423. "lstrip": false,
  424. "normalized": false,
  425. "rstrip": false,
  426. "single_word": false,
  427. "special": true
  428. },
  429. "100309": {
  430. "content": "<value_",
  431. "lstrip": false,
  432. "normalized": false,
  433. "rstrip": false,
  434. "single_word": false,
  435. "special": true
  436. },
  437. "100310": {
  438. "content": "</value_",
  439. "lstrip": false,
  440. "normalized": false,
  441. "rstrip": false,
  442. "single_word": false,
  443. "special": true
  444. },
  445. "100311": {
  446. "content": "<link_",
  447. "lstrip": false,
  448. "normalized": false,
  449. "rstrip": false,
  450. "single_word": false,
  451. "special": true
  452. },
  453. "100312": {
  454. "content": "<chart>",
  455. "lstrip": false,
  456. "normalized": false,
  457. "rstrip": false,
  458. "single_word": false,
  459. "special": true
  460. },
  461. "100313": {
  462. "content": "</chart>",
  463. "lstrip": false,
  464. "normalized": false,
  465. "rstrip": false,
  466. "single_word": false,
  467. "special": true
  468. },
  469. "100314": {
  470. "content": "<page_break>",
  471. "lstrip": false,
  472. "normalized": false,
  473. "rstrip": false,
  474. "single_word": false,
  475. "special": true
  476. },
  477. "100315": {
  478. "content": "<smiles>",
  479. "lstrip": false,
  480. "normalized": false,
  481. "rstrip": false,
  482. "single_word": false,
  483. "special": true
  484. },
  485. "100316": {
  486. "content": "</smiles>",
  487. "lstrip": false,
  488. "normalized": false,
  489. "rstrip": false,
  490. "single_word": false,
  491. "special": true
  492. },
  493. "100317": {
  494. "content": "<loc_",
  495. "lstrip": false,
  496. "normalized": false,
  497. "rstrip": false,
  498. "single_word": false,
  499. "special": true
  500. },
  501. "100318": {
  502. "content": "<paragraph",
  503. "lstrip": false,
  504. "normalized": false,
  505. "rstrip": false,
  506. "single_word": false,
  507. "special": true
  508. },
  509. "100319": {
  510. "content": "</paragraph>",
  511. "lstrip": false,
  512. "normalized": false,
  513. "rstrip": false,
  514. "single_word": false,
  515. "special": true
  516. },
  517. "100320": {
  518. "content": "<references>",
  519. "lstrip": false,
  520. "normalized": false,
  521. "rstrip": false,
  522. "single_word": false,
  523. "special": true
  524. },
  525. "100321": {
  526. "content": "</references>",
  527. "lstrip": false,
  528. "normalized": false,
  529. "rstrip": false,
  530. "single_word": false,
  531. "special": true
  532. },
  533. "100322": {
  534. "content": "<ordered_list>",
  535. "lstrip": false,
  536. "normalized": false,
  537. "rstrip": false,
  538. "single_word": false,
  539. "special": true
  540. },
  541. "100323": {
  542. "content": "</ordered_list>",
  543. "lstrip": false,
  544. "normalized": false,
  545. "rstrip": false,
  546. "single_word": false,
  547. "special": true
  548. },
  549. "100324": {
  550. "content": "<unordered_list>",
  551. "lstrip": false,
  552. "normalized": false,
  553. "rstrip": false,
  554. "single_word": false,
  555. "special": true
  556. },
  557. "100325": {
  558. "content": "</unordered_list>",
  559. "lstrip": false,
  560. "normalized": false,
  561. "rstrip": false,
  562. "single_word": false,
  563. "special": true
  564. },
  565. "100326": {
  566. "content": "<group>",
  567. "lstrip": false,
  568. "normalized": false,
  569. "rstrip": false,
  570. "single_word": false,
  571. "special": true
  572. },
  573. "100327": {
  574. "content": "<doctag>",
  575. "lstrip": false,
  576. "normalized": false,
  577. "rstrip": false,
  578. "single_word": false,
  579. "special": true
  580. },
  581. "100328": {
  582. "content": "</doctag>",
  583. "lstrip": false,
  584. "normalized": false,
  585. "rstrip": false,
  586. "single_word": false,
  587. "special": true
  588. },
  589. "100329": {
  590. "content": "<text_break>",
  591. "lstrip": false,
  592. "normalized": false,
  593. "rstrip": false,
  594. "single_word": false,
  595. "special": true
  596. },
  597. "100330": {
  598. "content": "<fcel>",
  599. "lstrip": false,
  600. "normalized": false,
  601. "rstrip": false,
  602. "single_word": false,
  603. "special": true
  604. },
  605. "100331": {
  606. "content": "<ecel>",
  607. "lstrip": false,
  608. "normalized": false,
  609. "rstrip": false,
  610. "single_word": false,
  611. "special": true
  612. },
  613. "100332": {
  614. "content": "<lcel>",
  615. "lstrip": false,
  616. "normalized": false,
  617. "rstrip": false,
  618. "single_word": false,
  619. "special": true
  620. },
  621. "100333": {
  622. "content": "<ucel>",
  623. "lstrip": false,
  624. "normalized": false,
  625. "rstrip": false,
  626. "single_word": false,
  627. "special": true
  628. },
  629. "100334": {
  630. "content": "<xcel>",
  631. "lstrip": false,
  632. "normalized": false,
  633. "rstrip": false,
  634. "single_word": false,
  635. "special": true
  636. },
  637. "100335": {
  638. "content": "<nl>",
  639. "lstrip": false,
  640. "normalized": false,
  641. "rstrip": false,
  642. "single_word": false,
  643. "special": true
  644. },
  645. "100336": {
  646. "content": "<ched>",
  647. "lstrip": false,
  648. "normalized": false,
  649. "rstrip": false,
  650. "single_word": false,
  651. "special": true
  652. },
  653. "100337": {
  654. "content": "<rhed>",
  655. "lstrip": false,
  656. "normalized": false,
  657. "rstrip": false,
  658. "single_word": false,
  659. "special": true
  660. },
  661. "100338": {
  662. "content": "<end_of_utterance>",
  663. "lstrip": false,
  664. "normalized": false,
  665. "rstrip": false,
  666. "single_word": false,
  667. "special": true
  668. },
  669. "100339": {
  670. "content": "<fake_token_around_image>",
  671. "lstrip": false,
  672. "normalized": false,
  673. "rstrip": false,
  674. "single_word": false,
  675. "special": true
  676. },
  677. "100340": {
  678. "content": "<global-img>",
  679. "lstrip": false,
  680. "normalized": false,
  681. "rstrip": false,
  682. "single_word": false,
  683. "special": true
  684. },
  685. "100341": {
  686. "content": "<|unused_72|>",
  687. "lstrip": false,
  688. "normalized": false,
  689. "rstrip": false,
  690. "single_word": false,
  691. "special": true
  692. },
  693. "100342": {
  694. "content": "<|unused_73|>",
  695. "lstrip": false,
  696. "normalized": false,
  697. "rstrip": false,
  698. "single_word": false,
  699. "special": true
  700. },
  701. "100343": {
  702. "content": "<|unused_74|>",
  703. "lstrip": false,
  704. "normalized": false,
  705. "rstrip": false,
  706. "single_word": false,
  707. "special": true
  708. },
  709. "100344": {
  710. "content": "<|unused_75|>",
  711. "lstrip": false,
  712. "normalized": false,
  713. "rstrip": false,
  714. "single_word": false,
  715. "special": true
  716. },
  717. "100345": {
  718. "content": "<|unused_76|>",
  719. "lstrip": false,
  720. "normalized": false,
  721. "rstrip": false,
  722. "single_word": false,
  723. "special": true
  724. },
  725. "100346": {
  726. "content": "<|unused_77|>",
  727. "lstrip": false,
  728. "normalized": false,
  729. "rstrip": false,
  730. "single_word": false,
  731. "special": true
  732. },
  733. "100347": {
  734. "content": "<|unused_78|>",
  735. "lstrip": false,
  736. "normalized": false,
  737. "rstrip": false,
  738. "single_word": false,
  739. "special": true
  740. },
  741. "100348": {
  742. "content": "<|unused_79|>",
  743. "lstrip": false,
  744. "normalized": false,
  745. "rstrip": false,
  746. "single_word": false,
  747. "special": true
  748. },
  749. "100349": {
  750. "content": "<|unused_80|>",
  751. "lstrip": false,
  752. "normalized": false,
  753. "rstrip": false,
  754. "single_word": false,
  755. "special": true
  756. },
  757. "100350": {
  758. "content": "<|unused_81|>",
  759. "lstrip": false,
  760. "normalized": false,
  761. "rstrip": false,
  762. "single_word": false,
  763. "special": true
  764. },
  765. "100351": {
  766. "content": "<|unused_82|>",
  767. "lstrip": false,
  768. "normalized": false,
  769. "rstrip": false,
  770. "single_word": false,
  771. "special": true
  772. },
  773. "100352": {
  774. "content": "<row_1_col_1>",
  775. "lstrip": false,
  776. "normalized": false,
  777. "rstrip": false,
  778. "single_word": false,
  779. "special": true
  780. },
  781. "100353": {
  782. "content": "<row_1_col_2>",
  783. "lstrip": false,
  784. "normalized": false,
  785. "rstrip": false,
  786. "single_word": false,
  787. "special": true
  788. },
  789. "100354": {
  790. "content": "<row_1_col_3>",
  791. "lstrip": false,
  792. "normalized": false,
  793. "rstrip": false,
  794. "single_word": false,
  795. "special": true
  796. },
  797. "100355": {
  798. "content": "<row_1_col_4>",
  799. "lstrip": false,
  800. "normalized": false,
  801. "rstrip": false,
  802. "single_word": false,
  803. "special": true
  804. },
  805. "100356": {
  806. "content": "<row_1_col_5>",
  807. "lstrip": false,
  808. "normalized": false,
  809. "rstrip": false,
  810. "single_word": false,
  811. "special": true
  812. },
  813. "100357": {
  814. "content": "<row_1_col_6>",
  815. "lstrip": false,
  816. "normalized": false,
  817. "rstrip": false,
  818. "single_word": false,
  819. "special": true
  820. },
  821. "100358": {
  822. "content": "<row_2_col_1>",
  823. "lstrip": false,
  824. "normalized": false,
  825. "rstrip": false,
  826. "single_word": false,
  827. "special": true
  828. },
  829. "100359": {
  830. "content": "<row_2_col_2>",
  831. "lstrip": false,
  832. "normalized": false,
  833. "rstrip": false,
  834. "single_word": false,
  835. "special": true
  836. },
  837. "100360": {
  838. "content": "<row_2_col_3>",
  839. "lstrip": false,
  840. "normalized": false,
  841. "rstrip": false,
  842. "single_word": false,
  843. "special": true
  844. },
  845. "100361": {
  846. "content": "<row_2_col_4>",
  847. "lstrip": false,
  848. "normalized": false,
  849. "rstrip": false,
  850. "single_word": false,
  851. "special": true
  852. },
  853. "100362": {
  854. "content": "<row_2_col_5>",
  855. "lstrip": false,
  856. "normalized": false,
  857. "rstrip": false,
  858. "single_word": false,
  859. "special": true
  860. },
  861. "100363": {
  862. "content": "<row_2_col_6>",
  863. "lstrip": false,
  864. "normalized": false,
  865. "rstrip": false,
  866. "single_word": false,
  867. "special": true
  868. },
  869. "100364": {
  870. "content": "<row_3_col_1>",
  871. "lstrip": false,
  872. "normalized": false,
  873. "rstrip": false,
  874. "single_word": false,
  875. "special": true
  876. },
  877. "100365": {
  878. "content": "<row_3_col_2>",
  879. "lstrip": false,
  880. "normalized": false,
  881. "rstrip": false,
  882. "single_word": false,
  883. "special": true
  884. },
  885. "100366": {
  886. "content": "<row_3_col_3>",
  887. "lstrip": false,
  888. "normalized": false,
  889. "rstrip": false,
  890. "single_word": false,
  891. "special": true
  892. },
  893. "100367": {
  894. "content": "<row_3_col_4>",
  895. "lstrip": false,
  896. "normalized": false,
  897. "rstrip": false,
  898. "single_word": false,
  899. "special": true
  900. },
  901. "100368": {
  902. "content": "<row_3_col_5>",
  903. "lstrip": false,
  904. "normalized": false,
  905. "rstrip": false,
  906. "single_word": false,
  907. "special": true
  908. },
  909. "100369": {
  910. "content": "<row_3_col_6>",
  911. "lstrip": false,
  912. "normalized": false,
  913. "rstrip": false,
  914. "single_word": false,
  915. "special": true
  916. },
  917. "100370": {
  918. "content": "<row_4_col_1>",
  919. "lstrip": false,
  920. "normalized": false,
  921. "rstrip": false,
  922. "single_word": false,
  923. "special": true
  924. },
  925. "100371": {
  926. "content": "<row_4_col_2>",
  927. "lstrip": false,
  928. "normalized": false,
  929. "rstrip": false,
  930. "single_word": false,
  931. "special": true
  932. },
  933. "100372": {
  934. "content": "<row_4_col_3>",
  935. "lstrip": false,
  936. "normalized": false,
  937. "rstrip": false,
  938. "single_word": false,
  939. "special": true
  940. },
  941. "100373": {
  942. "content": "<row_4_col_4>",
  943. "lstrip": false,
  944. "normalized": false,
  945. "rstrip": false,
  946. "single_word": false,
  947. "special": true
  948. },
  949. "100374": {
  950. "content": "<row_4_col_5>",
  951. "lstrip": false,
  952. "normalized": false,
  953. "rstrip": false,
  954. "single_word": false,
  955. "special": true
  956. },
  957. "100375": {
  958. "content": "<row_4_col_6>",
  959. "lstrip": false,
  960. "normalized": false,
  961. "rstrip": false,
  962. "single_word": false,
  963. "special": true
  964. },
  965. "100376": {
  966. "content": "<row_5_col_1>",
  967. "lstrip": false,
  968. "normalized": false,
  969. "rstrip": false,
  970. "single_word": false,
  971. "special": true
  972. },
  973. "100377": {
  974. "content": "<row_5_col_2>",
  975. "lstrip": false,
  976. "normalized": false,
  977. "rstrip": false,
  978. "single_word": false,
  979. "special": true
  980. },
  981. "100378": {
  982. "content": "<row_5_col_3>",
  983. "lstrip": false,
  984. "normalized": false,
  985. "rstrip": false,
  986. "single_word": false,
  987. "special": true
  988. },
  989. "100379": {
  990. "content": "<row_5_col_4>",
  991. "lstrip": false,
  992. "normalized": false,
  993. "rstrip": false,
  994. "single_word": false,
  995. "special": true
  996. },
  997. "100380": {
  998. "content": "<row_5_col_5>",
  999. "lstrip": false,
  1000. "normalized": false,
  1001. "rstrip": false,
  1002. "single_word": false,
  1003. "special": true
  1004. },
  1005. "100381": {
  1006. "content": "<row_5_col_6>",
  1007. "lstrip": false,
  1008. "normalized": false,
  1009. "rstrip": false,
  1010. "single_word": false,
  1011. "special": true
  1012. },
  1013. "100382": {
  1014. "content": "<row_6_col_1>",
  1015. "lstrip": false,
  1016. "normalized": false,
  1017. "rstrip": false,
  1018. "single_word": false,
  1019. "special": true
  1020. },
  1021. "100383": {
  1022. "content": "<row_6_col_2>",
  1023. "lstrip": false,
  1024. "normalized": false,
  1025. "rstrip": false,
  1026. "single_word": false,
  1027. "special": true
  1028. },
  1029. "100384": {
  1030. "content": "<row_6_col_3>",
  1031. "lstrip": false,
  1032. "normalized": false,
  1033. "rstrip": false,
  1034. "single_word": false,
  1035. "special": true
  1036. },
  1037. "100385": {
  1038. "content": "<row_6_col_4>",
  1039. "lstrip": false,
  1040. "normalized": false,
  1041. "rstrip": false,
  1042. "single_word": false,
  1043. "special": true
  1044. },
  1045. "100386": {
  1046. "content": "<row_6_col_5>",
  1047. "lstrip": false,
  1048. "normalized": false,
  1049. "rstrip": false,
  1050. "single_word": false,
  1051. "special": true
  1052. },
  1053. "100387": {
  1054. "content": "<row_6_col_6>",
  1055. "lstrip": false,
  1056. "normalized": false,
  1057. "rstrip": false,
  1058. "single_word": false,
  1059. "special": true
  1060. }
  1061. },
  1062. "additional_special_tokens": [
  1063. "<fake_token_around_image>",
  1064. "<image>",
  1065. "<end_of_utterance>"
  1066. ],
  1067. "bos_token": "<|end_of_text|>",
  1068. "chat_template": "<|start_of_role|>{% for message in messages %}{{message['role']}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% endif %}{% endfor %}<end_of_utterance>\n{% endfor %}{% if add_generation_prompt %}{{ 'assistant:' }}{% endif %}",
  1069. "clean_up_tokenization_spaces": false,
  1070. "eos_token": "<end_of_utterance>",
  1071. "errors": "replace",
  1072. "extra_special_tokens": {},
  1073. "model_max_length": 8192,
  1074. "pad_to_multiple_of": null,
  1075. "pad_token": "<|pad|>",
  1076. "pad_token_type_id": 0,
  1077. "padding_side": "left",
  1078. "max_length": 8192,
  1079. "truncation": true,
  1080. "padding": "longest",
  1081. "processor_class": "Idefics3Processor",
  1082. "stride": 0,
  1083. "tokenizer_class": "GPT2Tokenizer",
  1084. "truncation_side": "right",
  1085. "truncation_strategy": "longest_first",
  1086. "unk_token": "<|unk|>"
  1087. }