special_tokens_map.json 1.0 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253
  1. {
  2. "additional_special_tokens": [
  3. {
  4. "content": "<fake_token_around_image>",
  5. "lstrip": false,
  6. "normalized": false,
  7. "rstrip": false,
  8. "single_word": false
  9. },
  10. {
  11. "content": "<image>",
  12. "lstrip": false,
  13. "normalized": false,
  14. "rstrip": false,
  15. "single_word": false
  16. },
  17. {
  18. "content": "<end_of_utterance>",
  19. "lstrip": false,
  20. "normalized": false,
  21. "rstrip": false,
  22. "single_word": false
  23. }
  24. ],
  25. "bos_token": {
  26. "content": "<|end_of_text|>",
  27. "lstrip": false,
  28. "normalized": false,
  29. "rstrip": false,
  30. "single_word": false
  31. },
  32. "eos_token": {
  33. "content": "<end_of_utterance>",
  34. "lstrip": false,
  35. "normalized": false,
  36. "rstrip": false,
  37. "single_word": false
  38. },
  39. "pad_token": {
  40. "content": "<|pad|>",
  41. "lstrip": false,
  42. "normalized": false,
  43. "rstrip": false,
  44. "single_word": false
  45. },
  46. "unk_token": {
  47. "content": "<|unk|>",
  48. "lstrip": false,
  49. "normalized": false,
  50. "rstrip": false,
  51. "single_word": false
  52. }
  53. }