1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253 |
- {
- "additional_special_tokens": [
- {
- "content": "<fake_token_around_image>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false
- },
- {
- "content": "<image>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false
- },
- {
- "content": "<end_of_utterance>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false
- }
- ],
- "bos_token": {
- "content": "<|end_of_text|>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false
- },
- "eos_token": {
- "content": "<end_of_utterance>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false
- },
- "pad_token": {
- "content": "<|pad|>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false
- },
- "unk_token": {
- "content": "<|unk|>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false
- }
- }
|