Add `<image_placeholder>` token

#5
by Xenova HF staff - opened
Files changed (1) hide show
  1. tokenizer.json +9 -0
tokenizer.json CHANGED
@@ -5348,6 +5348,15 @@
5348
  "rstrip": false,
5349
  "normalized": false,
5350
  "special": true
 
 
 
 
 
 
 
 
 
5351
  }
5352
  ],
5353
  "normalizer": {
 
5348
  "rstrip": false,
5349
  "normalized": false,
5350
  "special": true
5351
+ },
5352
+ {
5353
+ "id": 100594,
5354
+ "content": "<image_placeholder>",
5355
+ "single_word": false,
5356
+ "lstrip": false,
5357
+ "rstrip": false,
5358
+ "normalized": false,
5359
+ "special": true
5360
  }
5361
  ],
5362
  "normalizer": {