huseinzol05 commited on
Commit
8f91cdc
1 Parent(s): 3c0ec09

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +1 -0
  2. vocab.json +0 -0
tokenizer.json CHANGED
@@ -14587,6 +14587,7 @@
14587
  "end_of_word_suffix": "",
14588
  "fuse_unk": false,
14589
  "byte_fallback": false,
 
14590
  "vocab": {
14591
  "!": 0,
14592
  "\"": 1,
 
14587
  "end_of_word_suffix": "",
14588
  "fuse_unk": false,
14589
  "byte_fallback": false,
14590
+ "ignore_merges": false,
14591
  "vocab": {
14592
  "!": 0,
14593
  "\"": 1,
vocab.json CHANGED
The diff for this file is too large to render. See raw diff