Narsil HF staff commited on
Commit
1cfbb8d
1 Parent(s): e0422e9

Fix slow test https://github.com/huggingface/transformers/issues/23400

Browse files
Files changed (1) hide show
  1. tokenizer.json +7 -1
tokenizer.json CHANGED
@@ -77,10 +77,16 @@
77
  "type_id": 0
78
  }
79
  },
 
 
 
 
 
 
80
  {
81
  "Sequence": {
82
  "id": "B",
83
- "type_id": 0
84
  }
85
  }
86
  ],
 
77
  "type_id": 0
78
  }
79
  },
80
+ {
81
+ "SpecialToken": {
82
+ "id": "<s>",
83
+ "type_id": 1
84
+ }
85
+ },
86
  {
87
  "Sequence": {
88
  "id": "B",
89
+ "type_id": 1
90
  }
91
  }
92
  ],