Wonder-Griffin commited on
Commit
7dec169
1 Parent(s): bb32704

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +15 -1
tokenizer_config.json CHANGED
@@ -1 +1,15 @@
1
- {"model_max_length": 512}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_type": "WordPiece",
3
+ "max_len": 512,
4
+ "padding": "max_length",
5
+ "truncation": true,
6
+ "add_special_tokens": true,
7
+ "special_tokens": {
8
+ "eos_token": " ",
9
+ "bos_token": " ",
10
+ "pad_token": " ",
11
+ "unk_token": " ",
12
+ "cls_token": " ",
13
+ "sep_token": " "
14
+ }
15
+ }