add modified tokenizer.json
Browse files- tokenizer.json +7 -1
tokenizer.json
CHANGED
|
@@ -49,7 +49,13 @@
|
|
| 49 |
"normalized": false
|
| 50 |
}
|
| 51 |
],
|
| 52 |
-
"normalizer":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
"pre_tokenizer": { "type": "BertPreTokenizer" },
|
| 54 |
"post_processor": {
|
| 55 |
"type": "TemplateProcessing",
|
|
|
|
| 49 |
"normalized": false
|
| 50 |
}
|
| 51 |
],
|
| 52 |
+
"normalizer": {
|
| 53 |
+
"type": "BertNormalizer",
|
| 54 |
+
"clean_text": true,
|
| 55 |
+
"handle_chinese_chars": false,
|
| 56 |
+
"strip_accents": null,
|
| 57 |
+
"lowercase": false
|
| 58 |
+
},
|
| 59 |
"pre_tokenizer": { "type": "BertPreTokenizer" },
|
| 60 |
"post_processor": {
|
| 61 |
"type": "TemplateProcessing",
|