thefrigidliquidation
commited on
Commit
•
d648763
1
Parent(s):
5034acf
Remove usage of patched tokenizer for training
Browse files- tokenizer_config.json +1 -1
tokenizer_config.json
CHANGED
@@ -19,6 +19,6 @@
|
|
19 |
"special_tokens_map_file": null,
|
20 |
"src_lang": "jpn_Jpan",
|
21 |
"tgt_lang": "eng_Latn",
|
22 |
-
"tokenizer_class": "
|
23 |
"unk_token": "<unk>"
|
24 |
}
|
|
|
19 |
"special_tokens_map_file": null,
|
20 |
"src_lang": "jpn_Jpan",
|
21 |
"tgt_lang": "eng_Latn",
|
22 |
+
"tokenizer_class": "NllbTokenizer",
|
23 |
"unk_token": "<unk>"
|
24 |
}
|