Asaf Yehoodai
commited on
Commit
•
618be9a
1
Parent(s):
0a9159d
add tokenizer
Browse files- added_tokens.json +1 -0
- source.spm +0 -0
- special_tokens_map.json +1 -0
- target.spm +0 -0
- tokenizer_config.json +1 -0
- vocab.json +0 -0
added_tokens.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"<ENT0>": 65839, "</ENT0>": 65840, "<ENT1>": 65841, "</ENT1>": 65842, "<ENT2>": 65843, "</ENT2>": 65844, "<ENT3>": 65845, "</ENT3>": 65846, "<ENT4>": 65847, "</ENT4>": 65848, "<ENT5>": 65849, "</ENT5>": 65850, "<ENT6>": 65851, "</ENT6>": 65852, "<ENT7>": 65853, "</ENT7>": 65854, "<ENT8>": 65855, "</ENT8>": 65856, "<ENT9>": 65857, "</ENT9>": 65858, "<ENT10>": 65859, "</ENT10>": 65860, "<ENT11>": 65861, "</ENT11>": 65862, "<ENT12>": 65863, "</ENT12>": 65864, "<ENT13>": 65865, "</ENT13>": 65866, "<ENT14>": 65867, "</ENT14>": 65868, "<ENT15>": 65869, "</ENT15>": 65870, "<ENT16>": 65871, "</ENT16>": 65872, "<ENT17>": 65873, "</ENT17>": 65874, "<ENT18>": 65875, "</ENT18>": 65876, "<ENT19>": 65877, "</ENT19>": 65878, "<MALE>": 65879, "</MALE>": 65880, "<FEMALE>": 65881, "</FEMALE>": 65882}
|
source.spm
ADDED
|
Binary file (799 kB). View file
|
|
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}
|
target.spm
ADDED
|
Binary file (890 kB). View file
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"source_lang": "en", "target_lang": "he", "unk_token": "<unk>", "eos_token": "</s>", "pad_token": "<pad>", "model_max_length": 512, "sp_model_kwargs": {}, "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "/cs/labs/oabend/asaf305612939/co-ref-project/mt-output/paper_models/he/en_he_gender", "tokenizer_class": "MarianTokenizer"}
|
vocab.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|