Commit
•
0fd2a54
1
Parent(s):
1c1f409
Upload tokenizer
Browse files- .gitattributes +1 -0
- tokenizer.json +0 -0
- tokenizer_config.json +1 -1
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
tokenizer.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
CHANGED
|
@@ -187,7 +187,7 @@
|
|
| 187 |
},
|
| 188 |
"bos_token": null,
|
| 189 |
"chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{'<|im_start|>user\n' + message['content'] + '<|im_end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|im_start|>assistant\n' + message['content'] + '<|im_end|>\n' }}{% else %}{{ '<|im_start|>system\n' + message['content'] + '<|im_end|>\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
|
| 190 |
-
"clean_up_tokenization_spaces":
|
| 191 |
"eos_token": "<|im_end|>",
|
| 192 |
"model_max_length": 1000000000000000019884624838656,
|
| 193 |
"pad_token": "<|PAD_TOKEN|>",
|
|
|
|
| 187 |
},
|
| 188 |
"bos_token": null,
|
| 189 |
"chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{'<|im_start|>user\n' + message['content'] + '<|im_end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|im_start|>assistant\n' + message['content'] + '<|im_end|>\n' }}{% else %}{{ '<|im_start|>system\n' + message['content'] + '<|im_end|>\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
|
| 190 |
+
"clean_up_tokenization_spaces": false,
|
| 191 |
"eos_token": "<|im_end|>",
|
| 192 |
"model_max_length": 1000000000000000019884624838656,
|
| 193 |
"pad_token": "<|PAD_TOKEN|>",
|