{ | |
"additional_special_tokens": [ | |
"<unk>", | |
"<s>", | |
"</s>", | |
"<|fim_prefix|>", | |
"<|fim_middle|>", | |
"<|fim_suffix|>", | |
"<|fim_pad|>", | |
"<|filename|>", | |
"<|gh_stars|>", | |
"<|issue_start|>", | |
"<|issue_comment|>", | |
"<|issue_closed|>", | |
"<|jupyter_start|>", | |
"<|jupyter_text|>", | |
"<|jupyter_code|>", | |
"<|jupyter_output|>", | |
"<|empty_output|>", | |
"<|commit_before|>", | |
"<|commit_msg|>", | |
"<|commit_after|>", | |
"<|reponame|>", | |
"<|im_start|>", | |
"<|im_end|>", | |
"<|sys_start|>", | |
"<|sys_end|>" | |
], | |
"bos_token": "<s>", | |
"eos_token": "</s>", | |
"unk_token": "<unk>" | |
} | |