add tokenizer

Browse files

Files changed (6) hide show

added_tokens.json +1 -1
merges.txt +0 -0
special_tokens_map.json +1 -1
tokenizer.json +0 -0
tokenizer_config.json +1 -1
vocab.json +0 -0

added_tokens.json CHANGED Viewed

	@@ -1 +1 @@
1	- {"\n": ~~50258, "<\|endoftext\|>":~~ 50257}


1	+ {"\n": 50257}

merges.txt CHANGED Viewed

The diff for this file is too large to render. See raw diff

special_tokens_map.json CHANGED Viewed

	@@ -1 +1 @@
1	- {"bos_token": "<\|endoftext\|>", "eos_token": "<\|endoftext\|>", "unk_token": "<\|endoftext\|>"}


1	+ {"bos_token": {"content": "<\|endoftext\|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "eos_token": {"content": "<\|endoftext\|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "unk_token": {"content": "<\|endoftext\|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}}

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json CHANGED Viewed

	@@ -1 +1 @@
1	- {"unk_token": "<\|endoftext\|>", "bos_token": "<\|endoftext\|>", "eos_token": "<\|endoftext\|>", "add_prefix_space": false, "special_tokens_map_file": null, "name_or_path": "~~imthanhlv~~/~~gpt2news~~", "tokenizer_class": "GPT2Tokenizer"}

+ {"unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "special_tokens_map_file": null, "name_or_path": "danghuy1999/gpt2-viwiki", "errors": "replace", "tokenizer_class": "GPT2Tokenizer"}

vocab.json CHANGED Viewed

The diff for this file is too large to render. See raw diff