Samuael commited on
Commit
672bb13
1 Parent(s): 1d3ed4b

Update tokenizer_config to add <tigre>

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +3 -3
tokenizer_config.json CHANGED
@@ -58,7 +58,7 @@
58
  "special": true
59
  },
60
  "15003": {
61
- "content": "<extra_id_96>",
62
  "lstrip": true,
63
  "normalized": false,
64
  "rstrip": true,
@@ -931,7 +931,7 @@
931
  "<extra_id_93>",
932
  "<extra_id_94>",
933
  "<extra_id_95>",
934
- "<extra_id_96>",
935
  "<geez>",
936
  "<tig>",
937
  "<am>"
@@ -941,7 +941,7 @@
941
  "do_basic_tokenize": true,
942
  "do_lower_case": true,
943
  "eos_token": "</s>",
944
- "extra_ids": 97,
945
  "legacy": false,
946
  "model_max_length": 1000000000000000019884624838656,
947
  "pad_token": "<pad>",
 
58
  "special": true
59
  },
60
  "15003": {
61
+ "content": "<tigre>",
62
  "lstrip": true,
63
  "normalized": false,
64
  "rstrip": true,
 
931
  "<extra_id_93>",
932
  "<extra_id_94>",
933
  "<extra_id_95>",
934
+ "<tigre>",
935
  "<geez>",
936
  "<tig>",
937
  "<am>"
 
941
  "do_basic_tokenize": true,
942
  "do_lower_case": true,
943
  "eos_token": "</s>",
944
+ "extra_ids": 96,
945
  "legacy": false,
946
  "model_max_length": 1000000000000000019884624838656,
947
  "pad_token": "<pad>",