Hugging Face's logo

sardinelab
/
xTower-multishot-w-quality-and-correction

Text Generation
Transformers
Safetensors
llama
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
nunonmg commited on
Commit
7d2097c
•
1 Parent(s): 0f9429b

Upload tokenizer

Browse files
Files changed (4) hide show
  1. added_tokens.json +2 -7
  2. special_tokens_map.json +1 -22
  3. tokenizer.json +1 -46
  4. tokenizer_config.json +2 -45
added_tokens.json CHANGED
@@ -1,9 +1,4 @@
1
  {
2
- "<CLS>": 32000,
3
- "<EOD>": 32002,
4
- "<MASK>": 32003,
5
- "<PAD>": 32004,
6
- "<SEP>": 32001,
7
- "<|im_end|>": 32005,
8
- "<|im_start|>": 32006
9
  }
 
1
  {
2
+ "<|im_end|>": 32000,
3
+ "<|im_start|>": 32001
 
 
 
 
 
4
  }
special_tokens_map.json CHANGED
@@ -6,13 +6,6 @@
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
- "cls_token": {
10
- "content": "<CLS>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
  "eos_token": {
17
  "content": "<|im_end|>",
18
  "lstrip": false,
@@ -20,22 +13,8 @@
20
  "rstrip": false,
21
  "single_word": false
22
  },
23
- "mask_token": {
24
- "content": "<MASK>",
25
- "lstrip": false,
26
- "normalized": false,
27
- "rstrip": false,
28
- "single_word": false
29
- },
30
  "pad_token": {
31
- "content": "<PAD>",
32
- "lstrip": false,
33
- "normalized": false,
34
- "rstrip": false,
35
- "single_word": false
36
- },
37
- "sep_token": {
38
- "content": "<SEP>",
39
  "lstrip": false,
40
  "normalized": false,
41
  "rstrip": false,
 
6
  "rstrip": false,
7
  "single_word": false
8
  },
 
 
 
 
 
 
 
9
  "eos_token": {
10
  "content": "<|im_end|>",
11
  "lstrip": false,
 
13
  "rstrip": false,
14
  "single_word": false
15
  },
 
 
 
 
 
 
 
16
  "pad_token": {
17
+ "content": "</s>",
 
 
 
 
 
 
 
18
  "lstrip": false,
19
  "normalized": false,
20
  "rstrip": false,
tokenizer.json CHANGED
@@ -32,51 +32,6 @@
32
  },
33
  {
34
  "id": 32000,
35
- "content": "<CLS>",
36
- "single_word": false,
37
- "lstrip": false,
38
- "rstrip": false,
39
- "normalized": false,
40
- "special": true
41
- },
42
- {
43
- "id": 32001,
44
- "content": "<SEP>",
45
- "single_word": false,
46
- "lstrip": false,
47
- "rstrip": false,
48
- "normalized": false,
49
- "special": true
50
- },
51
- {
52
- "id": 32002,
53
- "content": "<EOD>",
54
- "single_word": false,
55
- "lstrip": false,
56
- "rstrip": false,
57
- "normalized": false,
58
- "special": true
59
- },
60
- {
61
- "id": 32003,
62
- "content": "<MASK>",
63
- "single_word": false,
64
- "lstrip": false,
65
- "rstrip": false,
66
- "normalized": false,
67
- "special": true
68
- },
69
- {
70
- "id": 32004,
71
- "content": "<PAD>",
72
- "single_word": false,
73
- "lstrip": false,
74
- "rstrip": false,
75
- "normalized": false,
76
- "special": true
77
- },
78
- {
79
- "id": 32005,
80
  "content": "<|im_end|>",
81
  "single_word": false,
82
  "lstrip": false,
@@ -85,7 +40,7 @@
85
  "special": true
86
  },
87
  {
88
- "id": 32006,
89
  "content": "<|im_start|>",
90
  "single_word": false,
91
  "lstrip": false,
 
32
  },
33
  {
34
  "id": 32000,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  "content": "<|im_end|>",
36
  "single_word": false,
37
  "lstrip": false,
 
40
  "special": true
41
  },
42
  {
43
+ "id": 32001,
44
  "content": "<|im_start|>",
45
  "single_word": false,
46
  "lstrip": false,
tokenizer_config.json CHANGED
@@ -27,46 +27,6 @@
27
  "special": true
28
  },
29
  "32000": {
30
- "content": "<CLS>",
31
- "lstrip": false,
32
- "normalized": false,
33
- "rstrip": false,
34
- "single_word": false,
35
- "special": true
36
- },
37
- "32001": {
38
- "content": "<SEP>",
39
- "lstrip": false,
40
- "normalized": false,
41
- "rstrip": false,
42
- "single_word": false,
43
- "special": true
44
- },
45
- "32002": {
46
- "content": "<EOD>",
47
- "lstrip": false,
48
- "normalized": false,
49
- "rstrip": false,
50
- "single_word": false,
51
- "special": true
52
- },
53
- "32003": {
54
- "content": "<MASK>",
55
- "lstrip": false,
56
- "normalized": false,
57
- "rstrip": false,
58
- "single_word": false,
59
- "special": true
60
- },
61
- "32004": {
62
- "content": "<PAD>",
63
- "lstrip": false,
64
- "normalized": false,
65
- "rstrip": false,
66
- "single_word": false,
67
- "special": true
68
- },
69
- "32005": {
70
  "content": "<|im_end|>",
71
  "lstrip": false,
72
  "normalized": false,
@@ -74,7 +34,7 @@
74
  "single_word": false,
75
  "special": true
76
  },
77
- "32006": {
78
  "content": "<|im_start|>",
79
  "lstrip": false,
80
  "normalized": false,
@@ -86,14 +46,11 @@
86
  "bos_token": "<s>",
87
  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
88
  "clean_up_tokenization_spaces": false,
89
- "cls_token": "<CLS>",
90
  "eos_token": "<|im_end|>",
91
  "legacy": false,
92
- "mask_token": "<MASK>",
93
  "model_max_length": 1000000000000000019884624838656,
94
- "pad_token": "<PAD>",
95
  "padding_side": "right",
96
- "sep_token": "<SEP>",
97
  "sp_model_kwargs": {},
98
  "tokenizer_class": "LlamaTokenizer",
99
  "unk_token": "<unk>",
 
27
  "special": true
28
  },
29
  "32000": {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  "content": "<|im_end|>",
31
  "lstrip": false,
32
  "normalized": false,
 
34
  "single_word": false,
35
  "special": true
36
  },
37
+ "32001": {
38
  "content": "<|im_start|>",
39
  "lstrip": false,
40
  "normalized": false,
 
46
  "bos_token": "<s>",
47
  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
48
  "clean_up_tokenization_spaces": false,
 
49
  "eos_token": "<|im_end|>",
50
  "legacy": false,
 
51
  "model_max_length": 1000000000000000019884624838656,
52
+ "pad_token": "</s>",
53
  "padding_side": "right",
 
54
  "sp_model_kwargs": {},
55
  "tokenizer_class": "LlamaTokenizer",
56
  "unk_token": "<unk>",