Hugging Face's logo

Unbabel
/
TowerBase-13B-v0.1

Translation
Transformers
PyTorch
llama
text-generation
text-generation-inference
Inference Endpoints
Model card Files Community
jmprcp commited on
Commit
8f14f6d
•
1 Parent(s): 6a2429f

corrected tokenizer

Browse files
Files changed (4) hide show
  1. special_tokens_map.json +0 -4
  2. tokenizer.json +0 -45
  3. tokenizer.model +3 -0
  4. tokenizer_config.json +1 -44
special_tokens_map.json CHANGED
@@ -6,7 +6,6 @@
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
- "cls_token": "<CLS>",
10
  "eos_token": {
11
  "content": "</s>",
12
  "lstrip": false,
@@ -14,9 +13,6 @@
14
  "rstrip": false,
15
  "single_word": false
16
  },
17
- "mask_token": "<MASK>",
18
- "pad_token": "<PAD>",
19
- "sep_token": "<SEP>",
20
  "unk_token": {
21
  "content": "<unk>",
22
  "lstrip": false,
 
6
  "rstrip": false,
7
  "single_word": false
8
  },
 
9
  "eos_token": {
10
  "content": "</s>",
11
  "lstrip": false,
 
13
  "rstrip": false,
14
  "single_word": false
15
  },
 
 
 
16
  "unk_token": {
17
  "content": "<unk>",
18
  "lstrip": false,
tokenizer.json CHANGED
@@ -29,51 +29,6 @@
29
  "rstrip": false,
30
  "normalized": false,
31
  "special": true
32
- },
33
- {
34
- "id": 32000,
35
- "content": "<CLS>",
36
- "single_word": false,
37
- "lstrip": false,
38
- "rstrip": false,
39
- "normalized": false,
40
- "special": true
41
- },
42
- {
43
- "id": 32001,
44
- "content": "<SEP>",
45
- "single_word": false,
46
- "lstrip": false,
47
- "rstrip": false,
48
- "normalized": false,
49
- "special": true
50
- },
51
- {
52
- "id": 32002,
53
- "content": "<EOD>",
54
- "single_word": false,
55
- "lstrip": false,
56
- "rstrip": false,
57
- "normalized": false,
58
- "special": true
59
- },
60
- {
61
- "id": 32003,
62
- "content": "<MASK>",
63
- "single_word": false,
64
- "lstrip": false,
65
- "rstrip": false,
66
- "normalized": false,
67
- "special": true
68
- },
69
- {
70
- "id": 32004,
71
- "content": "<PAD>",
72
- "single_word": false,
73
- "lstrip": false,
74
- "rstrip": false,
75
- "normalized": false,
76
- "special": true
77
  }
78
  ],
79
  "normalizer": {
 
29
  "rstrip": false,
30
  "normalized": false,
31
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
  }
33
  ],
34
  "normalizer": {
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
tokenizer_config.json CHANGED
@@ -23,58 +23,15 @@
23
  "rstrip": false,
24
  "single_word": false,
25
  "special": true
26
- },
27
- "32000": {
28
- "content": "<CLS>",
29
- "lstrip": false,
30
- "normalized": false,
31
- "rstrip": false,
32
- "single_word": false,
33
- "special": true
34
- },
35
- "32001": {
36
- "content": "<SEP>",
37
- "lstrip": false,
38
- "normalized": false,
39
- "rstrip": false,
40
- "single_word": false,
41
- "special": true
42
- },
43
- "32002": {
44
- "content": "<EOD>",
45
- "lstrip": false,
46
- "normalized": false,
47
- "rstrip": false,
48
- "single_word": false,
49
- "special": true
50
- },
51
- "32003": {
52
- "content": "<MASK>",
53
- "lstrip": false,
54
- "normalized": false,
55
- "rstrip": false,
56
- "single_word": false,
57
- "special": true
58
- },
59
- "32004": {
60
- "content": "<PAD>",
61
- "lstrip": false,
62
- "normalized": false,
63
- "rstrip": false,
64
- "single_word": false,
65
- "special": true
66
  }
67
  },
68
  "bos_token": "<s>",
69
  "clean_up_tokenization_spaces": false,
70
- "cls_token": "<CLS>",
71
  "eos_token": "</s>",
72
  "legacy": false,
73
- "mask_token": "<MASK>",
74
  "model_max_length": 1000000000000000019884624838656,
75
- "pad_token": "<PAD>",
76
  "padding_side": "right",
77
- "sep_token": "<SEP>",
78
  "sp_model_kwargs": {},
79
  "tokenizer_class": "LlamaTokenizer",
80
  "unk_token": "<unk>",
 
23
  "rstrip": false,
24
  "single_word": false,
25
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
  }
27
  },
28
  "bos_token": "<s>",
29
  "clean_up_tokenization_spaces": false,
 
30
  "eos_token": "</s>",
31
  "legacy": false,
 
32
  "model_max_length": 1000000000000000019884624838656,
33
+ "pad_token": null,
34
  "padding_side": "right",
 
35
  "sp_model_kwargs": {},
36
  "tokenizer_class": "LlamaTokenizer",
37
  "unk_token": "<unk>",