herwoww commited on
Commit
d1a02aa
1 Parent(s): ac8ecd1

Upload processor

Browse files
Files changed (2) hide show
  1. spm_char.model +2 -2
  2. tokenizer_config.json +6 -6
spm_char.model CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4f4b56b4bbf0637c59a5c9c8531362bf8ce2bcde4fcd28ce0a0ec26cdf70a905
3
- size 403344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c669267db63ef4762f4ebccdfb81ac69a3bc838e1d9bb0e7f0404d2a4f9ce247
3
+ size 403675
tokenizer_config.json CHANGED
@@ -1,7 +1,7 @@
1
  {
2
  "added_tokens_decoder": {
3
  "0": {
4
- "content": "<unk>",
5
  "lstrip": false,
6
  "normalized": false,
7
  "rstrip": false,
@@ -9,7 +9,7 @@
9
  "special": true
10
  },
11
  "1": {
12
- "content": "<s>",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
@@ -24,15 +24,15 @@
24
  "single_word": false,
25
  "special": true
26
  },
27
- "86": {
28
- "content": "<pad>",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
32
  "single_word": false,
33
  "special": true
34
  },
35
- "87": {
36
  "content": "<mask>",
37
  "lstrip": true,
38
  "normalized": true,
@@ -40,7 +40,7 @@
40
  "single_word": false,
41
  "special": true
42
  },
43
- "88": {
44
  "content": "<ctc_blank>",
45
  "lstrip": false,
46
  "normalized": true,
 
1
  {
2
  "added_tokens_decoder": {
3
  "0": {
4
+ "content": "<s>",
5
  "lstrip": false,
6
  "normalized": false,
7
  "rstrip": false,
 
9
  "special": true
10
  },
11
  "1": {
12
+ "content": "<pad>",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
 
24
  "single_word": false,
25
  "special": true
26
  },
27
+ "3": {
28
+ "content": "<unk>",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
32
  "single_word": false,
33
  "special": true
34
  },
35
+ "112": {
36
  "content": "<mask>",
37
  "lstrip": true,
38
  "normalized": true,
 
40
  "single_word": false,
41
  "special": true
42
  },
43
+ "113": {
44
  "content": "<ctc_blank>",
45
  "lstrip": false,
46
  "normalized": true,