mirfan899 commited on
Commit
c2031b5
1 Parent(s): ba0d488

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +29 -29
vocab.json CHANGED
@@ -1,35 +1,35 @@
1
  {
2
- ",": 26,
3
- "-": 15,
4
- "0": 25,
5
- "1": 17,
6
- "2": 6,
7
  "?": 0,
8
- "A": 24,
9
- "B": 18,
10
- "C": 10,
11
- "D": 13,
12
- "E": 4,
13
  "F": 20,
14
- "G": 19,
15
- "H": 28,
16
- "I": 23,
17
- "J": 3,
18
- "K": 29,
19
- "L": 7,
20
- "M": 14,
21
- "N": 8,
22
- "O": 5,
23
- "P": 2,
24
- "R": 16,
25
- "S": 12,
26
- "T": 21,
27
- "U": 30,
28
- "V": 27,
29
- "W": 22,
30
- "Y": 11,
31
- "Z": 1,
32
  "[PAD]": 32,
33
  "[UNK]": 31,
34
- "|": 9
35
  }
 
1
  {
2
+ ",": 28,
3
+ "-": 16,
4
+ "0": 3,
5
+ "1": 12,
6
+ "2": 25,
7
  "?": 0,
8
+ "A": 10,
9
+ "B": 22,
10
+ "C": 5,
11
+ "D": 1,
12
+ "E": 19,
13
  "F": 20,
14
+ "G": 7,
15
+ "H": 24,
16
+ "I": 17,
17
+ "J": 29,
18
+ "K": 9,
19
+ "L": 26,
20
+ "M": 13,
21
+ "N": 15,
22
+ "O": 8,
23
+ "P": 23,
24
+ "R": 27,
25
+ "S": 11,
26
+ "T": 30,
27
+ "U": 18,
28
+ "V": 6,
29
+ "W": 4,
30
+ "Y": 2,
31
+ "Z": 21,
32
  "[PAD]": 32,
33
  "[UNK]": 31,
34
+ "|": 14
35
  }