manjugeorge commited on
Commit
e41638a
1 Parent(s): 5093b3b

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +83 -72
vocab.json CHANGED
@@ -1,76 +1,87 @@
1
  {
2
- "[PAD]": 74,
3
- "[UNK]": 74,
4
- "|": 16,
5
- "": 34,
6
- "": 50,
7
- "": 24,
8
- "": 70,
9
- "": 25,
10
- "": 31,
11
- "": 15,
12
- "": 41,
13
- "": 13,
14
- "": 18,
15
- "": 53,
16
- "": 45,
17
- "": 58,
18
- "": 47,
19
- "": 10,
20
- "": 22,
21
- "": 33,
22
- "": 52,
23
- "": 61,
24
- "": 71,
25
- "": 59,
26
- "": 67,
27
- "": 64,
28
- "": 23,
29
- "": 65,
30
- "": 40,
31
- "": 0,
32
- "": 29,
33
- "": 66,
34
- "": 38,
35
- "": 54,
36
- "": 32,
37
- "": 63,
38
- "": 42,
39
- "": 7,
40
- "": 43,
 
 
 
 
 
 
 
41
  "ഫ": 55,
42
  "ബ": 56,
43
- "ഭ": 68,
44
- "മ": 12,
45
- "യ": 5,
46
- "ര": 57,
47
- "റ": 44,
48
- "ല": 8,
49
- "ള": 3,
50
- "ഴ": 11,
51
- "വ": 1,
52
- "ശ": 37,
53
- "ഷ": 46,
54
- "സ": 14,
55
- "ഹ": 26,
56
- "ാ": 17,
57
- "ി": 49,
58
- "ീ": 6,
59
- "ു": 28,
60
- "ൂ": 35,
61
- "ൃ": 48,
62
- "െ": 36,
63
- "േ": 62,
64
- "ൈ": 9,
65
- "ൊ": 21,
66
- "ോ": 27,
67
- "ൌ": 30,
68
- "്": 4,
69
- "ൗ": 39,
70
- "": 2,
71
- "": 19,
72
- "": 51,
73
- "": 60,
74
- "": 20,
75
- "": 69
 
 
 
 
76
  }
 
1
  {
2
+ "\n": 66,
3
+ "0": 77,
4
+ "1": 39,
5
+ "2": 28,
6
+ "3": 15,
7
+ "4": 4,
8
+ "5": 33,
9
+ "7": 11,
10
+ "9": 14,
11
+ "[PAD]": 84,
12
+ "[UNK]": 83,
13
+ "o": 73,
14
+ "|": 69,
15
+ "": 75,
16
+ "": 5,
17
+ "": 42,
18
+ "": 74,
19
+ "": 27,
20
+ "": 23,
21
+ "": 50,
22
+ "": 70,
23
+ "": 81,
24
+ "": 31,
25
+ "": 37,
26
+ "": 52,
27
+ "": 2,
28
+ "": 3,
29
+ "": 9,
30
+ "": 32,
31
+ "": 57,
32
+ "": 61,
33
+ "": 36,
34
+ "": 18,
35
+ "": 59,
36
+ "": 82,
37
+ "": 78,
38
+ "": 44,
39
+ "": 48,
40
+ "": 38,
41
+ "ണ": 49,
42
+ "ത": 41,
43
+ "ഥ": 8,
44
+ "ദ": 17,
45
+ "ധ": 79,
46
+ "ന": 65,
47
+ "പ": 51,
48
  "ഫ": 55,
49
  "ബ": 56,
50
+ "ഭ": 60,
51
+ "മ": 22,
52
+ "യ": 63,
53
+ "ര": 35,
54
+ "റ": 12,
55
+ "ല": 20,
56
+ "ള": 19,
57
+ "ഴ": 68,
58
+ "വ": 0,
59
+ "ശ": 30,
60
+ "ഷ": 13,
61
+ "സ": 45,
62
+ "ഹ": 25,
63
+ "ാ": 46,
64
+ "ി": 54,
65
+ "ീ": 64,
66
+ "ു": 26,
67
+ "ൂ": 10,
68
+ "ൃ": 53,
69
+ "െ": 29,
70
+ "േ": 58,
71
+ "ൈ": 21,
72
+ "ൊ": 72,
73
+ "ോ": 7,
74
+ "ൌ": 6,
75
+ "്": 1,
76
+ "ൗ": 76,
77
+ "": 62,
78
+ "": 16,
79
+ "": 43,
80
+ "": 34,
81
+ "": 47,
82
+ "": 24,
83
+ "‍": 80,
84
+ "“": 71,
85
+ "”": 40,
86
+ "": 67
87
  }