iamTangsang
commited on
Commit
•
0dcde82
1
Parent(s):
d3ac471
Upload processor
Browse files- added_tokens.json +2 -2
- tokenizer_config.json +2 -2
- vocab.json +1 -11
added_tokens.json
CHANGED
@@ -1,4 +1,4 @@
|
|
1 |
{
|
2 |
-
"</s>":
|
3 |
-
"<s>":
|
4 |
}
|
|
|
1 |
{
|
2 |
+
"</s>": 70,
|
3 |
+
"<s>": 69
|
4 |
}
|
tokenizer_config.json
CHANGED
@@ -16,7 +16,7 @@
|
|
16 |
"single_word": false,
|
17 |
"special": false
|
18 |
},
|
19 |
-
"
|
20 |
"content": "<s>",
|
21 |
"lstrip": false,
|
22 |
"normalized": false,
|
@@ -24,7 +24,7 @@
|
|
24 |
"single_word": false,
|
25 |
"special": true
|
26 |
},
|
27 |
-
"
|
28 |
"content": "</s>",
|
29 |
"lstrip": false,
|
30 |
"normalized": false,
|
|
|
16 |
"single_word": false,
|
17 |
"special": false
|
18 |
},
|
19 |
+
"69": {
|
20 |
"content": "<s>",
|
21 |
"lstrip": false,
|
22 |
"normalized": false,
|
|
|
24 |
"single_word": false,
|
25 |
"special": true
|
26 |
},
|
27 |
+
"70": {
|
28 |
"content": "</s>",
|
29 |
"lstrip": false,
|
30 |
"normalized": false,
|
vocab.json
CHANGED
@@ -67,15 +67,5 @@
|
|
67 |
"ॐ": 65,
|
68 |
"॑": 66,
|
69 |
"ॠ": 67,
|
70 |
-
"
|
71 |
-
"१": 69,
|
72 |
-
"२": 70,
|
73 |
-
"३": 71,
|
74 |
-
"४": 72,
|
75 |
-
"५": 73,
|
76 |
-
"६": 74,
|
77 |
-
"७": 75,
|
78 |
-
"८": 76,
|
79 |
-
"९": 77,
|
80 |
-
"॰": 78
|
81 |
}
|
|
|
67 |
"ॐ": 65,
|
68 |
"॑": 66,
|
69 |
"ॠ": 67,
|
70 |
+
"॰": 68
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
71 |
}
|