Yasinjan99's picture
Upload tokenizer
6b2fe00
raw
history blame contribute delete
529 Bytes
{
"[PAD]": 43,
"[UNK]": 42,
"|": 0,
"«": 1,
"»": 2,
"،": 3,
"؛": 4,
"؟": 5,
"ئ": 6,
"ا": 7,
"ب": 8,
"ت": 9,
"ج": 10,
"خ": 11,
"د": 12,
"ر": 13,
"ز": 14,
"س": 15,
"ش": 16,
"غ": 17,
"ف": 18,
"ق": 19,
"ك": 20,
"ل": 21,
"م": 22,
"ن": 23,
"و": 24,
"ى": 25,
"ي": 26,
"پ": 27,
"چ": 28,
"ژ": 29,
"ڭ": 30,
"گ": 31,
"ھ": 32,
"ۆ": 33,
"ۇ": 34,
"ۈ": 35,
"ۋ": 36,
"ې": 37,
"ە": 38,
"—": 39,
"‹": 40,
"›": 41
}