karasu-OpenChat / added_tokens.json
yongtae-jp's picture
Upload tokenizer
0ffac1b
{
"<|end_of_turn|>": 32056,
"<|extra_0|>": 32000,
"<|extra_10|>": 32010,
"<|extra_11|>": 32011,
"<|extra_12|>": 32012,
"<|extra_13|>": 32013,
"<|extra_14|>": 32014,
"<|extra_15|>": 32015,
"<|extra_16|>": 32016,
"<|extra_17|>": 32017,
"<|extra_18|>": 32018,
"<|extra_19|>": 32019,
"<|extra_1|>": 32001,
"<|extra_20|>": 32020,
"<|extra_21|>": 32021,
"<|extra_22|>": 32022,
"<|extra_23|>": 32023,
"<|extra_24|>": 32024,
"<|extra_25|>": 32025,
"<|extra_26|>": 32026,
"<|extra_27|>": 32027,
"<|extra_28|>": 32028,
"<|extra_29|>": 32029,
"<|extra_2|>": 32002,
"<|extra_30|>": 32030,
"<|extra_31|>": 32031,
"<|extra_32|>": 32032,
"<|extra_33|>": 32033,
"<|extra_34|>": 32034,
"<|extra_35|>": 32035,
"<|extra_36|>": 32036,
"<|extra_37|>": 32037,
"<|extra_38|>": 32038,
"<|extra_39|>": 32039,
"<|extra_3|>": 32003,
"<|extra_40|>": 32040,
"<|extra_41|>": 32041,
"<|extra_42|>": 32042,
"<|extra_43|>": 32043,
"<|extra_44|>": 32044,
"<|extra_45|>": 32045,
"<|extra_46|>": 32046,
"<|extra_47|>": 32047,
"<|extra_48|>": 32048,
"<|extra_49|>": 32049,
"<|extra_4|>": 32004,
"<|extra_50|>": 32050,
"<|extra_51|>": 32051,
"<|extra_52|>": 32052,
"<|extra_53|>": 32053,
"<|extra_54|>": 32054,
"<|extra_5|>": 32005,
"<|extra_6|>": 32006,
"<|extra_7|>": 32007,
"<|extra_8|>": 32008,
"<|extra_9|>": 32009,
"<|pad_0|>": 32055
}