add tokenizer

added_tokens.json

@@ -0,0 +1 @@
+{"COND_LEXICAL_DIV_85": 32118, "COND_SEMANTIC_SIM_40": 32128, "COND_SYNTACTIC_DIV_5": 32150, "COND_LEXICAL_DIV_30": 32106, "COND_SEMANTIC_SIM_25": 32125, "COND_LEXICAL_DIV_90": 32119, "COND_SEMANTIC_SIM_5": 32130, "COND_SYNTACTIC_DIV_80": 32157, "COND_LEXICAL_DIV_15": 32103, "COND_LEXICAL_DIV_0": 32100, "COND_LEXICAL_DIV_95": 32120, "COND_SEMANTIC_SIM_70": 32135, "COND_SEMANTIC_SIM_15": 32123, "COND_LEXICAL_DIV_100": 32102, "COND_SYNTACTIC_DIV_35": 32147, "COND_SYNTACTIC_DIV_45": 32149, "COND_SEMANTIC_SIM_35": 32127, "COND_SEMANTIC_SIM_45": 32129, "COND_SYNTACTIC_DIV_15": 32143, "COND_LEXICAL_DIV_25": 32105, "COND_SEMANTIC_SIM_55": 32132, "COND_LEXICAL_DIV_40": 32108, "COND_LEXICAL_DIV_10": 32101, "COND_SEMANTIC_SIM_20": 32124, "COND_LEXICAL_DIV_55": 32112, "COND_LEXICAL_DIV_35": 32107, "COND_LEXICAL_DIV_80": 32117, "COND_SEMANTIC_SIM_65": 32134, "COND_SEMANTIC_SIM_95": 32140, "COND_LEXICAL_DIV_65": 32114, "COND_SYNTACTIC_DIV_0": 32141, "COND_SYNTACTIC_DIV_60": 32153, "COND_SYNTACTIC_DIV_25": 32145, "COND_SYNTACTIC_DIV_20": 32144, "COND_SYNTACTIC_DIV_70": 32155, "COND_SYNTACTIC_DIV_55": 32152, "COND_SEMANTIC_SIM_10": 32122, "COND_SYNTACTIC_DIV_50": 32151, "COND_SEMANTIC_SIM_85": 32138, "COND_LEXICAL_DIV_45": 32109, "COND_SEMANTIC_SIM_60": 32133, "COND_SYNTACTIC_DIV_30": 32146, "COND_SYNTACTIC_DIV_40": 32148, "COND_SYNTACTIC_DIV_10": 32142, "COND_SEMANTIC_SIM_0": 32121, "COND_LEXICAL_DIV_60": 32113, "COND_SYNTACTIC_DIV_75": 32156, "COND_SEMANTIC_SIM_80": 32137, "COND_SEMANTIC_SIM_75": 32136, "COND_LEXICAL_DIV_5": 32110, "COND_SYNTACTIC_DIV_65": 32154, "COND_SEMANTIC_SIM_50": 32131, "COND_LEXICAL_DIV_50": 32111, "COND_LEXICAL_DIV_70": 32115, "COND_SEMANTIC_SIM_30": 32126, "COND_SEMANTIC_SIM_90": 32139, "COND_LEXICAL_DIV_20": 32104, "COND_LEXICAL_DIV_75": 32116}

special_tokens_map.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "additional_special_tokens": ["COND_LEXICAL_DIV_0", "COND_LEXICAL_DIV_10", "COND_LEXICAL_DIV_100", "COND_LEXICAL_DIV_15", "COND_LEXICAL_DIV_20", "COND_LEXICAL_DIV_25", "COND_LEXICAL_DIV_30", "COND_LEXICAL_DIV_35", "COND_LEXICAL_DIV_40", "COND_LEXICAL_DIV_45", "COND_LEXICAL_DIV_5", "COND_LEXICAL_DIV_50", "COND_LEXICAL_DIV_55", "COND_LEXICAL_DIV_60", "COND_LEXICAL_DIV_65", "COND_LEXICAL_DIV_70", "COND_LEXICAL_DIV_75", "COND_LEXICAL_DIV_80", "COND_LEXICAL_DIV_85", "COND_LEXICAL_DIV_90", "COND_LEXICAL_DIV_95", "COND_SEMANTIC_SIM_0", "COND_SEMANTIC_SIM_10", "COND_SEMANTIC_SIM_15", "COND_SEMANTIC_SIM_20", "COND_SEMANTIC_SIM_25", "COND_SEMANTIC_SIM_30", "COND_SEMANTIC_SIM_35", "COND_SEMANTIC_SIM_40", "COND_SEMANTIC_SIM_45", "COND_SEMANTIC_SIM_5", "COND_SEMANTIC_SIM_50", "COND_SEMANTIC_SIM_55", "COND_SEMANTIC_SIM_60", "COND_SEMANTIC_SIM_65", "COND_SEMANTIC_SIM_70", "COND_SEMANTIC_SIM_75", "COND_SEMANTIC_SIM_80", "COND_SEMANTIC_SIM_85", "COND_SEMANTIC_SIM_90", "COND_SEMANTIC_SIM_95", "COND_SYNTACTIC_DIV_0", "COND_SYNTACTIC_DIV_10", "COND_SYNTACTIC_DIV_15", "COND_SYNTACTIC_DIV_20", "COND_SYNTACTIC_DIV_25", "COND_SYNTACTIC_DIV_30", "COND_SYNTACTIC_DIV_35", "COND_SYNTACTIC_DIV_40", "COND_SYNTACTIC_DIV_45", "COND_SYNTACTIC_DIV_5", "COND_SYNTACTIC_DIV_50", "COND_SYNTACTIC_DIV_55", "COND_SYNTACTIC_DIV_60", "COND_SYNTACTIC_DIV_65", "COND_SYNTACTIC_DIV_70", "COND_SYNTACTIC_DIV_75", "COND_SYNTACTIC_DIV_80"]}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d60acb128cf7b7f2536e8f38a5b18a05535c9e14c7a355904270e15b0945ea86
+size 791656

tokenizer_config.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "extra_ids": 0, "additional_special_tokens": ["COND_LEXICAL_DIV_0", "COND_LEXICAL_DIV_10", "COND_LEXICAL_DIV_100", "COND_LEXICAL_DIV_15", "COND_LEXICAL_DIV_20", "COND_LEXICAL_DIV_25", "COND_LEXICAL_DIV_30", "COND_LEXICAL_DIV_35", "COND_LEXICAL_DIV_40", "COND_LEXICAL_DIV_45", "COND_LEXICAL_DIV_5", "COND_LEXICAL_DIV_50", "COND_LEXICAL_DIV_55", "COND_LEXICAL_DIV_60", "COND_LEXICAL_DIV_65", "COND_LEXICAL_DIV_70", "COND_LEXICAL_DIV_75", "COND_LEXICAL_DIV_80", "COND_LEXICAL_DIV_85", "COND_LEXICAL_DIV_90", "COND_LEXICAL_DIV_95", "COND_SEMANTIC_SIM_0", "COND_SEMANTIC_SIM_10", "COND_SEMANTIC_SIM_15", "COND_SEMANTIC_SIM_20", "COND_SEMANTIC_SIM_25", "COND_SEMANTIC_SIM_30", "COND_SEMANTIC_SIM_35", "COND_SEMANTIC_SIM_40", "COND_SEMANTIC_SIM_45", "COND_SEMANTIC_SIM_5", "COND_SEMANTIC_SIM_50", "COND_SEMANTIC_SIM_55", "COND_SEMANTIC_SIM_60", "COND_SEMANTIC_SIM_65", "COND_SEMANTIC_SIM_70", "COND_SEMANTIC_SIM_75", "COND_SEMANTIC_SIM_80", "COND_SEMANTIC_SIM_85", "COND_SEMANTIC_SIM_90", "COND_SEMANTIC_SIM_95", "COND_SYNTACTIC_DIV_0", "COND_SYNTACTIC_DIV_10", "COND_SYNTACTIC_DIV_15", "COND_SYNTACTIC_DIV_20", "COND_SYNTACTIC_DIV_25", "COND_SYNTACTIC_DIV_30", "COND_SYNTACTIC_DIV_35", "COND_SYNTACTIC_DIV_40", "COND_SYNTACTIC_DIV_45", "COND_SYNTACTIC_DIV_5", "COND_SYNTACTIC_DIV_50", "COND_SYNTACTIC_DIV_55", "COND_SYNTACTIC_DIV_60", "COND_SYNTACTIC_DIV_65", "COND_SYNTACTIC_DIV_70", "COND_SYNTACTIC_DIV_75", "COND_SYNTACTIC_DIV_80"], "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "/dccstor/tslm-gen/experiments/cond-bleurt/outputs/t5-base-cond-parabk2-bleurt-lr5e-4-v1", "tokenizer_class": "T5Tokenizer"}