Muennighoff commited on
Commit
2c63c2f
1 Parent(s): 8afd49e

Add external models

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. app.py +137 -31
  2. results/.DS_Store +0 -0
  3. results/LASER2/AmazonCounterfactualClassification.json +43 -0
  4. results/LASER2/AmazonPolarityClassification.json +14 -0
  5. results/LASER2/AmazonReviewsClassification.json +49 -0
  6. results/LASER2/ArguAna.json +31 -0
  7. results/LASER2/ArxivClusteringP2P.json +9 -0
  8. results/LASER2/ArxivClusteringS2S.json +9 -0
  9. results/LASER2/AskUbuntuDupQuestions.json +9 -0
  10. results/LASER2/BIOSSES.json +19 -0
  11. results/LASER2/BUCC.json +31 -0
  12. results/LASER2/Banking77Classification.json +12 -0
  13. results/LASER2/BiorxivClusteringP2P.json +9 -0
  14. results/LASER2/BiorxivClusteringS2S.json +9 -0
  15. results/LASER2/CQADupstackAndroidRetrieval.json +31 -0
  16. results/LASER2/CQADupstackEnglishRetrieval.json +31 -0
  17. results/LASER2/CQADupstackGamingRetrieval.json +31 -0
  18. results/LASER2/CQADupstackGisRetrieval.json +31 -0
  19. results/LASER2/CQADupstackMathematicaRetrieval.json +31 -0
  20. results/LASER2/CQADupstackPhysicsRetrieval.json +31 -0
  21. results/LASER2/CQADupstackProgrammersRetrieval.json +31 -0
  22. results/LASER2/CQADupstackRetrieval.json +31 -0
  23. results/LASER2/CQADupstackStatsRetrieval.json +31 -0
  24. results/LASER2/CQADupstackTexRetrieval.json +31 -0
  25. results/LASER2/CQADupstackUnixRetrieval.json +31 -0
  26. results/LASER2/CQADupstackWebmastersRetrieval.json +31 -0
  27. results/LASER2/CQADupstackWordpressRetrieval.json +31 -0
  28. results/LASER2/ClimateFEVER.json +31 -0
  29. results/LASER2/DBPedia.json +31 -0
  30. results/LASER2/EmotionClassification.json +12 -0
  31. results/LASER2/FEVER.json +31 -0
  32. results/LASER2/FiQA2018.json +31 -0
  33. results/LASER2/HotpotQA.json +31 -0
  34. results/LASER2/ImdbClassification.json +14 -0
  35. results/LASER2/MSMARCO.json +58 -0
  36. results/LASER2/MTOPDomainClassification.json +49 -0
  37. results/LASER2/MTOPIntentClassification.json +49 -0
  38. results/LASER2/MassiveIntentClassification.json +364 -0
  39. results/LASER2/MassiveScenarioClassification.json +364 -0
  40. results/LASER2/MedrxivClusteringP2P.json +9 -0
  41. results/LASER2/MedrxivClusteringS2S.json +9 -0
  42. results/LASER2/MindSmallReranking.json +9 -0
  43. results/LASER2/NFCorpus.json +31 -0
  44. results/LASER2/NQ.json +31 -0
  45. results/LASER2/QuoraRetrieval.json +31 -0
  46. results/LASER2/RedditClustering.json +9 -0
  47. results/LASER2/RedditClusteringP2P.json +9 -0
  48. results/LASER2/SCIDOCS.json +31 -0
  49. results/LASER2/SICK-R.json +19 -0
  50. results/LASER2/STS12.json +19 -0
app.py CHANGED
@@ -1,3 +1,4 @@
 
1
  import gradio as gr
2
  import pandas as pd
3
  from huggingface_hub import HfApi, hf_hub_download
@@ -29,6 +30,8 @@ TASK_LIST_CLASSIFICATION = [
29
  "TweetSentimentExtractionClassification",
30
  ]
31
 
 
 
32
  TASK_LIST_CLUSTERING = [
33
  "ArxivClusteringP2P",
34
  "ArxivClusteringS2S",
@@ -74,6 +77,20 @@ TASK_LIST_RETRIEVAL = [
74
  "TRECCOVID",
75
  ]
76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
77
  TASK_LIST_STS = [
78
  "BIOSSES",
79
  "SICK-R",
@@ -87,6 +104,7 @@ TASK_LIST_STS = [
87
  "STSBenchmark",
88
  ]
89
 
 
90
 
91
  TASK_LIST_SUMMARIZATION = [
92
  "SummEval",
@@ -105,19 +123,107 @@ TASK_TO_METRIC = {
105
  "Summarization": "cos_sim_spearman",
106
  }
107
 
108
- def make_clickable_model(model_name):
109
  # Remove user from model name
110
- model_name_show = " ".join(model_name.split("/")[1:])
111
- link = "https://huggingface.co/" + model_name
 
112
  return (
113
- f'<a target="_blank" style="text-decoration: underline" href="{link}">{model_name_show}</a>'
114
  )
115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
 
117
  def get_mteb_data(tasks=["Clustering"], langs=[], cast_to_str=True, task_to_metric=TASK_TO_METRIC):
118
  api = HfApi()
119
  models = api.list_models(filter="mteb")
 
120
  df_list = []
 
 
 
 
 
 
 
 
 
 
 
 
121
  for model in models:
122
  readme_path = hf_hub_download(model.modelId, filename="README.md")
123
  meta = metadata_load(readme_path)
@@ -154,8 +260,8 @@ def get_mteb_data(tasks=["Clustering"], langs=[], cast_to_str=True, task_to_metr
154
  return df.astype(str) # Cast to str as Gradio does not accept floats
155
  return df
156
 
157
- def get_mteb_average(get_all_avgs=False):
158
- global DATA_OVERALL, DATA_CLASSIFICATION_EN, DATA_CLUSTERING, DATA_PAIR_CLASSIFICATION, DATA_RERANKING, DATA_RETRIEVAL, DATA_STS_EN, DATA_SUMMARIZATION
159
  DATA_OVERALL = get_mteb_data(
160
  tasks=[
161
  "Classification",
@@ -169,6 +275,11 @@ def get_mteb_average(get_all_avgs=False):
169
  langs=["en", "en-en"],
170
  cast_to_str=False
171
  )
 
 
 
 
 
172
 
173
  DATA_OVERALL.insert(1, f"Average ({len(TASK_LIST_EN)} datasets)", DATA_OVERALL[TASK_LIST_EN].mean(axis=1, skipna=False))
174
  DATA_OVERALL.insert(2, f"Classification Average ({len(TASK_LIST_CLASSIFICATION)} datasets)", DATA_OVERALL[TASK_LIST_CLASSIFICATION].mean(axis=1, skipna=False))
@@ -204,7 +315,7 @@ with block:
204
  gr.Markdown(f"""
205
  Massive Text Embedding Benchmark (MTEB) Leaderboard. To submit, refer to the <a href="https://github.com/embeddings-benchmark/mteb#leaderboard" target="_blank" style="text-decoration: underline">MTEB GitHub repository</a> 🤗
206
 
207
- - **Total Scores**: TODO
208
  - **Total Models**: {len(DATA_OVERALL)}
209
  - **Total Users**: TODO
210
  """)
@@ -232,7 +343,7 @@ with block:
232
  gr.Markdown("""
233
  **Bitext Mining Leaderboard 🎌**
234
 
235
- - **Metric:** Accuracy (accuracy)
236
  - **Languages:** 117
237
  """)
238
  with gr.Row():
@@ -242,7 +353,7 @@ with block:
242
  )
243
  with gr.Row():
244
  data_run = gr.Button("Refresh")
245
- task_bitext_mining = gr.Variable(value="BitextMining")
246
  data_run.click(
247
  get_mteb_data,
248
  inputs=[task_bitext_mining],
@@ -265,7 +376,7 @@ with block:
265
  )
266
  with gr.Row():
267
  data_run_classification_en = gr.Button("Refresh")
268
- task_classification_en = gr.Variable(value="Classification")
269
  lang_classification_en = gr.Variable(value=["en"])
270
  data_run_classification_en.click(
271
  get_mteb_data,
@@ -285,12 +396,12 @@ with block:
285
  """)
286
  with gr.Row():
287
  data_classification = gr.components.Dataframe(
288
- datatype=["markdown"] * 500, # hack when we don't know how many columns
289
  type="pandas",
290
  )
291
  with gr.Row():
292
  data_run = gr.Button("Refresh")
293
- task_classification = gr.Variable(value="Classification")
294
  data_run.click(
295
  get_mteb_data,
296
  inputs=[task_classification],
@@ -307,13 +418,12 @@ with block:
307
  with gr.Row():
308
  data_clustering = gr.components.Dataframe(
309
  DATA_CLUSTERING,
310
- datatype="markdown",
311
  type="pandas",
312
- col_count=(len(DATA_CLUSTERING.columns), "fixed"),
313
  )
314
  with gr.Row():
315
  data_run = gr.Button("Refresh")
316
- task_clustering = gr.Variable(value="Clustering")
317
  data_run.click(
318
  get_mteb_data,
319
  inputs=[task_clustering],
@@ -330,13 +440,12 @@ with block:
330
  with gr.Row():
331
  data_pair_classification = gr.components.Dataframe(
332
  DATA_PAIR_CLASSIFICATION,
333
- datatype="markdown",
334
  type="pandas",
335
- col_count=(len(DATA_PAIR_CLASSIFICATION.columns), "fixed"),
336
  )
337
  with gr.Row():
338
  data_run = gr.Button("Refresh")
339
- task_pair_classification = gr.Variable(value="PairClassification")
340
  data_run.click(
341
  get_mteb_data,
342
  inputs=[task_pair_classification],
@@ -358,7 +467,7 @@ with block:
358
  )
359
  with gr.Row():
360
  data_run = gr.Button("Refresh")
361
- task_retrieval = gr.Variable(value="Retrieval")
362
  data_run.click(
363
  get_mteb_data, inputs=[task_retrieval], outputs=data_retrieval
364
  )
@@ -373,13 +482,12 @@ with block:
373
  with gr.Row():
374
  data_reranking = gr.components.Dataframe(
375
  DATA_RERANKING,
376
- datatype="markdown",
377
  type="pandas",
378
- col_count=(len(DATA_RERANKING.columns), "fixed"),
379
  )
380
  with gr.Row():
381
  data_run = gr.Button("Refresh")
382
- task_reranking = gr.Variable(value="Reranking")
383
  metric_reranking = gr.Variable(value="map")
384
  data_run.click(
385
  get_mteb_data, inputs=[task_reranking], outputs=data_reranking
@@ -396,15 +504,14 @@ with block:
396
  with gr.Row():
397
  data_sts_en = gr.components.Dataframe(
398
  DATA_STS_EN,
399
- datatype="markdown",
400
  type="pandas",
401
- col_count=(len(DATA_STS_EN.columns), "fixed"),
402
  )
403
  with gr.Row():
404
- data_run_en = gr.Button("Refresh")
405
- task_sts_en = gr.Variable(value="STS")
406
  lang_sts_en = gr.Variable(value=["en", "en-en"])
407
- data_run.click(
408
  get_mteb_data,
409
  inputs=[task_sts_en, lang_sts_en],
410
  outputs=data_sts_en,
@@ -424,7 +531,7 @@ with block:
424
  )
425
  with gr.Row():
426
  data_run = gr.Button("Refresh")
427
- task_sts = gr.Variable(value="STS")
428
  data_run.click(get_mteb_data, inputs=[task_sts], outputs=data_sts)
429
  with gr.TabItem("Summarization"):
430
  with gr.Row():
@@ -436,14 +543,13 @@ with block:
436
  """)
437
  with gr.Row():
438
  data_summarization = gr.components.Dataframe(
439
- DATA_SUMMARIZATION,
440
  datatype="markdown",
441
  type="pandas",
442
- col_count=(len(DATA_SUMMARIZATION.columns), "fixed"),
443
  )
444
  with gr.Row():
445
  data_run = gr.Button("Refresh")
446
- task_summarization = gr.Variable(value="Summarization")
447
  data_run.click(
448
  get_mteb_data,
449
  inputs=[task_summarization],
 
1
+ from datasets import load_dataset
2
  import gradio as gr
3
  import pandas as pd
4
  from huggingface_hub import HfApi, hf_hub_download
 
30
  "TweetSentimentExtractionClassification",
31
  ]
32
 
33
+ TASK_LIST_CLASSIFICATION_NORM = [x.replace(" (en)", "") for x in TASK_LIST_CLASSIFICATION]
34
+
35
  TASK_LIST_CLUSTERING = [
36
  "ArxivClusteringP2P",
37
  "ArxivClusteringS2S",
 
77
  "TRECCOVID",
78
  ]
79
 
80
+ TASK_LIST_RETRIEVAL_NORM = TASK_LIST_RETRIEVAL + ["CQADupstackAndroidRetrieval",
81
+ "CQADupstackEnglishRetrieval",
82
+ "CQADupstackGamingRetrieval",
83
+ "CQADupstackGisRetrieval",
84
+ "CQADupstackMathematicaRetrieval",
85
+ "CQADupstackPhysicsRetrieval",
86
+ "CQADupstackProgrammersRetrieval",
87
+ "CQADupstackStatsRetrieval",
88
+ "CQADupstackTexRetrieval",
89
+ "CQADupstackUnixRetrieval",
90
+ "CQADupstackWebmastersRetrieval",
91
+ "CQADupstackWordpressRetrieval"
92
+ ]
93
+
94
  TASK_LIST_STS = [
95
  "BIOSSES",
96
  "SICK-R",
 
104
  "STSBenchmark",
105
  ]
106
 
107
+ TASK_LIST_STS_NORM = [x.replace(" (en)", "").replace(" (en-en)", "") for x in TASK_LIST_STS]
108
 
109
  TASK_LIST_SUMMARIZATION = [
110
  "SummEval",
 
123
  "Summarization": "cos_sim_spearman",
124
  }
125
 
126
+ def make_clickable_model(model_name, link=None):
127
  # Remove user from model name
128
+ model_name = model_name.split("/")[-1]
129
+ if link is None:
130
+ link = "https://huggingface.co/" + model_name
131
  return (
132
+ f'<a target="_blank" style="text-decoration: underline" href="{link}">{model_name}</a>'
133
  )
134
 
135
+ # Models without metadata, thus we cannot fetch their results naturally
136
+ EXTERNAL_MODELS = [
137
+ "LASER2",
138
+ "LaBSE",
139
+ "all-MiniLM-L12-v2",
140
+ "all-MiniLM-L6-v2",
141
+ "all-mpnet-base-v2",
142
+ "allenai-specter",
143
+ "bert-base-uncased",
144
+ "contriever-base-msmarco",
145
+ "glove.6B.300d",
146
+ "gtr-t5-base",
147
+ "gtr-t5-large",
148
+ "gtr-t5-xl",
149
+ "gtr-t5-xxl",
150
+ "komninos",
151
+ "msmarco-bert-co-condensor",
152
+ "paraphrase-multilingual-MiniLM-L12-v2",
153
+ "paraphrase-multilingual-mpnet-base-v2",
154
+ "sentence-t5-base",
155
+ "sentence-t5-large",
156
+ "sentence-t5-xl",
157
+ "sentence-t5-xxl",
158
+ "sgpt-bloom-1b3-nli",
159
+ "sgpt-bloom-7b1-msmarco",
160
+ "sgpt-nli-bloom-1b3",
161
+ "sup-simcse-bert-base-uncased",
162
+ # "text-similarity-ada-001",
163
+ "unsup-simcse-bert-base-uncased",
164
+ ]
165
+ EXTERNAL_MODEL_TO_LINK = {
166
+ "LASER2": "https://github.com/facebookresearch/LASER",
167
+ "text-similarity-ada-001": "https://beta.openai.com/docs/guides/embeddings/types-of-embedding-models",
168
+ }
169
+ EXTERNAL_MODEL_RESULTS = {model: {k: {v: []} for k, v in TASK_TO_METRIC.items()} for model in EXTERNAL_MODELS}
170
+
171
+ def add_lang(examples):
172
+ if not(examples["eval_language"]):
173
+ examples["mteb_dataset_name_with_lang"] = examples["mteb_dataset_name"]
174
+ else:
175
+ examples["mteb_dataset_name_with_lang"] = examples["mteb_dataset_name"] + f' ({examples["eval_language"]})'
176
+ return examples
177
+
178
+ def add_task(examples):
179
+ # Could be added to the dataset loading script instead
180
+ if examples["mteb_dataset_name"] in TASK_LIST_CLASSIFICATION_NORM:
181
+ examples["mteb_task"] = "Classification"
182
+ elif examples["mteb_dataset_name"] in TASK_LIST_CLUSTERING:
183
+ examples["mteb_task"] = "Clustering"
184
+ elif examples["mteb_dataset_name"] in TASK_LIST_PAIR_CLASSIFICATION:
185
+ examples["mteb_task"] = "PairClassification"
186
+ elif examples["mteb_dataset_name"] in TASK_LIST_RERANKING:
187
+ examples["mteb_task"] = "Reranking"
188
+ elif examples["mteb_dataset_name"] in TASK_LIST_RETRIEVAL_NORM:
189
+ examples["mteb_task"] = "Retrieval"
190
+ elif examples["mteb_dataset_name"] in TASK_LIST_STS_NORM:
191
+ examples["mteb_task"] = "STS"
192
+ elif examples["mteb_dataset_name"] in TASK_LIST_SUMMARIZATION:
193
+ examples["mteb_task"] = "Summarization"
194
+ else:
195
+ examples["mteb_task"] = "BitextMining"
196
+ return examples
197
+
198
+ for model in EXTERNAL_MODELS:
199
+ ds = load_dataset("mteb/results", model)
200
+ ds = ds.map(add_lang)
201
+ ds = ds.map(add_task)
202
+ base_dict = {"Model": make_clickable_model(model, link=EXTERNAL_MODEL_TO_LINK.get(model, "https://huggingface.co/mteb/leaderboard"))}
203
+ # For now only one metric per task - Could add more metrics lateron
204
+ for task, metric in TASK_TO_METRIC.items():
205
+ ds_dict = ds.filter(lambda x: (x["mteb_task"] == task) and (x["metric"] == metric))["test"].to_dict()
206
+ ds_dict = {k: round(v, 2) for k, v in zip(ds_dict["mteb_dataset_name_with_lang"], ds_dict["score"])}
207
+ EXTERNAL_MODEL_RESULTS[model][task][metric].append({**base_dict, **ds_dict})
208
+
209
 
210
  def get_mteb_data(tasks=["Clustering"], langs=[], cast_to_str=True, task_to_metric=TASK_TO_METRIC):
211
  api = HfApi()
212
  models = api.list_models(filter="mteb")
213
+ # Initialize list to models that we cannot fetch metadata from
214
  df_list = []
215
+ for model in EXTERNAL_MODEL_RESULTS:
216
+ results_list = [res for task in tasks for res in EXTERNAL_MODEL_RESULTS[model][task][task_to_metric[task]]]
217
+ if langs:
218
+ # Would be cleaner to rely on an extra language column instead
219
+ langs_format = [f"({lang})" for lang in langs]
220
+ res = {k: v for d in results_list for k, v in d.items() if any([k.split(" ")[-1] in (k, x) for x in langs_format])}
221
+ else:
222
+ res = {k: v for d in results_list for k, v in d.items()}
223
+ # Model & at least one result
224
+ if len(res) > 1:
225
+ df_list.append(res)
226
+
227
  for model in models:
228
  readme_path = hf_hub_download(model.modelId, filename="README.md")
229
  meta = metadata_load(readme_path)
 
260
  return df.astype(str) # Cast to str as Gradio does not accept floats
261
  return df
262
 
263
+ def get_mteb_average():
264
+ global DATA_OVERALL, DATA_CLASSIFICATION_EN, DATA_CLUSTERING, DATA_PAIR_CLASSIFICATION, DATA_RERANKING, DATA_RETRIEVAL, DATA_STS_EN, DATA_SUMMARIZATION, NUM_SCORES
265
  DATA_OVERALL = get_mteb_data(
266
  tasks=[
267
  "Classification",
 
275
  langs=["en", "en-en"],
276
  cast_to_str=False
277
  )
278
+ # Approximation (Missing Bitext Mining & including some nans)
279
+ NUM_SCORES = DATA_OVERALL.shape[0] * DATA_OVERALL.shape[1]
280
+
281
+ print("OVERALLDATA", DATA_OVERALL)
282
+ DATA_OVERALL.to_csv("overall.csv")
283
 
284
  DATA_OVERALL.insert(1, f"Average ({len(TASK_LIST_EN)} datasets)", DATA_OVERALL[TASK_LIST_EN].mean(axis=1, skipna=False))
285
  DATA_OVERALL.insert(2, f"Classification Average ({len(TASK_LIST_CLASSIFICATION)} datasets)", DATA_OVERALL[TASK_LIST_CLASSIFICATION].mean(axis=1, skipna=False))
 
315
  gr.Markdown(f"""
316
  Massive Text Embedding Benchmark (MTEB) Leaderboard. To submit, refer to the <a href="https://github.com/embeddings-benchmark/mteb#leaderboard" target="_blank" style="text-decoration: underline">MTEB GitHub repository</a> 🤗
317
 
318
+ - **Total Scores**: >{NUM_SCORES}
319
  - **Total Models**: {len(DATA_OVERALL)}
320
  - **Total Users**: TODO
321
  """)
 
343
  gr.Markdown("""
344
  **Bitext Mining Leaderboard 🎌**
345
 
346
+ - **Metric:** F1 (f1)
347
  - **Languages:** 117
348
  """)
349
  with gr.Row():
 
353
  )
354
  with gr.Row():
355
  data_run = gr.Button("Refresh")
356
+ task_bitext_mining = gr.Variable(value=["BitextMining"])
357
  data_run.click(
358
  get_mteb_data,
359
  inputs=[task_bitext_mining],
 
376
  )
377
  with gr.Row():
378
  data_run_classification_en = gr.Button("Refresh")
379
+ task_classification_en = gr.Variable(value=["Classification"])
380
  lang_classification_en = gr.Variable(value=["en"])
381
  data_run_classification_en.click(
382
  get_mteb_data,
 
396
  """)
397
  with gr.Row():
398
  data_classification = gr.components.Dataframe(
399
+ datatype=["markdown"] * 200, # hack when we don't know how many columns
400
  type="pandas",
401
  )
402
  with gr.Row():
403
  data_run = gr.Button("Refresh")
404
+ task_classification = gr.Variable(value=["Classification"])
405
  data_run.click(
406
  get_mteb_data,
407
  inputs=[task_classification],
 
418
  with gr.Row():
419
  data_clustering = gr.components.Dataframe(
420
  DATA_CLUSTERING,
421
+ datatype=["markdown"] * len(DATA_CLUSTERING.columns) * 2,
422
  type="pandas",
 
423
  )
424
  with gr.Row():
425
  data_run = gr.Button("Refresh")
426
+ task_clustering = gr.Variable(value=["Clustering"])
427
  data_run.click(
428
  get_mteb_data,
429
  inputs=[task_clustering],
 
440
  with gr.Row():
441
  data_pair_classification = gr.components.Dataframe(
442
  DATA_PAIR_CLASSIFICATION,
443
+ datatype=["markdown"] * len(DATA_PAIR_CLASSIFICATION.columns) * 2,
444
  type="pandas",
 
445
  )
446
  with gr.Row():
447
  data_run = gr.Button("Refresh")
448
+ task_pair_classification = gr.Variable(value=["PairClassification"])
449
  data_run.click(
450
  get_mteb_data,
451
  inputs=[task_pair_classification],
 
467
  )
468
  with gr.Row():
469
  data_run = gr.Button("Refresh")
470
+ task_retrieval = gr.Variable(value=["Retrieval"])
471
  data_run.click(
472
  get_mteb_data, inputs=[task_retrieval], outputs=data_retrieval
473
  )
 
482
  with gr.Row():
483
  data_reranking = gr.components.Dataframe(
484
  DATA_RERANKING,
485
+ datatype=["markdown"] * len(DATA_RERANKING.columns) * 2,
486
  type="pandas",
 
487
  )
488
  with gr.Row():
489
  data_run = gr.Button("Refresh")
490
+ task_reranking = gr.Variable(value=["Reranking"])
491
  metric_reranking = gr.Variable(value="map")
492
  data_run.click(
493
  get_mteb_data, inputs=[task_reranking], outputs=data_reranking
 
504
  with gr.Row():
505
  data_sts_en = gr.components.Dataframe(
506
  DATA_STS_EN,
507
+ datatype=["markdown"] * len(DATA_STS_EN.columns) * 2,
508
  type="pandas",
 
509
  )
510
  with gr.Row():
511
+ data_run_sts_en = gr.Button("Refresh")
512
+ task_sts_en = gr.Variable(value=["STS"])
513
  lang_sts_en = gr.Variable(value=["en", "en-en"])
514
+ data_run_sts_en.click(
515
  get_mteb_data,
516
  inputs=[task_sts_en, lang_sts_en],
517
  outputs=data_sts_en,
 
531
  )
532
  with gr.Row():
533
  data_run = gr.Button("Refresh")
534
+ task_sts = gr.Variable(value=["STS"])
535
  data_run.click(get_mteb_data, inputs=[task_sts], outputs=data_sts)
536
  with gr.TabItem("Summarization"):
537
  with gr.Row():
 
543
  """)
544
  with gr.Row():
545
  data_summarization = gr.components.Dataframe(
546
+ DATA_SUMMARIZATION * len(DATA_SUMMARIZATION.columns) * 2,
547
  datatype="markdown",
548
  type="pandas",
 
549
  )
550
  with gr.Row():
551
  data_run = gr.Button("Refresh")
552
+ task_summarization = gr.Variable(value=["Summarization"])
553
  data_run.click(
554
  get_mteb_data,
555
  inputs=[task_summarization],
results/.DS_Store ADDED
Binary file (6.15 kB). View file
 
results/LASER2/AmazonCounterfactualClassification.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "de": {
6
+ "accuracy": 0.6781584582441113,
7
+ "accuracy_stderr": 0.060279798858073545,
8
+ "ap": 0.8036240553535807,
9
+ "ap_stderr": 0.03476499899077643,
10
+ "f1": 0.6628493463277175,
11
+ "f1_stderr": 0.05804533112556245,
12
+ "main_score": 0.6781584582441113
13
+ },
14
+ "en": {
15
+ "accuracy": 0.7683582089552239,
16
+ "accuracy_stderr": 0.03737785483516161,
17
+ "ap": 0.40076479274021654,
18
+ "ap_stderr": 0.05081532982471566,
19
+ "f1": 0.70787800776529,
20
+ "f1_stderr": 0.03884967003850526,
21
+ "main_score": 0.7683582089552239
22
+ },
23
+ "en-ext": {
24
+ "accuracy": 0.7616941529235383,
25
+ "accuracy_stderr": 0.05609726317155699,
26
+ "ap": 0.23620239901382217,
27
+ "ap_stderr": 0.055900376704944924,
28
+ "f1": 0.6259005944326002,
29
+ "f1_stderr": 0.057023255773266515,
30
+ "main_score": 0.7616941529235383
31
+ },
32
+ "evaluation_time": 162.63,
33
+ "ja": {
34
+ "accuracy": 0.6875802997858672,
35
+ "accuracy_stderr": 0.057291619728276316,
36
+ "ap": 0.18157282477398815,
37
+ "ap_stderr": 0.0359805625991896,
38
+ "f1": 0.5601658468471795,
39
+ "f1_stderr": 0.047780178480722454,
40
+ "main_score": 0.6875802997858672
41
+ }
42
+ }
43
+ }
results/LASER2/AmazonPolarityClassification.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "accuracy": 0.6100945,
6
+ "accuracy_stderr": 0.032754584770685165,
7
+ "ap": 0.5701234744666878,
8
+ "ap_stderr": 0.023665879113998516,
9
+ "evaluation_time": 169.16,
10
+ "f1": 0.6049258458477238,
11
+ "f1_stderr": 0.03882387073720782,
12
+ "main_score": 0.6100945
13
+ }
14
+ }
results/LASER2/AmazonReviewsClassification.json ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "de": {
6
+ "accuracy": 0.31068,
7
+ "accuracy_stderr": 0.030799766232879097,
8
+ "f1": 0.2938071341251565,
9
+ "f1_stderr": 0.03301311313116498,
10
+ "main_score": 0.31068
11
+ },
12
+ "en": {
13
+ "accuracy": 0.2871,
14
+ "accuracy_stderr": 0.031948740194254914,
15
+ "f1": 0.2763831660571802,
16
+ "f1_stderr": 0.03216710195455939,
17
+ "main_score": 0.2871
18
+ },
19
+ "es": {
20
+ "accuracy": 0.32724000000000003,
21
+ "accuracy_stderr": 0.015933311018115466,
22
+ "f1": 0.310782596824498,
23
+ "f1_stderr": 0.022188814588153163,
24
+ "main_score": 0.32724000000000003
25
+ },
26
+ "evaluation_time": 295.02,
27
+ "fr": {
28
+ "accuracy": 0.31116,
29
+ "accuracy_stderr": 0.030352304690089024,
30
+ "f1": 0.2995469284574527,
31
+ "f1_stderr": 0.03141580285250744,
32
+ "main_score": 0.31116
33
+ },
34
+ "ja": {
35
+ "accuracy": 0.28935999999999995,
36
+ "accuracy_stderr": 0.024075680675735834,
37
+ "f1": 0.2818735717046802,
38
+ "f1_stderr": 0.023753772760779744,
39
+ "main_score": 0.28935999999999995
40
+ },
41
+ "zh": {
42
+ "accuracy": 0.30892000000000003,
43
+ "accuracy_stderr": 0.02032696730946355,
44
+ "f1": 0.2990186813313857,
45
+ "f1_stderr": 0.021581437215568936,
46
+ "main_score": 0.30892000000000003
47
+ }
48
+ }
49
+ }
results/LASER2/ArguAna.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 62.18,
4
+ "map_at_1": 0.06117,
5
+ "map_at_10": 0.10455,
6
+ "map_at_100": 0.11002,
7
+ "map_at_1000": 0.11084,
8
+ "map_at_3": 0.09187,
9
+ "map_at_5": 0.09834,
10
+ "ndcg_at_1": 0.06117,
11
+ "ndcg_at_10": 0.12856,
12
+ "ndcg_at_100": 0.1601,
13
+ "ndcg_at_1000": 0.18712,
14
+ "ndcg_at_3": 0.10188,
15
+ "ndcg_at_5": 0.11358,
16
+ "precision_at_1": 0.06117,
17
+ "precision_at_10": 0.02055,
18
+ "precision_at_100": 0.00365,
19
+ "precision_at_1000": 0.00059,
20
+ "precision_at_3": 0.04362,
21
+ "precision_at_5": 0.03186,
22
+ "recall_at_1": 0.06117,
23
+ "recall_at_10": 0.20555,
24
+ "recall_at_100": 0.36486,
25
+ "recall_at_1000": 0.58962,
26
+ "recall_at_3": 0.13087,
27
+ "recall_at_5": 0.15932
28
+ },
29
+ "dataset_version": null,
30
+ "mteb_version": "0.0.2"
31
+ }
results/LASER2/ArxivClusteringP2P.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 3687.79,
4
+ "v_measure": 0.1776823856238192,
5
+ "v_measure_std": 0.15680242731305624
6
+ },
7
+ "dataset_version": null,
8
+ "mteb_version": "0.0.2"
9
+ }
results/LASER2/ArxivClusteringS2S.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 398.92,
4
+ "v_measure": 0.1239260518556585,
5
+ "v_measure_std": 0.16362867463758127
6
+ },
7
+ "dataset_version": null,
8
+ "mteb_version": "0.0.2"
9
+ }
results/LASER2/AskUbuntuDupQuestions.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 12.12,
4
+ "map": 0.4898595820868522,
5
+ "mrr": 0.6257276964340676
6
+ },
7
+ "dataset_version": null,
8
+ "mteb_version": "0.0.2"
9
+ }
results/LASER2/BIOSSES.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "cos_sim": {
4
+ "pearson": 0.6212936580597783,
5
+ "spearman": 0.6200864182463187
6
+ },
7
+ "euclidean": {
8
+ "pearson": 0.6471589936856987,
9
+ "spearman": 0.68256374544906
10
+ },
11
+ "evaluation_time": 10.33,
12
+ "manhattan": {
13
+ "pearson": 0.66773509051844,
14
+ "spearman": 0.7192212072202181
15
+ }
16
+ },
17
+ "dataset_version": null,
18
+ "mteb_version": "0.0.2"
19
+ }
results/LASER2/BUCC.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "de-en": {
6
+ "accuracy": 0.9926931106471816,
7
+ "f1": 0.9921016005567153,
8
+ "precision": 0.9918058455114822,
9
+ "recall": 0.9926931106471816
10
+ },
11
+ "evaluation_time": 642.77,
12
+ "fr-en": {
13
+ "accuracy": 0.985362095531587,
14
+ "f1": 0.9838946364370093,
15
+ "precision": 0.9831609068897205,
16
+ "recall": 0.985362095531587
17
+ },
18
+ "ru-en": {
19
+ "accuracy": 0.9774160027710426,
20
+ "f1": 0.9762152176423046,
21
+ "precision": 0.9756148250779356,
22
+ "recall": 0.9774160027710426
23
+ },
24
+ "zh-en": {
25
+ "accuracy": 0.9778830963665087,
26
+ "f1": 0.9770054414604177,
27
+ "precision": 0.9765666140073723,
28
+ "recall": 0.9778830963665087
29
+ }
30
+ }
31
+ }
results/LASER2/Banking77Classification.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "accuracy": 0.5775974025974026,
4
+ "accuracy_stderr": 0.01037438007056524,
5
+ "evaluation_time": 121.97,
6
+ "f1": 0.5693279554001911,
7
+ "f1_stderr": 0.010505608700462006,
8
+ "main_score": 0.5775974025974026
9
+ },
10
+ "dataset_version": null,
11
+ "mteb_version": "0.0.2"
12
+ }
results/LASER2/BiorxivClusteringP2P.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 768.68,
4
+ "v_measure": 0.12399936477309108,
5
+ "v_measure_std": 0.005255888066724005
6
+ },
7
+ "dataset_version": null,
8
+ "mteb_version": "0.0.2"
9
+ }
results/LASER2/BiorxivClusteringS2S.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 85.25,
4
+ "v_measure": 0.08827421024926384,
5
+ "v_measure_std": 0.005326348835553592
6
+ },
7
+ "dataset_version": null,
8
+ "mteb_version": "0.0.2"
9
+ }
results/LASER2/CQADupstackAndroidRetrieval.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "evaluation_time": 91.48,
6
+ "map_at_1": 0.0,
7
+ "map_at_10": 0.0,
8
+ "map_at_100": 0.0,
9
+ "map_at_1000": 0.0,
10
+ "map_at_3": 0.0,
11
+ "map_at_5": 0.0,
12
+ "ndcg_at_1": 0.0,
13
+ "ndcg_at_10": 0.0,
14
+ "ndcg_at_100": 0.0,
15
+ "ndcg_at_1000": 0.0,
16
+ "ndcg_at_3": 0.0,
17
+ "ndcg_at_5": 0.0,
18
+ "precision_at_1": 0.0,
19
+ "precision_at_10": 0.0,
20
+ "precision_at_100": 0.0,
21
+ "precision_at_1000": 0.0,
22
+ "precision_at_3": 0.0,
23
+ "precision_at_5": 0.0,
24
+ "recall_at_1": 0.0,
25
+ "recall_at_10": 0.0,
26
+ "recall_at_100": 0.0,
27
+ "recall_at_1000": 0.0,
28
+ "recall_at_3": 0.0,
29
+ "recall_at_5": 0.0
30
+ }
31
+ }
results/LASER2/CQADupstackEnglishRetrieval.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "evaluation_time": 101.88,
6
+ "map_at_1": 0.0418,
7
+ "map_at_10": 0.06151,
8
+ "map_at_100": 0.06587,
9
+ "map_at_1000": 0.06677,
10
+ "map_at_3": 0.05546,
11
+ "map_at_5": 0.05904,
12
+ "ndcg_at_1": 0.0535,
13
+ "ndcg_at_10": 0.07521,
14
+ "ndcg_at_100": 0.09953,
15
+ "ndcg_at_1000": 0.12755,
16
+ "ndcg_at_3": 0.06367,
17
+ "ndcg_at_5": 0.06975,
18
+ "precision_at_1": 0.0535,
19
+ "precision_at_10": 0.01459,
20
+ "precision_at_100": 0.00331,
21
+ "precision_at_1000": 0.00076,
22
+ "precision_at_3": 0.031,
23
+ "precision_at_5": 0.02318,
24
+ "recall_at_1": 0.0418,
25
+ "recall_at_10": 0.10225,
26
+ "recall_at_100": 0.21357,
27
+ "recall_at_1000": 0.41667,
28
+ "recall_at_3": 0.06958,
29
+ "recall_at_5": 0.08551
30
+ }
31
+ }
results/LASER2/CQADupstackGamingRetrieval.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "evaluation_time": 147.83,
6
+ "map_at_1": 0.05906,
7
+ "map_at_10": 0.08391,
8
+ "map_at_100": 0.08922,
9
+ "map_at_1000": 0.09014,
10
+ "map_at_3": 0.07512,
11
+ "map_at_5": 0.07958,
12
+ "ndcg_at_1": 0.06897,
13
+ "ndcg_at_10": 0.1013,
14
+ "ndcg_at_100": 0.13153,
15
+ "ndcg_at_1000": 0.16093,
16
+ "ndcg_at_3": 0.08346,
17
+ "ndcg_at_5": 0.09095,
18
+ "precision_at_1": 0.06897,
19
+ "precision_at_10": 0.01799,
20
+ "precision_at_100": 0.00366,
21
+ "precision_at_1000": 0.00068,
22
+ "precision_at_3": 0.03845,
23
+ "precision_at_5": 0.02796,
24
+ "recall_at_1": 0.05906,
25
+ "recall_at_10": 0.1418,
26
+ "recall_at_100": 0.28657,
27
+ "recall_at_1000": 0.51596,
28
+ "recall_at_3": 0.09328,
29
+ "recall_at_5": 0.11166
30
+ }
31
+ }
results/LASER2/CQADupstackGisRetrieval.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "evaluation_time": 324.7,
6
+ "map_at_1": 0.01808,
7
+ "map_at_10": 0.02773,
8
+ "map_at_100": 0.03035,
9
+ "map_at_1000": 0.03104,
10
+ "map_at_3": 0.02467,
11
+ "map_at_5": 0.02658,
12
+ "ndcg_at_1": 0.02034,
13
+ "ndcg_at_10": 0.03397,
14
+ "ndcg_at_100": 0.05043,
15
+ "ndcg_at_1000": 0.07373,
16
+ "ndcg_at_3": 0.02754,
17
+ "ndcg_at_5": 0.03101,
18
+ "precision_at_1": 0.02034,
19
+ "precision_at_10": 0.00565,
20
+ "precision_at_100": 0.0015,
21
+ "precision_at_1000": 0.00038,
22
+ "precision_at_3": 0.01205,
23
+ "precision_at_5": 0.00904,
24
+ "recall_at_1": 0.01808,
25
+ "recall_at_10": 0.05104,
26
+ "recall_at_100": 0.13422,
27
+ "recall_at_1000": 0.31965,
28
+ "recall_at_3": 0.0339,
29
+ "recall_at_5": 0.042
30
+ }
31
+ }
results/LASER2/CQADupstackMathematicaRetrieval.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "evaluation_time": 217.6,
6
+ "map_at_1": 0.01116,
7
+ "map_at_10": 0.01733,
8
+ "map_at_100": 0.02,
9
+ "map_at_1000": 0.02081,
10
+ "map_at_3": 0.01452,
11
+ "map_at_5": 0.0161,
12
+ "ndcg_at_1": 0.01493,
13
+ "ndcg_at_10": 0.02331,
14
+ "ndcg_at_100": 0.03985,
15
+ "ndcg_at_1000": 0.06932,
16
+ "ndcg_at_3": 0.01734,
17
+ "ndcg_at_5": 0.02033,
18
+ "precision_at_1": 0.01493,
19
+ "precision_at_10": 0.00485,
20
+ "precision_at_100": 0.00162,
21
+ "precision_at_1000": 0.0005,
22
+ "precision_at_3": 0.00912,
23
+ "precision_at_5": 0.00746,
24
+ "recall_at_1": 0.01116,
25
+ "recall_at_10": 0.03629,
26
+ "recall_at_100": 0.11457,
27
+ "recall_at_1000": 0.34389,
28
+ "recall_at_3": 0.02008,
29
+ "recall_at_5": 0.02748
30
+ }
31
+ }
results/LASER2/CQADupstackPhysicsRetrieval.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "evaluation_time": 209.46,
6
+ "map_at_1": 0.03631,
7
+ "map_at_10": 0.05496,
8
+ "map_at_100": 0.05935,
9
+ "map_at_1000": 0.0603,
10
+ "map_at_3": 0.05063,
11
+ "map_at_5": 0.05293,
12
+ "ndcg_at_1": 0.04812,
13
+ "ndcg_at_10": 0.06844,
14
+ "ndcg_at_100": 0.09373,
15
+ "ndcg_at_1000": 0.12299,
16
+ "ndcg_at_3": 0.06059,
17
+ "ndcg_at_5": 0.06386,
18
+ "precision_at_1": 0.04812,
19
+ "precision_at_10": 0.01347,
20
+ "precision_at_100": 0.00319,
21
+ "precision_at_1000": 0.0007,
22
+ "precision_at_3": 0.03144,
23
+ "precision_at_5": 0.02214,
24
+ "recall_at_1": 0.03631,
25
+ "recall_at_10": 0.09384,
26
+ "recall_at_100": 0.20952,
27
+ "recall_at_1000": 0.42465,
28
+ "recall_at_3": 0.06884,
29
+ "recall_at_5": 0.07864
30
+ }
31
+ }
results/LASER2/CQADupstackProgrammersRetrieval.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "evaluation_time": 222.46,
6
+ "map_at_1": 0.03015,
7
+ "map_at_10": 0.04033,
8
+ "map_at_100": 0.04335,
9
+ "map_at_1000": 0.04404,
10
+ "map_at_3": 0.03578,
11
+ "map_at_5": 0.03706,
12
+ "ndcg_at_1": 0.04224,
13
+ "ndcg_at_10": 0.05053,
14
+ "ndcg_at_100": 0.06996,
15
+ "ndcg_at_1000": 0.09367,
16
+ "ndcg_at_3": 0.04125,
17
+ "ndcg_at_5": 0.04305,
18
+ "precision_at_1": 0.04224,
19
+ "precision_at_10": 0.00959,
20
+ "precision_at_100": 0.00232,
21
+ "precision_at_1000": 0.00052,
22
+ "precision_at_3": 0.01941,
23
+ "precision_at_5": 0.01324,
24
+ "recall_at_1": 0.03015,
25
+ "recall_at_10": 0.07077,
26
+ "recall_at_100": 0.16386,
27
+ "recall_at_1000": 0.34298,
28
+ "recall_at_3": 0.04318,
29
+ "recall_at_5": 0.0486
30
+ }
31
+ }
results/LASER2/CQADupstackRetrieval.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "evaluation_time": 485.55,
6
+ "map_at_1": 0.02353666666666667,
7
+ "map_at_10": 0.033482500000000005,
8
+ "map_at_100": 0.0360425,
9
+ "map_at_1000": 0.03663416666666667,
10
+ "map_at_3": 0.029990833333333335,
11
+ "map_at_5": 0.031795833333333336,
12
+ "ndcg_at_1": 0.02917833333333333,
13
+ "ndcg_at_10": 0.04119,
14
+ "ndcg_at_100": 0.056311666666666677,
15
+ "ndcg_at_1000": 0.07594583333333334,
16
+ "ndcg_at_3": 0.034133333333333335,
17
+ "ndcg_at_5": 0.03712166666666667,
18
+ "precision_at_1": 0.02917833333333333,
19
+ "precision_at_10": 0.007701666666666667,
20
+ "precision_at_100": 0.0017991666666666666,
21
+ "precision_at_1000": 0.00041666666666666664,
22
+ "precision_at_3": 0.016209166666666667,
23
+ "precision_at_5": 0.011824999999999999,
24
+ "recall_at_1": 0.02353666666666667,
25
+ "recall_at_10": 0.0576375,
26
+ "recall_at_100": 0.1290925,
27
+ "recall_at_1000": 0.279035,
28
+ "recall_at_3": 0.03772166666666667,
29
+ "recall_at_5": 0.04548166666666667
30
+ }
31
+ }
results/LASER2/CQADupstackStatsRetrieval.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "evaluation_time": 344.51,
6
+ "map_at_1": 0.03055,
7
+ "map_at_10": 0.04272,
8
+ "map_at_100": 0.0457,
9
+ "map_at_1000": 0.04643,
10
+ "map_at_3": 0.03867,
11
+ "map_at_5": 0.04104,
12
+ "ndcg_at_1": 0.03681,
13
+ "ndcg_at_10": 0.05179,
14
+ "ndcg_at_100": 0.06878,
15
+ "ndcg_at_1000": 0.0944,
16
+ "ndcg_at_3": 0.04365,
17
+ "ndcg_at_5": 0.04723,
18
+ "precision_at_1": 0.03681,
19
+ "precision_at_10": 0.00982,
20
+ "precision_at_100": 0.00204,
21
+ "precision_at_1000": 0.00048,
22
+ "precision_at_3": 0.02096,
23
+ "precision_at_5": 0.01472,
24
+ "recall_at_1": 0.03055,
25
+ "recall_at_10": 0.0699,
26
+ "recall_at_100": 0.15083,
27
+ "recall_at_1000": 0.35566,
28
+ "recall_at_3": 0.04728,
29
+ "recall_at_5": 0.05725
30
+ }
31
+ }
results/LASER2/CQADupstackTexRetrieval.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "evaluation_time": 1027.41,
6
+ "map_at_1": 0.02132,
7
+ "map_at_10": 0.02908,
8
+ "map_at_100": 0.03136,
9
+ "map_at_1000": 0.03195,
10
+ "map_at_3": 0.02581,
11
+ "map_at_5": 0.02767,
12
+ "ndcg_at_1": 0.02512,
13
+ "ndcg_at_10": 0.03569,
14
+ "ndcg_at_100": 0.04952,
15
+ "ndcg_at_1000": 0.06975,
16
+ "ndcg_at_3": 0.02911,
17
+ "ndcg_at_5": 0.03214,
18
+ "precision_at_1": 0.02512,
19
+ "precision_at_10": 0.00664,
20
+ "precision_at_100": 0.0017,
21
+ "precision_at_1000": 0.00042,
22
+ "precision_at_3": 0.01342,
23
+ "precision_at_5": 0.00998,
24
+ "recall_at_1": 0.02132,
25
+ "recall_at_10": 0.05016,
26
+ "recall_at_100": 0.11483,
27
+ "recall_at_1000": 0.26865,
28
+ "recall_at_3": 0.03175,
29
+ "recall_at_5": 0.0398
30
+ }
31
+ }
results/LASER2/CQADupstackUnixRetrieval.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "evaluation_time": 528.37,
6
+ "map_at_1": 0.03401,
7
+ "map_at_10": 0.04402,
8
+ "map_at_100": 0.04709,
9
+ "map_at_1000": 0.04787,
10
+ "map_at_3": 0.03923,
11
+ "map_at_5": 0.04155,
12
+ "ndcg_at_1": 0.04011,
13
+ "ndcg_at_10": 0.05332,
14
+ "ndcg_at_100": 0.07162,
15
+ "ndcg_at_1000": 0.09671,
16
+ "ndcg_at_3": 0.04299,
17
+ "ndcg_at_5": 0.04714,
18
+ "precision_at_1": 0.04011,
19
+ "precision_at_10": 0.00942,
20
+ "precision_at_100": 0.00215,
21
+ "precision_at_1000": 0.0005,
22
+ "precision_at_3": 0.01866,
23
+ "precision_at_5": 0.01418,
24
+ "recall_at_1": 0.03401,
25
+ "recall_at_10": 0.07361,
26
+ "recall_at_100": 0.15881,
27
+ "recall_at_1000": 0.34893,
28
+ "recall_at_3": 0.04477,
29
+ "recall_at_5": 0.05484
30
+ }
31
+ }
results/LASER2/CQADupstackWebmastersRetrieval.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "evaluation_time": 16.59,
6
+ "map_at_1": 0.0,
7
+ "map_at_10": 0.0002,
8
+ "map_at_100": 0.00022,
9
+ "map_at_1000": 0.00026,
10
+ "map_at_3": 0.0,
11
+ "map_at_5": 0.0,
12
+ "ndcg_at_1": 0.0,
13
+ "ndcg_at_10": 0.00072,
14
+ "ndcg_at_100": 0.00079,
15
+ "ndcg_at_1000": 0.0023,
16
+ "ndcg_at_3": 0.0,
17
+ "ndcg_at_5": 0.0,
18
+ "precision_at_1": 0.0,
19
+ "precision_at_10": 0.0004,
20
+ "precision_at_100": 0.0001,
21
+ "precision_at_1000": 6e-05,
22
+ "precision_at_3": 0.0,
23
+ "precision_at_5": 0.0,
24
+ "recall_at_1": 0.0,
25
+ "recall_at_10": 0.00199,
26
+ "recall_at_100": 0.00233,
27
+ "recall_at_1000": 0.01138,
28
+ "recall_at_3": 0.0,
29
+ "recall_at_5": 0.0
30
+ }
31
+ }
results/LASER2/CQADupstackWordpressRetrieval.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "evaluation_time": 485.55,
6
+ "map_at_1": 0.0,
7
+ "map_at_10": 0.0,
8
+ "map_at_100": 0.0,
9
+ "map_at_1000": 0.0,
10
+ "map_at_3": 0.0,
11
+ "map_at_5": 0.0,
12
+ "ndcg_at_1": 0.0,
13
+ "ndcg_at_10": 0.0,
14
+ "ndcg_at_100": 0.0,
15
+ "ndcg_at_1000": 0.0,
16
+ "ndcg_at_3": 0.0,
17
+ "ndcg_at_5": 0.0,
18
+ "precision_at_1": 0.0,
19
+ "precision_at_10": 0.0,
20
+ "precision_at_100": 0.0,
21
+ "precision_at_1000": 0.0,
22
+ "precision_at_3": 0.0,
23
+ "precision_at_5": 0.0,
24
+ "recall_at_1": 0.0,
25
+ "recall_at_10": 0.0,
26
+ "recall_at_100": 0.0,
27
+ "recall_at_1000": 0.0,
28
+ "recall_at_3": 0.0,
29
+ "recall_at_5": 0.0
30
+ }
31
+ }
results/LASER2/ClimateFEVER.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 14616.31,
4
+ "map_at_1": 0.0017,
5
+ "map_at_10": 0.00242,
6
+ "map_at_100": 0.00249,
7
+ "map_at_1000": 0.00253,
8
+ "map_at_3": 0.00216,
9
+ "map_at_5": 0.00236,
10
+ "ndcg_at_1": 0.00391,
11
+ "ndcg_at_10": 0.0036,
12
+ "ndcg_at_100": 0.00436,
13
+ "ndcg_at_1000": 0.00594,
14
+ "ndcg_at_3": 0.00321,
15
+ "ndcg_at_5": 0.0034,
16
+ "precision_at_1": 0.00391,
17
+ "precision_at_10": 0.00098,
18
+ "precision_at_100": 0.00019,
19
+ "precision_at_1000": 5e-05,
20
+ "precision_at_3": 0.00239,
21
+ "precision_at_5": 0.00169,
22
+ "recall_at_1": 0.0017,
23
+ "recall_at_10": 0.00409,
24
+ "recall_at_100": 0.00711,
25
+ "recall_at_1000": 0.0171,
26
+ "recall_at_3": 0.00279,
27
+ "recall_at_5": 0.00366
28
+ },
29
+ "dataset_version": null,
30
+ "mteb_version": "0.0.2"
31
+ }
results/LASER2/DBPedia.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 5917.84,
4
+ "map_at_1": 0.00141,
5
+ "map_at_10": 0.00304,
6
+ "map_at_100": 0.0043,
7
+ "map_at_1000": 0.0048,
8
+ "map_at_3": 0.00219,
9
+ "map_at_5": 0.00252,
10
+ "ndcg_at_1": 0.0275,
11
+ "ndcg_at_10": 0.01526,
12
+ "ndcg_at_100": 0.0154,
13
+ "ndcg_at_1000": 0.02439,
14
+ "ndcg_at_3": 0.0202,
15
+ "ndcg_at_5": 0.01735,
16
+ "precision_at_1": 0.04,
17
+ "precision_at_10": 0.016,
18
+ "precision_at_100": 0.00582,
19
+ "precision_at_1000": 0.00199,
20
+ "precision_at_3": 0.0275,
21
+ "precision_at_5": 0.021,
22
+ "recall_at_1": 0.00141,
23
+ "recall_at_10": 0.00557,
24
+ "recall_at_100": 0.01832,
25
+ "recall_at_1000": 0.05513,
26
+ "recall_at_3": 0.00273,
27
+ "recall_at_5": 0.00369
28
+ },
29
+ "dataset_version": null,
30
+ "mteb_version": "0.0.2"
31
+ }
results/LASER2/EmotionClassification.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "accuracy": 0.24830000000000002,
4
+ "accuracy_stderr": 0.03350611884417532,
5
+ "evaluation_time": 114.99,
6
+ "f1": 0.22363001081759562,
7
+ "f1_stderr": 0.028095131986252592,
8
+ "main_score": 0.24830000000000002
9
+ },
10
+ "dataset_version": null,
11
+ "mteb_version": "0.0.2"
12
+ }
results/LASER2/FEVER.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 16228.4,
4
+ "map_at_1": 0.00572,
5
+ "map_at_10": 0.00687,
6
+ "map_at_100": 0.00718,
7
+ "map_at_1000": 0.00724,
8
+ "map_at_3": 0.0064,
9
+ "map_at_5": 0.00666,
10
+ "ndcg_at_1": 0.006,
11
+ "ndcg_at_10": 0.00767,
12
+ "ndcg_at_100": 0.0093,
13
+ "ndcg_at_1000": 0.01125,
14
+ "ndcg_at_3": 0.00672,
15
+ "ndcg_at_5": 0.00718,
16
+ "precision_at_1": 0.006,
17
+ "precision_at_10": 0.00104,
18
+ "precision_at_100": 0.00019,
19
+ "precision_at_1000": 4e-05,
20
+ "precision_at_3": 0.00255,
21
+ "precision_at_5": 0.00177,
22
+ "recall_at_1": 0.00572,
23
+ "recall_at_10": 0.01,
24
+ "recall_at_100": 0.0178,
25
+ "recall_at_1000": 0.03382,
26
+ "recall_at_3": 0.0073,
27
+ "recall_at_5": 0.0085
28
+ },
29
+ "dataset_version": null,
30
+ "mteb_version": "0.0.2"
31
+ }
results/LASER2/FiQA2018.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 233.94,
4
+ "map_at_1": 0.00421,
5
+ "map_at_10": 0.01026,
6
+ "map_at_100": 0.01161,
7
+ "map_at_1000": 0.01212,
8
+ "map_at_3": 0.00814,
9
+ "map_at_5": 0.00903,
10
+ "ndcg_at_1": 0.00926,
11
+ "ndcg_at_10": 0.01725,
12
+ "ndcg_at_100": 0.02736,
13
+ "ndcg_at_1000": 0.04678,
14
+ "ndcg_at_3": 0.0134,
15
+ "ndcg_at_5": 0.01445,
16
+ "precision_at_1": 0.00926,
17
+ "precision_at_10": 0.00602,
18
+ "precision_at_100": 0.00182,
19
+ "precision_at_1000": 0.00052,
20
+ "precision_at_3": 0.0108,
21
+ "precision_at_5": 0.00864,
22
+ "recall_at_1": 0.00421,
23
+ "recall_at_10": 0.02583,
24
+ "recall_at_100": 0.06514,
25
+ "recall_at_1000": 0.19262,
26
+ "recall_at_3": 0.01344,
27
+ "recall_at_5": 0.01738
28
+ },
29
+ "dataset_version": null,
30
+ "mteb_version": "0.0.2"
31
+ }
results/LASER2/HotpotQA.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 7848.6,
4
+ "map_at_1": 0.03167,
5
+ "map_at_10": 0.04056,
6
+ "map_at_100": 0.04202,
7
+ "map_at_1000": 0.04232,
8
+ "map_at_3": 0.03754,
9
+ "map_at_5": 0.03914,
10
+ "ndcg_at_1": 0.06334,
11
+ "ndcg_at_10": 0.05504,
12
+ "ndcg_at_100": 0.06469,
13
+ "ndcg_at_1000": 0.0756,
14
+ "ndcg_at_3": 0.04785,
15
+ "ndcg_at_5": 0.0512,
16
+ "precision_at_1": 0.06334,
17
+ "precision_at_10": 0.01214,
18
+ "precision_at_100": 0.00201,
19
+ "precision_at_1000": 0.00035,
20
+ "precision_at_3": 0.02957,
21
+ "precision_at_5": 0.02039,
22
+ "recall_at_1": 0.03167,
23
+ "recall_at_10": 0.0607,
24
+ "recall_at_100": 0.10061,
25
+ "recall_at_1000": 0.17508,
26
+ "recall_at_3": 0.04436,
27
+ "recall_at_5": 0.05098
28
+ },
29
+ "dataset_version": null,
30
+ "mteb_version": "0.0.2"
31
+ }
results/LASER2/ImdbClassification.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "accuracy": 0.57584,
4
+ "accuracy_stderr": 0.027472639480035414,
5
+ "ap": 0.5455790619149001,
6
+ "ap_stderr": 0.0190473646119646,
7
+ "evaluation_time": 2045.42,
8
+ "f1": 0.5721092756521572,
9
+ "f1_stderr": 0.02840547286714844,
10
+ "main_score": 0.57584
11
+ },
12
+ "dataset_version": null,
13
+ "mteb_version": "0.0.2"
14
+ }
results/LASER2/MSMARCO.json ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "dev": {
5
+ "evaluation_time": 14000.54,
6
+ "map_at_1": 0.0053,
7
+ "map_at_10": 0.00883,
8
+ "map_at_100": 0.01009,
9
+ "map_at_1000": 0.01042,
10
+ "map_at_3": 0.00775,
11
+ "map_at_5": 0.00829,
12
+ "ndcg_at_1": 0.00544,
13
+ "ndcg_at_10": 0.01087,
14
+ "ndcg_at_100": 0.01866,
15
+ "ndcg_at_1000": 0.0308,
16
+ "ndcg_at_3": 0.00854,
17
+ "ndcg_at_5": 0.00953,
18
+ "precision_at_1": 0.00544,
19
+ "precision_at_10": 0.00176,
20
+ "precision_at_100": 0.00059,
21
+ "precision_at_1000": 0.00017,
22
+ "precision_at_3": 0.00368,
23
+ "precision_at_5": 0.00269,
24
+ "recall_at_1": 0.0053,
25
+ "recall_at_10": 0.01734,
26
+ "recall_at_100": 0.05722,
27
+ "recall_at_1000": 0.15848,
28
+ "recall_at_3": 0.01074,
29
+ "recall_at_5": 0.01318
30
+ },
31
+ "test": {
32
+ "evaluation_time": 11497.8,
33
+ "map_at_1": 0.00015,
34
+ "map_at_10": 0.00195,
35
+ "map_at_100": 0.00444,
36
+ "map_at_1000": 0.00628,
37
+ "map_at_3": 0.00091,
38
+ "map_at_5": 0.00131,
39
+ "ndcg_at_1": 0.03101,
40
+ "ndcg_at_10": 0.03593,
41
+ "ndcg_at_100": 0.02811,
42
+ "ndcg_at_1000": 0.04373,
43
+ "ndcg_at_3": 0.03705,
44
+ "ndcg_at_5": 0.0394,
45
+ "precision_at_1": 0.04651,
46
+ "precision_at_10": 0.05116,
47
+ "precision_at_100": 0.02256,
48
+ "precision_at_1000": 0.00751,
49
+ "precision_at_3": 0.06202,
50
+ "precision_at_5": 0.06047,
51
+ "recall_at_1": 0.00015,
52
+ "recall_at_10": 0.00374,
53
+ "recall_at_100": 0.01999,
54
+ "recall_at_1000": 0.0669,
55
+ "recall_at_3": 0.00186,
56
+ "recall_at_5": 0.00244
57
+ }
58
+ }
results/LASER2/MTOPDomainClassification.json ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "de": {
6
+ "accuracy": 0.7407720484643561,
7
+ "accuracy_stderr": 0.02097470751351384,
8
+ "f1": 0.7246747959017267,
9
+ "f1_stderr": 0.021409255315184686,
10
+ "main_score": 0.7407720484643561
11
+ },
12
+ "en": {
13
+ "accuracy": 0.7535795713634291,
14
+ "accuracy_stderr": 0.01932929756272018,
15
+ "f1": 0.7471023646717295,
16
+ "f1_stderr": 0.020603379350450358,
17
+ "main_score": 0.7535795713634291
18
+ },
19
+ "es": {
20
+ "accuracy": 0.7346897931954637,
21
+ "accuracy_stderr": 0.029275171505061988,
22
+ "f1": 0.7201695057514836,
23
+ "f1_stderr": 0.02619166045098598,
24
+ "main_score": 0.7346897931954637
25
+ },
26
+ "evaluation_time": 238.88,
27
+ "fr": {
28
+ "accuracy": 0.7226119636705294,
29
+ "accuracy_stderr": 0.017579729530633047,
30
+ "f1": 0.7138676065809201,
31
+ "f1_stderr": 0.016450702159748233,
32
+ "main_score": 0.7226119636705294
33
+ },
34
+ "hi": {
35
+ "accuracy": 0.7295087845105772,
36
+ "accuracy_stderr": 0.022688800454798208,
37
+ "f1": 0.7163844311220117,
38
+ "f1_stderr": 0.019843811193564726,
39
+ "main_score": 0.7295087845105772
40
+ },
41
+ "th": {
42
+ "accuracy": 0.7267631103074141,
43
+ "accuracy_stderr": 0.014703529100373161,
44
+ "f1": 0.7222556760062775,
45
+ "f1_stderr": 0.012793360924583051,
46
+ "main_score": 0.7267631103074141
47
+ }
48
+ }
49
+ }
results/LASER2/MTOPIntentClassification.json ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "de": {
6
+ "accuracy": 0.5162299239222317,
7
+ "accuracy_stderr": 0.01832258118122447,
8
+ "f1": 0.3256925158951781,
9
+ "f1_stderr": 0.006031602034364266,
10
+ "main_score": 0.5162299239222317
11
+ },
12
+ "en": {
13
+ "accuracy": 0.49473324213406294,
14
+ "accuracy_stderr": 0.026360858836492213,
15
+ "f1": 0.3300910657296617,
16
+ "f1_stderr": 0.012151253776376927,
17
+ "main_score": 0.49473324213406294
18
+ },
19
+ "es": {
20
+ "accuracy": 0.5274516344229486,
21
+ "accuracy_stderr": 0.01871901414866834,
22
+ "f1": 0.33394567108321266,
23
+ "f1_stderr": 0.012228976278261429,
24
+ "main_score": 0.5274516344229486
25
+ },
26
+ "evaluation_time": 288.21,
27
+ "fr": {
28
+ "accuracy": 0.5011901033510806,
29
+ "accuracy_stderr": 0.02349600059161113,
30
+ "f1": 0.33814182942295407,
31
+ "f1_stderr": 0.008555560889430382,
32
+ "main_score": 0.5011901033510806
33
+ },
34
+ "hi": {
35
+ "accuracy": 0.45546790964503403,
36
+ "accuracy_stderr": 0.019083595788646334,
37
+ "f1": 0.27716607594942344,
38
+ "f1_stderr": 0.008465873041149053,
39
+ "main_score": 0.45546790964503403
40
+ },
41
+ "th": {
42
+ "accuracy": 0.5006871609403254,
43
+ "accuracy_stderr": 0.032474576694908,
44
+ "f1": 0.3475254801351875,
45
+ "f1_stderr": 0.01064830577932404,
46
+ "main_score": 0.5006871609403254
47
+ }
48
+ }
49
+ }
results/LASER2/MassiveIntentClassification.json ADDED
@@ -0,0 +1,364 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "af": {
6
+ "accuracy": 0.38012777404169473,
7
+ "accuracy_stderr": 0.018054485755774126,
8
+ "f1": 0.36372620791961896,
9
+ "f1_stderr": 0.011605337850752963,
10
+ "main_score": 0.38012777404169473
11
+ },
12
+ "am": {
13
+ "accuracy": 0.1269670477471419,
14
+ "accuracy_stderr": 0.024740686380275122,
15
+ "f1": 0.10125430985923822,
16
+ "f1_stderr": 0.014610275424102794,
17
+ "main_score": 0.1269670477471419
18
+ },
19
+ "ar": {
20
+ "accuracy": 0.3716207128446537,
21
+ "accuracy_stderr": 0.015242658586425109,
22
+ "f1": 0.3513819183071391,
23
+ "f1_stderr": 0.00952968827923218,
24
+ "main_score": 0.3716207128446537
25
+ },
26
+ "az": {
27
+ "accuracy": 0.19979825151311364,
28
+ "accuracy_stderr": 0.023512716506208713,
29
+ "f1": 0.20755182437536165,
30
+ "f1_stderr": 0.020871688941913907,
31
+ "main_score": 0.19979825151311364
32
+ },
33
+ "bn": {
34
+ "accuracy": 0.4251176866173504,
35
+ "accuracy_stderr": 0.015417634386490787,
36
+ "f1": 0.41808038563926464,
37
+ "f1_stderr": 0.015199386985110714,
38
+ "main_score": 0.4251176866173504
39
+ },
40
+ "cy": {
41
+ "accuracy": 0.17326832548755883,
42
+ "accuracy_stderr": 0.012898104386975457,
43
+ "f1": 0.18929774881076256,
44
+ "f1_stderr": 0.011323358864015197,
45
+ "main_score": 0.17326832548755883
46
+ },
47
+ "da": {
48
+ "accuracy": 0.4560860793544048,
49
+ "accuracy_stderr": 0.01951406090649724,
50
+ "f1": 0.4499828344382091,
51
+ "f1_stderr": 0.012363163669498014,
52
+ "main_score": 0.4560860793544048
53
+ },
54
+ "de": {
55
+ "accuracy": 0.44788164088769333,
56
+ "accuracy_stderr": 0.014665941706559757,
57
+ "f1": 0.4331277340788812,
58
+ "f1_stderr": 0.01522229039527465,
59
+ "main_score": 0.44788164088769333
60
+ },
61
+ "el": {
62
+ "accuracy": 0.4670813718897109,
63
+ "accuracy_stderr": 0.016337233490102906,
64
+ "f1": 0.4610946954649364,
65
+ "f1_stderr": 0.0176594232674652,
66
+ "main_score": 0.4670813718897109
67
+ },
68
+ "en": {
69
+ "accuracy": 0.4790854068594485,
70
+ "accuracy_stderr": 0.01717352517420073,
71
+ "f1": 0.464939146287881,
72
+ "f1_stderr": 0.012710556685691056,
73
+ "main_score": 0.4790854068594485
74
+ },
75
+ "es": {
76
+ "accuracy": 0.45437121721587087,
77
+ "accuracy_stderr": 0.014659040367775445,
78
+ "f1": 0.46286614708385815,
79
+ "f1_stderr": 0.01400822366938995,
80
+ "main_score": 0.45437121721587087
81
+ },
82
+ "evaluation_time": 2187.96,
83
+ "fa": {
84
+ "accuracy": 0.45013449899125757,
85
+ "accuracy_stderr": 0.018101267052589458,
86
+ "f1": 0.4440276709633043,
87
+ "f1_stderr": 0.02139785887315828,
88
+ "main_score": 0.45013449899125757
89
+ },
90
+ "fi": {
91
+ "accuracy": 0.45938130464021515,
92
+ "accuracy_stderr": 0.019983852698166247,
93
+ "f1": 0.4467651558171412,
94
+ "f1_stderr": 0.013228992239689133,
95
+ "main_score": 0.45938130464021515
96
+ },
97
+ "fr": {
98
+ "accuracy": 0.4613315400134499,
99
+ "accuracy_stderr": 0.019672058058454584,
100
+ "f1": 0.46149191584616656,
101
+ "f1_stderr": 0.01462338783534165,
102
+ "main_score": 0.4613315400134499
103
+ },
104
+ "he": {
105
+ "accuracy": 0.42545393409549426,
106
+ "accuracy_stderr": 0.015641146997013137,
107
+ "f1": 0.406277912154974,
108
+ "f1_stderr": 0.015808885221407522,
109
+ "main_score": 0.42545393409549426
110
+ },
111
+ "hi": {
112
+ "accuracy": 0.40201748486886346,
113
+ "accuracy_stderr": 0.014870504329345259,
114
+ "f1": 0.39955511381980663,
115
+ "f1_stderr": 0.012540162228863491,
116
+ "main_score": 0.40201748486886346
117
+ },
118
+ "hu": {
119
+ "accuracy": 0.4277404169468729,
120
+ "accuracy_stderr": 0.012916849604535326,
121
+ "f1": 0.41428552778550787,
122
+ "f1_stderr": 0.01796504857153477,
123
+ "main_score": 0.4277404169468729
124
+ },
125
+ "hy": {
126
+ "accuracy": 0.28073301950235374,
127
+ "accuracy_stderr": 0.022799686666722056,
128
+ "f1": 0.27060617914010276,
129
+ "f1_stderr": 0.011002544912943118,
130
+ "main_score": 0.28073301950235374
131
+ },
132
+ "id": {
133
+ "accuracy": 0.4581035642232683,
134
+ "accuracy_stderr": 0.01719550009501075,
135
+ "f1": 0.44698433663656323,
136
+ "f1_stderr": 0.016362307533275874,
137
+ "main_score": 0.4581035642232683
138
+ },
139
+ "is": {
140
+ "accuracy": 0.39862138533960995,
141
+ "accuracy_stderr": 0.016819937773966665,
142
+ "f1": 0.3920844117598715,
143
+ "f1_stderr": 0.015188173368411415,
144
+ "main_score": 0.39862138533960995
145
+ },
146
+ "it": {
147
+ "accuracy": 0.4825487558843308,
148
+ "accuracy_stderr": 0.017093447378236502,
149
+ "f1": 0.4800020656000982,
150
+ "f1_stderr": 0.014433139843477095,
151
+ "main_score": 0.4825487558843308
152
+ },
153
+ "ja": {
154
+ "accuracy": 0.45299260255548085,
155
+ "accuracy_stderr": 0.020810866845063326,
156
+ "f1": 0.45499154063652886,
157
+ "f1_stderr": 0.015152638500548954,
158
+ "main_score": 0.45299260255548085
159
+ },
160
+ "jv": {
161
+ "accuracy": 0.24303967720242098,
162
+ "accuracy_stderr": 0.018405242193104722,
163
+ "f1": 0.24206441275862223,
164
+ "f1_stderr": 0.01230622384839183,
165
+ "main_score": 0.24303967720242098
166
+ },
167
+ "ka": {
168
+ "accuracy": 0.22700067249495626,
169
+ "accuracy_stderr": 0.011315564480105716,
170
+ "f1": 0.2262620454603394,
171
+ "f1_stderr": 0.013641428742103908,
172
+ "main_score": 0.22700067249495626
173
+ },
174
+ "km": {
175
+ "accuracy": 0.22481506388702083,
176
+ "accuracy_stderr": 0.017626681644496683,
177
+ "f1": 0.20273821769692973,
178
+ "f1_stderr": 0.010423853100542939,
179
+ "main_score": 0.22481506388702083
180
+ },
181
+ "kn": {
182
+ "accuracy": 0.043207800941492944,
183
+ "accuracy_stderr": 0.011439581992919204,
184
+ "f1": 0.019591698863663035,
185
+ "f1_stderr": 0.004395348203545399,
186
+ "main_score": 0.043207800941492944
187
+ },
188
+ "ko": {
189
+ "accuracy": 0.4426025554808339,
190
+ "accuracy_stderr": 0.024004890918808426,
191
+ "f1": 0.44147618521379844,
192
+ "f1_stderr": 0.01371530289091307,
193
+ "main_score": 0.4426025554808339
194
+ },
195
+ "lv": {
196
+ "accuracy": 0.3975453934095495,
197
+ "accuracy_stderr": 0.02350514180018239,
198
+ "f1": 0.4061787392665471,
199
+ "f1_stderr": 0.02079966697693215,
200
+ "main_score": 0.3975453934095495
201
+ },
202
+ "ml": {
203
+ "accuracy": 0.4133490248823134,
204
+ "accuracy_stderr": 0.022600758621339218,
205
+ "f1": 0.40286261835633236,
206
+ "f1_stderr": 0.019466174082752453,
207
+ "main_score": 0.4133490248823134
208
+ },
209
+ "mn": {
210
+ "accuracy": 0.16200403496973773,
211
+ "accuracy_stderr": 0.0100095627651343,
212
+ "f1": 0.16450176344768086,
213
+ "f1_stderr": 0.01474426560593057,
214
+ "main_score": 0.16200403496973773
215
+ },
216
+ "ms": {
217
+ "accuracy": 0.4322797579018157,
218
+ "accuracy_stderr": 0.01759072503596667,
219
+ "f1": 0.4248684950682716,
220
+ "f1_stderr": 0.016635102291421434,
221
+ "main_score": 0.4322797579018157
222
+ },
223
+ "my": {
224
+ "accuracy": 0.25373234700739744,
225
+ "accuracy_stderr": 0.014838536825778608,
226
+ "f1": 0.2222952356057096,
227
+ "f1_stderr": 0.013501549124465934,
228
+ "main_score": 0.25373234700739744
229
+ },
230
+ "nb": {
231
+ "accuracy": 0.3773705447209146,
232
+ "accuracy_stderr": 0.021859215543374035,
233
+ "f1": 0.3822420276063557,
234
+ "f1_stderr": 0.01781935165480971,
235
+ "main_score": 0.3773705447209146
236
+ },
237
+ "nl": {
238
+ "accuracy": 0.45000000000000007,
239
+ "accuracy_stderr": 0.016798952275448243,
240
+ "f1": 0.4390949593879808,
241
+ "f1_stderr": 0.013457897429685057,
242
+ "main_score": 0.45000000000000007
243
+ },
244
+ "pl": {
245
+ "accuracy": 0.4499327505043712,
246
+ "accuracy_stderr": 0.018087145352772453,
247
+ "f1": 0.42978932594492064,
248
+ "f1_stderr": 0.020420111856607633,
249
+ "main_score": 0.4499327505043712
250
+ },
251
+ "pt": {
252
+ "accuracy": 0.4855413584398117,
253
+ "accuracy_stderr": 0.01347928867036143,
254
+ "f1": 0.48571088423157766,
255
+ "f1_stderr": 0.013222183663454462,
256
+ "main_score": 0.4855413584398117
257
+ },
258
+ "ro": {
259
+ "accuracy": 0.44300605245460656,
260
+ "accuracy_stderr": 0.021637858328300265,
261
+ "f1": 0.44740069972880264,
262
+ "f1_stderr": 0.01577190962835917,
263
+ "main_score": 0.44300605245460656
264
+ },
265
+ "ru": {
266
+ "accuracy": 0.44290517821116343,
267
+ "accuracy_stderr": 0.021596827649085326,
268
+ "f1": 0.43747519563352766,
269
+ "f1_stderr": 0.019105088547632505,
270
+ "main_score": 0.44290517821116343
271
+ },
272
+ "sl": {
273
+ "accuracy": 0.44717552118359116,
274
+ "accuracy_stderr": 0.01726980272383147,
275
+ "f1": 0.4583434006703727,
276
+ "f1_stderr": 0.016199198460333516,
277
+ "main_score": 0.44717552118359116
278
+ },
279
+ "sq": {
280
+ "accuracy": 0.4612306657700067,
281
+ "accuracy_stderr": 0.01641084055632617,
282
+ "f1": 0.4549674955169706,
283
+ "f1_stderr": 0.013851645495939875,
284
+ "main_score": 0.4612306657700067
285
+ },
286
+ "sv": {
287
+ "accuracy": 0.4594821788836584,
288
+ "accuracy_stderr": 0.020832831141935834,
289
+ "f1": 0.44306077100248525,
290
+ "f1_stderr": 0.019118675204039762,
291
+ "main_score": 0.4594821788836584
292
+ },
293
+ "sw": {
294
+ "accuracy": 0.31893073301950237,
295
+ "accuracy_stderr": 0.011235144820777232,
296
+ "f1": 0.31391383206208073,
297
+ "f1_stderr": 0.014373830497544215,
298
+ "main_score": 0.31893073301950237
299
+ },
300
+ "ta": {
301
+ "accuracy": 0.2963349024882313,
302
+ "accuracy_stderr": 0.012793365954792035,
303
+ "f1": 0.2888382113446147,
304
+ "f1_stderr": 0.01179745104077985,
305
+ "main_score": 0.2963349024882313
306
+ },
307
+ "te": {
308
+ "accuracy": 0.36028917283120376,
309
+ "accuracy_stderr": 0.018898933014207392,
310
+ "f1": 0.3438728561319695,
311
+ "f1_stderr": 0.015037693640197282,
312
+ "main_score": 0.36028917283120376
313
+ },
314
+ "th": {
315
+ "accuracy": 0.4338601210490921,
316
+ "accuracy_stderr": 0.01838326805184409,
317
+ "f1": 0.44123009965336957,
318
+ "f1_stderr": 0.014967999591595584,
319
+ "main_score": 0.4338601210490921
320
+ },
321
+ "tl": {
322
+ "accuracy": 0.2973436449226631,
323
+ "accuracy_stderr": 0.01833399973821919,
324
+ "f1": 0.28090429460991373,
325
+ "f1_stderr": 0.012358692677216883,
326
+ "main_score": 0.2973436449226631
327
+ },
328
+ "tr": {
329
+ "accuracy": 0.43930733019502355,
330
+ "accuracy_stderr": 0.02477039571658176,
331
+ "f1": 0.4265561991600836,
332
+ "f1_stderr": 0.022060686946343926,
333
+ "main_score": 0.43930733019502355
334
+ },
335
+ "ur": {
336
+ "accuracy": 0.26109616677874925,
337
+ "accuracy_stderr": 0.011974612577756474,
338
+ "f1": 0.27607059745773804,
339
+ "f1_stderr": 0.011634207629783868,
340
+ "main_score": 0.26109616677874925
341
+ },
342
+ "vi": {
343
+ "accuracy": 0.44327505043712173,
344
+ "accuracy_stderr": 0.022061037910983305,
345
+ "f1": 0.43953676685827936,
346
+ "f1_stderr": 0.01965182342188036,
347
+ "main_score": 0.44327505043712173
348
+ },
349
+ "zh-CN": {
350
+ "accuracy": 0.40618695359784807,
351
+ "accuracy_stderr": 0.01878416975952488,
352
+ "f1": 0.40304322644486057,
353
+ "f1_stderr": 0.016358332856736345,
354
+ "main_score": 0.40618695359784807
355
+ },
356
+ "zh-TW": {
357
+ "accuracy": 0.3292871553463349,
358
+ "accuracy_stderr": 0.012964553395663082,
359
+ "f1": 0.3310118176304551,
360
+ "f1_stderr": 0.00939726932836698,
361
+ "main_score": 0.3292871553463349
362
+ }
363
+ }
364
+ }
results/LASER2/MassiveScenarioClassification.json ADDED
@@ -0,0 +1,364 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "af": {
6
+ "accuracy": 0.4710154673839947,
7
+ "accuracy_stderr": 0.018564271686437515,
8
+ "f1": 0.4451973179524046,
9
+ "f1_stderr": 0.016582236744477792,
10
+ "main_score": 0.4710154673839947
11
+ },
12
+ "am": {
13
+ "accuracy": 0.17696704774714192,
14
+ "accuracy_stderr": 0.03488947970284516,
15
+ "f1": 0.15821694371719605,
16
+ "f1_stderr": 0.018041297388910648,
17
+ "main_score": 0.17696704774714192
18
+ },
19
+ "ar": {
20
+ "accuracy": 0.4520847343644922,
21
+ "accuracy_stderr": 0.02052556073259087,
22
+ "f1": 0.4493556808838758,
23
+ "f1_stderr": 0.02219131959628148,
24
+ "main_score": 0.4520847343644922
25
+ },
26
+ "az": {
27
+ "accuracy": 0.2821116341627438,
28
+ "accuracy_stderr": 0.03302593905078847,
29
+ "f1": 0.25909697848350605,
30
+ "f1_stderr": 0.025892139767993743,
31
+ "main_score": 0.2821116341627438
32
+ },
33
+ "bn": {
34
+ "accuracy": 0.5052454606590451,
35
+ "accuracy_stderr": 0.025871484930688898,
36
+ "f1": 0.4856524051409231,
37
+ "f1_stderr": 0.02999372933773852,
38
+ "main_score": 0.5052454606590451
39
+ },
40
+ "cy": {
41
+ "accuracy": 0.2257565568258238,
42
+ "accuracy_stderr": 0.029955555426665198,
43
+ "f1": 0.22253995596411508,
44
+ "f1_stderr": 0.024577736879299292,
45
+ "main_score": 0.2257565568258238
46
+ },
47
+ "da": {
48
+ "accuracy": 0.5486550100874243,
49
+ "accuracy_stderr": 0.024402881290646793,
50
+ "f1": 0.5266171141921359,
51
+ "f1_stderr": 0.022103636920251905,
52
+ "main_score": 0.5486550100874243
53
+ },
54
+ "de": {
55
+ "accuracy": 0.543375924680565,
56
+ "accuracy_stderr": 0.021944881957252572,
57
+ "f1": 0.5252613318654367,
58
+ "f1_stderr": 0.026364483689257163,
59
+ "main_score": 0.543375924680565
60
+ },
61
+ "el": {
62
+ "accuracy": 0.5547074646940148,
63
+ "accuracy_stderr": 0.025938196719597618,
64
+ "f1": 0.5349766031105144,
65
+ "f1_stderr": 0.029002837280894762,
66
+ "main_score": 0.5547074646940148
67
+ },
68
+ "en": {
69
+ "accuracy": 0.5592131809011432,
70
+ "accuracy_stderr": 0.02460975514543639,
71
+ "f1": 0.5360164729184802,
72
+ "f1_stderr": 0.023764425252363358,
73
+ "main_score": 0.5592131809011432
74
+ },
75
+ "es": {
76
+ "accuracy": 0.5277404169468729,
77
+ "accuracy_stderr": 0.01930086885231061,
78
+ "f1": 0.5147488045351095,
79
+ "f1_stderr": 0.02102482512919075,
80
+ "main_score": 0.5277404169468729
81
+ },
82
+ "evaluation_time": 2170.8,
83
+ "fa": {
84
+ "accuracy": 0.5250168123739072,
85
+ "accuracy_stderr": 0.02656193333119369,
86
+ "f1": 0.5107120593131073,
87
+ "f1_stderr": 0.027581382894975626,
88
+ "main_score": 0.5250168123739072
89
+ },
90
+ "fi": {
91
+ "accuracy": 0.5263281775386685,
92
+ "accuracy_stderr": 0.028049993574005946,
93
+ "f1": 0.5028105658943992,
94
+ "f1_stderr": 0.029331037664081416,
95
+ "main_score": 0.5263281775386685
96
+ },
97
+ "fr": {
98
+ "accuracy": 0.5431741761936786,
99
+ "accuracy_stderr": 0.02330862085752492,
100
+ "f1": 0.5225553974023132,
101
+ "f1_stderr": 0.025371329635383216,
102
+ "main_score": 0.5431741761936786
103
+ },
104
+ "he": {
105
+ "accuracy": 0.5241425689307331,
106
+ "accuracy_stderr": 0.023080787167122134,
107
+ "f1": 0.5053900298960137,
108
+ "f1_stderr": 0.023963342674475203,
109
+ "main_score": 0.5241425689307331
110
+ },
111
+ "hi": {
112
+ "accuracy": 0.47370544720914587,
113
+ "accuracy_stderr": 0.02723430216840629,
114
+ "f1": 0.4497838830218261,
115
+ "f1_stderr": 0.029160940440385055,
116
+ "main_score": 0.47370544720914587
117
+ },
118
+ "hu": {
119
+ "accuracy": 0.5343308675184936,
120
+ "accuracy_stderr": 0.02571158946620075,
121
+ "f1": 0.5127958383061038,
122
+ "f1_stderr": 0.02800743447651377,
123
+ "main_score": 0.5343308675184936
124
+ },
125
+ "hy": {
126
+ "accuracy": 0.33574310692669806,
127
+ "accuracy_stderr": 0.0279984332671053,
128
+ "f1": 0.31782884203375084,
129
+ "f1_stderr": 0.024137129936083107,
130
+ "main_score": 0.33574310692669806
131
+ },
132
+ "id": {
133
+ "accuracy": 0.543813046402152,
134
+ "accuracy_stderr": 0.02173056397773423,
135
+ "f1": 0.5300232571973252,
136
+ "f1_stderr": 0.02368021779116069,
137
+ "main_score": 0.543813046402152
138
+ },
139
+ "is": {
140
+ "accuracy": 0.49778076664425014,
141
+ "accuracy_stderr": 0.02425237237474775,
142
+ "f1": 0.4706894365473266,
143
+ "f1_stderr": 0.026960565416528054,
144
+ "main_score": 0.49778076664425014
145
+ },
146
+ "it": {
147
+ "accuracy": 0.5483523873570949,
148
+ "accuracy_stderr": 0.0236096496608969,
149
+ "f1": 0.5335311466130319,
150
+ "f1_stderr": 0.02545386322306336,
151
+ "main_score": 0.5483523873570949
152
+ },
153
+ "ja": {
154
+ "accuracy": 0.541223940820444,
155
+ "accuracy_stderr": 0.02132943677295098,
156
+ "f1": 0.5297398634789287,
157
+ "f1_stderr": 0.02456627872645516,
158
+ "main_score": 0.541223940820444
159
+ },
160
+ "jv": {
161
+ "accuracy": 0.32713517148621385,
162
+ "accuracy_stderr": 0.035614449716496376,
163
+ "f1": 0.31631803435296824,
164
+ "f1_stderr": 0.02907835125753927,
165
+ "main_score": 0.32713517148621385
166
+ },
167
+ "ka": {
168
+ "accuracy": 0.2691997310020175,
169
+ "accuracy_stderr": 0.014025349472741056,
170
+ "f1": 0.2553767051487454,
171
+ "f1_stderr": 0.01162136511437043,
172
+ "main_score": 0.2691997310020175
173
+ },
174
+ "km": {
175
+ "accuracy": 0.2723268325487559,
176
+ "accuracy_stderr": 0.02335380952684497,
177
+ "f1": 0.2515485444512877,
178
+ "f1_stderr": 0.017257971891773076,
179
+ "main_score": 0.2723268325487559
180
+ },
181
+ "kn": {
182
+ "accuracy": 0.10063887020847344,
183
+ "accuracy_stderr": 0.024999293350221934,
184
+ "f1": 0.064525855332009,
185
+ "f1_stderr": 0.013368181011921797,
186
+ "main_score": 0.10063887020847344
187
+ },
188
+ "ko": {
189
+ "accuracy": 0.5200739744451917,
190
+ "accuracy_stderr": 0.028003601644626674,
191
+ "f1": 0.508622045242862,
192
+ "f1_stderr": 0.02866048570926957,
193
+ "main_score": 0.5200739744451917
194
+ },
195
+ "lv": {
196
+ "accuracy": 0.44821788836583726,
197
+ "accuracy_stderr": 0.03405873263075454,
198
+ "f1": 0.44404598642640175,
199
+ "f1_stderr": 0.034820002597344316,
200
+ "main_score": 0.44821788836583726
201
+ },
202
+ "ml": {
203
+ "accuracy": 0.49098856758574316,
204
+ "accuracy_stderr": 0.036953827315555375,
205
+ "f1": 0.477330044105713,
206
+ "f1_stderr": 0.04031567283455406,
207
+ "main_score": 0.49098856758574316
208
+ },
209
+ "mn": {
210
+ "accuracy": 0.2150975117686617,
211
+ "accuracy_stderr": 0.031789545064276226,
212
+ "f1": 0.19956650806636905,
213
+ "f1_stderr": 0.02753725989804031,
214
+ "main_score": 0.2150975117686617
215
+ },
216
+ "ms": {
217
+ "accuracy": 0.5360121049092131,
218
+ "accuracy_stderr": 0.024483088411342932,
219
+ "f1": 0.5186987112385978,
220
+ "f1_stderr": 0.02535803268828841,
221
+ "main_score": 0.5360121049092131
222
+ },
223
+ "my": {
224
+ "accuracy": 0.29717552118359114,
225
+ "accuracy_stderr": 0.028042233316740755,
226
+ "f1": 0.2811205919315657,
227
+ "f1_stderr": 0.027129866000958187,
228
+ "main_score": 0.29717552118359114
229
+ },
230
+ "nb": {
231
+ "accuracy": 0.439004707464694,
232
+ "accuracy_stderr": 0.027998816890400836,
233
+ "f1": 0.4292074186822793,
234
+ "f1_stderr": 0.026243622116762927,
235
+ "main_score": 0.439004707464694
236
+ },
237
+ "nl": {
238
+ "accuracy": 0.5333221250840617,
239
+ "accuracy_stderr": 0.025110033232975306,
240
+ "f1": 0.5077836780602125,
241
+ "f1_stderr": 0.027053518325785973,
242
+ "main_score": 0.5333221250840617
243
+ },
244
+ "pl": {
245
+ "accuracy": 0.5291526563550774,
246
+ "accuracy_stderr": 0.02756493168576702,
247
+ "f1": 0.5199897798231456,
248
+ "f1_stderr": 0.029788904479210038,
249
+ "main_score": 0.5291526563550774
250
+ },
251
+ "pt": {
252
+ "accuracy": 0.5341291190316073,
253
+ "accuracy_stderr": 0.0218184197703392,
254
+ "f1": 0.5199879514297052,
255
+ "f1_stderr": 0.02121614576868715,
256
+ "main_score": 0.5341291190316073
257
+ },
258
+ "ro": {
259
+ "accuracy": 0.5047747141896436,
260
+ "accuracy_stderr": 0.02555648952102798,
261
+ "f1": 0.4877286315335815,
262
+ "f1_stderr": 0.02639141604347184,
263
+ "main_score": 0.5047747141896436
264
+ },
265
+ "ru": {
266
+ "accuracy": 0.5184263618022864,
267
+ "accuracy_stderr": 0.030568502165383708,
268
+ "f1": 0.5141368540903201,
269
+ "f1_stderr": 0.033485640371287616,
270
+ "main_score": 0.5184263618022864
271
+ },
272
+ "sl": {
273
+ "accuracy": 0.5128782784129118,
274
+ "accuracy_stderr": 0.02677596327757706,
275
+ "f1": 0.5069111807663103,
276
+ "f1_stderr": 0.028984297990527246,
277
+ "main_score": 0.5128782784129118
278
+ },
279
+ "sq": {
280
+ "accuracy": 0.556523201075992,
281
+ "accuracy_stderr": 0.022907536560761797,
282
+ "f1": 0.5280391253265858,
283
+ "f1_stderr": 0.02766946734240689,
284
+ "main_score": 0.556523201075992
285
+ },
286
+ "sv": {
287
+ "accuracy": 0.5464357767316745,
288
+ "accuracy_stderr": 0.02703916904648807,
289
+ "f1": 0.5289646853571585,
290
+ "f1_stderr": 0.027215870565783958,
291
+ "main_score": 0.5464357767316745
292
+ },
293
+ "sw": {
294
+ "accuracy": 0.4204438466711499,
295
+ "accuracy_stderr": 0.02543427134780407,
296
+ "f1": 0.39698226248309787,
297
+ "f1_stderr": 0.02495774581205971,
298
+ "main_score": 0.4204438466711499
299
+ },
300
+ "ta": {
301
+ "accuracy": 0.3672158708809684,
302
+ "accuracy_stderr": 0.024620778778865062,
303
+ "f1": 0.35611628288260666,
304
+ "f1_stderr": 0.02508451846783393,
305
+ "main_score": 0.3672158708809684
306
+ },
307
+ "te": {
308
+ "accuracy": 0.4208137188971082,
309
+ "accuracy_stderr": 0.02723531925936748,
310
+ "f1": 0.39746768684058836,
311
+ "f1_stderr": 0.029476453506871073,
312
+ "main_score": 0.4208137188971082
313
+ },
314
+ "th": {
315
+ "accuracy": 0.521486213853396,
316
+ "accuracy_stderr": 0.02513073701134646,
317
+ "f1": 0.5038882871449852,
318
+ "f1_stderr": 0.025901347057158762,
319
+ "main_score": 0.521486213853396
320
+ },
321
+ "tl": {
322
+ "accuracy": 0.3734028244788164,
323
+ "accuracy_stderr": 0.036431060397290715,
324
+ "f1": 0.3428567103143635,
325
+ "f1_stderr": 0.03562323164275494,
326
+ "main_score": 0.3734028244788164
327
+ },
328
+ "tr": {
329
+ "accuracy": 0.5255548083389374,
330
+ "accuracy_stderr": 0.030376754244448495,
331
+ "f1": 0.5139704021609626,
332
+ "f1_stderr": 0.02797264681517368,
333
+ "main_score": 0.5255548083389374
334
+ },
335
+ "ur": {
336
+ "accuracy": 0.32599193006052457,
337
+ "accuracy_stderr": 0.03133503581066815,
338
+ "f1": 0.32576383434482714,
339
+ "f1_stderr": 0.02553427963140292,
340
+ "main_score": 0.32599193006052457
341
+ },
342
+ "vi": {
343
+ "accuracy": 0.5097175521183592,
344
+ "accuracy_stderr": 0.01667763764284436,
345
+ "f1": 0.49826499437648436,
346
+ "f1_stderr": 0.019814421278178205,
347
+ "main_score": 0.5097175521183592
348
+ },
349
+ "zh-CN": {
350
+ "accuracy": 0.5021856086079354,
351
+ "accuracy_stderr": 0.01872946746711234,
352
+ "f1": 0.49069359630568155,
353
+ "f1_stderr": 0.022400099688479058,
354
+ "main_score": 0.5021856086079354
355
+ },
356
+ "zh-TW": {
357
+ "accuracy": 0.4232347007397445,
358
+ "accuracy_stderr": 0.01456541641125866,
359
+ "f1": 0.41073538173900237,
360
+ "f1_stderr": 0.015525857226939374,
361
+ "main_score": 0.4232347007397445
362
+ }
363
+ }
364
+ }
results/LASER2/MedrxivClusteringP2P.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 549.14,
4
+ "v_measure": 0.17908142247465778,
5
+ "v_measure_std": 0.01580960913642727
6
+ },
7
+ "dataset_version": null,
8
+ "mteb_version": "0.0.2"
9
+ }
results/LASER2/MedrxivClusteringS2S.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 73.93,
4
+ "v_measure": 0.1662952889451872,
5
+ "v_measure_std": 0.018792494328507908
6
+ },
7
+ "dataset_version": null,
8
+ "mteb_version": "0.0.2"
9
+ }
results/LASER2/MindSmallReranking.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "evaluation_time": 103.4,
6
+ "map": 0.24788913161151171,
7
+ "mrr": 0.25454137908041313
8
+ }
9
+ }
results/LASER2/NFCorpus.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 44.54,
4
+ "map_at_1": 0.00051,
5
+ "map_at_10": 0.00303,
6
+ "map_at_100": 0.00588,
7
+ "map_at_1000": 0.01164,
8
+ "map_at_3": 0.00166,
9
+ "map_at_5": 0.00244,
10
+ "ndcg_at_1": 0.01548,
11
+ "ndcg_at_10": 0.02439,
12
+ "ndcg_at_100": 0.03883,
13
+ "ndcg_at_1000": 0.13695,
14
+ "ndcg_at_3": 0.0188,
15
+ "ndcg_at_5": 0.02153,
16
+ "precision_at_1": 0.01858,
17
+ "precision_at_10": 0.02508,
18
+ "precision_at_100": 0.01851,
19
+ "precision_at_1000": 0.01354,
20
+ "precision_at_3": 0.02167,
21
+ "precision_at_5": 0.02415,
22
+ "recall_at_1": 0.00051,
23
+ "recall_at_10": 0.01005,
24
+ "recall_at_100": 0.06311,
25
+ "recall_at_1000": 0.40172,
26
+ "recall_at_3": 0.00342,
27
+ "recall_at_5": 0.00638
28
+ },
29
+ "dataset_version": null,
30
+ "mteb_version": "0.0.2"
31
+ }
results/LASER2/NQ.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 6367.75,
4
+ "map_at_1": 0.00261,
5
+ "map_at_10": 0.00499,
6
+ "map_at_100": 0.00563,
7
+ "map_at_1000": 0.0058,
8
+ "map_at_3": 0.0044,
9
+ "map_at_5": 0.00473,
10
+ "ndcg_at_1": 0.00319,
11
+ "ndcg_at_10": 0.00642,
12
+ "ndcg_at_100": 0.01013,
13
+ "ndcg_at_1000": 0.01617,
14
+ "ndcg_at_3": 0.00514,
15
+ "ndcg_at_5": 0.00576,
16
+ "precision_at_1": 0.00319,
17
+ "precision_at_10": 0.00116,
18
+ "precision_at_100": 0.00033,
19
+ "precision_at_1000": 9e-05,
20
+ "precision_at_3": 0.00251,
21
+ "precision_at_5": 0.00185,
22
+ "recall_at_1": 0.00261,
23
+ "recall_at_10": 0.01004,
24
+ "recall_at_100": 0.02728,
25
+ "recall_at_1000": 0.07648,
26
+ "recall_at_3": 0.00661,
27
+ "recall_at_5": 0.00806
28
+ },
29
+ "dataset_version": null,
30
+ "mteb_version": "0.0.2"
31
+ }
results/LASER2/QuoraRetrieval.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 393.84,
4
+ "map_at_1": 0.54965,
5
+ "map_at_10": 0.66201,
6
+ "map_at_100": 0.6704,
7
+ "map_at_1000": 0.6709,
8
+ "map_at_3": 0.63478,
9
+ "map_at_5": 0.65101,
10
+ "ndcg_at_1": 0.6354,
11
+ "ndcg_at_10": 0.71145,
12
+ "ndcg_at_100": 0.74023,
13
+ "ndcg_at_1000": 0.7486,
14
+ "ndcg_at_3": 0.67488,
15
+ "ndcg_at_5": 0.69306,
16
+ "precision_at_1": 0.6354,
17
+ "precision_at_10": 0.10798,
18
+ "precision_at_100": 0.01348,
19
+ "precision_at_1000": 0.00149,
20
+ "precision_at_3": 0.29257,
21
+ "precision_at_5": 0.19412,
22
+ "recall_at_1": 0.54965,
23
+ "recall_at_10": 0.80505,
24
+ "recall_at_100": 0.92322,
25
+ "recall_at_1000": 0.97766,
26
+ "recall_at_3": 0.69938,
27
+ "recall_at_5": 0.7496
28
+ },
29
+ "dataset_version": null,
30
+ "mteb_version": "0.0.2"
31
+ }
results/LASER2/RedditClustering.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 251.71,
4
+ "v_measure": 0.0996202584007419,
5
+ "v_measure_std": 0.013494785830352516
6
+ },
7
+ "dataset_version": null,
8
+ "mteb_version": "0.0.2"
9
+ }
results/LASER2/RedditClusteringP2P.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_version": null,
3
+ "mteb_version": "0.0.2",
4
+ "test": {
5
+ "evaluation_time": 1784.6,
6
+ "v_measure": 0.264185580282609,
7
+ "v_measure_std": 0.08501487346178989
8
+ }
9
+ }
results/LASER2/SCIDOCS.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "evaluation_time": 164.89,
4
+ "map_at_1": 0.003,
5
+ "map_at_10": 0.00387,
6
+ "map_at_100": 0.00459,
7
+ "map_at_1000": 0.00532,
8
+ "map_at_3": 0.00333,
9
+ "map_at_5": 0.00356,
10
+ "ndcg_at_1": 0.015,
11
+ "ndcg_at_10": 0.00777,
12
+ "ndcg_at_100": 0.01565,
13
+ "ndcg_at_1000": 0.04821,
14
+ "ndcg_at_3": 0.00787,
15
+ "ndcg_at_5": 0.00647,
16
+ "precision_at_1": 0.015,
17
+ "precision_at_10": 0.0035,
18
+ "precision_at_100": 0.00162,
19
+ "precision_at_1000": 0.00098,
20
+ "precision_at_3": 0.006,
21
+ "precision_at_5": 0.0046,
22
+ "recall_at_1": 0.003,
23
+ "recall_at_10": 0.00718,
24
+ "recall_at_100": 0.03288,
25
+ "recall_at_1000": 0.19957,
26
+ "recall_at_3": 0.00373,
27
+ "recall_at_5": 0.00473
28
+ },
29
+ "dataset_version": null,
30
+ "mteb_version": "0.0.2"
31
+ }
results/LASER2/SICK-R.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "cos_sim": {
4
+ "pearson": 0.6710717395109536,
5
+ "spearman": 0.6285761430269268
6
+ },
7
+ "euclidean": {
8
+ "pearson": 0.5991815699401573,
9
+ "spearman": 0.5964342697857706
10
+ },
11
+ "evaluation_time": 14.19,
12
+ "manhattan": {
13
+ "pearson": 0.5607429216703508,
14
+ "spearman": 0.5531149921833667
15
+ }
16
+ },
17
+ "dataset_version": null,
18
+ "mteb_version": "0.0.2"
19
+ }
results/LASER2/STS12.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "test": {
3
+ "cos_sim": {
4
+ "pearson": 0.5997074409143551,
5
+ "spearman": 0.6260375600516247
6
+ },
7
+ "euclidean": {
8
+ "pearson": 0.49354765627824515,
9
+ "spearman": 0.5909106889528086
10
+ },
11
+ "evaluation_time": 11.98,
12
+ "manhattan": {
13
+ "pearson": 0.4558810047629682,
14
+ "spearman": 0.5570403794622609
15
+ }
16
+ },
17
+ "dataset_version": null,
18
+ "mteb_version": "0.0.2"
19
+ }