diff --git "a/README.md" "b/README.md" --- "a/README.md" +++ "b/README.md" @@ -1,3 +1,5803 @@ --- license: bigscience-bloom-rail-1.0 +language: + - ak + - ar + - as + - bm + - bn + - ca + - code + - en + - es + - eu + - fon + - fr + - gu + - hi + - id + - ig + - ki + - kn + - lg + - ln + - ml + - mr + - ne + - nso + - ny + - or + - pa + - pt + - rn + - rw + - sn + - st + - sw + - ta + - te + - tn + - ts + - tum + - tw + - ur + - vi + - wo + - xh + - yo + - zh + - zhs + - zht + - zu +tags: +- mteb +model-index: +- name: udever-bloom-3b + results: + - task: + type: STS + dataset: + type: C-MTEB/AFQMC + name: MTEB AFQMC + config: default + split: validation + revision: None + metrics: + - type: cos_sim_pearson + value: 30.0892025910701 + - type: cos_sim_spearman + value: 30.549960550731782 + - type: euclidean_pearson + value: 29.68940732194022 + - type: euclidean_spearman + value: 30.254869740623715 + - type: manhattan_pearson + value: 29.693089299297732 + - type: manhattan_spearman + value: 30.21293218369479 + - task: + type: STS + dataset: + type: C-MTEB/ATEC + name: MTEB ATEC + config: default + split: test + revision: None + metrics: + - type: cos_sim_pearson + value: 36.469490571108054 + - type: cos_sim_spearman + value: 37.34843946308442 + - type: euclidean_pearson + value: 39.697664194640886 + - type: euclidean_spearman + value: 37.623976566242334 + - type: manhattan_pearson + value: 39.8389981955552 + - type: manhattan_spearman + value: 37.689111419556 + - task: + type: Classification + dataset: + type: mteb/amazon_counterfactual + name: MTEB AmazonCounterfactualClassification (en) + config: en + split: test + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + metrics: + - type: accuracy + value: 78.8955223880597 + - type: ap + value: 43.270679598956285 + - type: f1 + value: 73.10740489387823 + - task: + type: Classification + dataset: + type: mteb/amazon_polarity + name: MTEB AmazonPolarityClassification + config: default + split: test + revision: e2d317d38cd51312af73b3d32a06d1a08b442046 + metrics: + - type: accuracy + value: 87.981225 + - type: ap + value: 83.55047186016726 + - type: f1 + value: 87.95185650917034 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (en) + config: en + split: test + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + metrics: + - type: accuracy + value: 42.58 + - type: f1 + value: 42.011158109228425 + - task: + type: Retrieval + dataset: + type: arguana + name: MTEB ArguAna + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 22.688 + - type: map_at_10 + value: 38.855000000000004 + - type: map_at_100 + value: 39.859 + - type: map_at_1000 + value: 39.871 + - type: map_at_3 + value: 33.428000000000004 + - type: map_at_5 + value: 36.571999999999996 + - type: mrr_at_1 + value: 23.044 + - type: mrr_at_10 + value: 39.022 + - type: mrr_at_100 + value: 40.019 + - type: mrr_at_1000 + value: 40.03 + - type: mrr_at_3 + value: 33.642 + - type: mrr_at_5 + value: 36.707 + - type: ndcg_at_1 + value: 22.688 + - type: ndcg_at_10 + value: 48.33 + - type: ndcg_at_100 + value: 52.616 + - type: ndcg_at_1000 + value: 52.891999999999996 + - type: ndcg_at_3 + value: 37.104 + - type: ndcg_at_5 + value: 42.764 + - type: precision_at_1 + value: 22.688 + - type: precision_at_10 + value: 7.881 + - type: precision_at_100 + value: 0.975 + - type: precision_at_1000 + value: 0.1 + - type: precision_at_3 + value: 15.931999999999999 + - type: precision_at_5 + value: 12.304 + - type: recall_at_1 + value: 22.688 + - type: recall_at_10 + value: 78.805 + - type: recall_at_100 + value: 97.51100000000001 + - type: recall_at_1000 + value: 99.644 + - type: recall_at_3 + value: 47.795 + - type: recall_at_5 + value: 61.522 + - task: + type: Clustering + dataset: + type: mteb/arxiv-clustering-p2p + name: MTEB ArxivClusteringP2P + config: default + split: test + revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d + metrics: + - type: v_measure + value: 45.37384003345981 + - task: + type: Clustering + dataset: + type: mteb/arxiv-clustering-s2s + name: MTEB ArxivClusteringS2S + config: default + split: test + revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 + metrics: + - type: v_measure + value: 36.52143615051018 + - task: + type: Reranking + dataset: + type: mteb/askubuntudupquestions-reranking + name: MTEB AskUbuntuDupQuestions + config: default + split: test + revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 + metrics: + - type: map + value: 59.91826882625199 + - type: mrr + value: 73.30530273051049 + - task: + type: STS + dataset: + type: mteb/biosses-sts + name: MTEB BIOSSES + config: default + split: test + revision: d3fb88f8f02e40887cd149695127462bbcf29b4a + metrics: + - type: cos_sim_pearson + value: 86.80556032491437 + - type: cos_sim_spearman + value: 84.81639043031876 + - type: euclidean_pearson + value: 84.20426417923026 + - type: euclidean_spearman + value: 83.53503593258247 + - type: manhattan_pearson + value: 84.25387997667964 + - type: manhattan_spearman + value: 83.11394200032217 + - task: + type: STS + dataset: + type: C-MTEB/BQ + name: MTEB BQ + config: default + split: test + revision: None + metrics: + - type: cos_sim_pearson + value: 47.017986848644625 + - type: cos_sim_spearman + value: 47.16708658456057 + - type: euclidean_pearson + value: 47.81098065168003 + - type: euclidean_spearman + value: 48.01014499886206 + - type: manhattan_pearson + value: 48.013333352251244 + - type: manhattan_spearman + value: 48.252964666749016 + - task: + type: BitextMining + dataset: + type: mteb/bucc-bitext-mining + name: MTEB BUCC (de-en) + config: de-en + split: test + revision: d51519689f32196a32af33b075a01d0e7c51e252 + metrics: + - type: accuracy + value: 71.78496868475992 + - type: f1 + value: 71.05715215634456 + - type: precision + value: 70.7532208520454 + - type: recall + value: 71.78496868475992 + - task: + type: BitextMining + dataset: + type: mteb/bucc-bitext-mining + name: MTEB BUCC (fr-en) + config: fr-en + split: test + revision: d51519689f32196a32af33b075a01d0e7c51e252 + metrics: + - type: accuracy + value: 98.34910851860005 + - type: f1 + value: 98.16751045564604 + - type: precision + value: 98.07762858610317 + - type: recall + value: 98.34910851860005 + - task: + type: BitextMining + dataset: + type: mteb/bucc-bitext-mining + name: MTEB BUCC (ru-en) + config: ru-en + split: test + revision: d51519689f32196a32af33b075a01d0e7c51e252 + metrics: + - type: accuracy + value: 59.965361967440245 + - type: f1 + value: 58.44898687503467 + - type: precision + value: 57.83301194437321 + - type: recall + value: 59.965361967440245 + - task: + type: BitextMining + dataset: + type: mteb/bucc-bitext-mining + name: MTEB BUCC (zh-en) + config: zh-en + split: test + revision: d51519689f32196a32af33b075a01d0e7c51e252 + metrics: + - type: accuracy + value: 98.63085834649816 + - type: f1 + value: 98.59575215025451 + - type: precision + value: 98.5781990521327 + - type: recall + value: 98.63085834649816 + - task: + type: Classification + dataset: + type: mteb/banking77 + name: MTEB Banking77Classification + config: default + split: test + revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 + metrics: + - type: accuracy + value: 84.15584415584416 + - type: f1 + value: 84.1389435939967 + - task: + type: Clustering + dataset: + type: mteb/biorxiv-clustering-p2p + name: MTEB BiorxivClusteringP2P + config: default + split: test + revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 + metrics: + - type: v_measure + value: 36.52184607783334 + - task: + type: Clustering + dataset: + type: mteb/biorxiv-clustering-s2s + name: MTEB BiorxivClusteringS2S + config: default + split: test + revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 + metrics: + - type: v_measure + value: 31.976191171733653 + - task: + type: Clustering + dataset: + type: C-MTEB/CLSClusteringP2P + name: MTEB CLSClusteringP2P + config: default + split: test + revision: None + metrics: + - type: v_measure + value: 36.733774048381484 + - task: + type: Clustering + dataset: + type: C-MTEB/CLSClusteringS2S + name: MTEB CLSClusteringS2S + config: default + split: test + revision: None + metrics: + - type: v_measure + value: 36.451952183379056 + - task: + type: Reranking + dataset: + type: C-MTEB/CMedQAv1-reranking + name: MTEB CMedQAv1 + config: default + split: test + revision: None + metrics: + - type: map + value: 68.9131612041328 + - type: mrr + value: 73.47626984126985 + - task: + type: Reranking + dataset: + type: C-MTEB/CMedQAv2-reranking + name: MTEB CMedQAv2 + config: default + split: test + revision: None + metrics: + - type: map + value: 69.42233467142258 + - type: mrr + value: 74.22722222222221 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackAndroidRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 32.943 + - type: map_at_10 + value: 42.796 + - type: map_at_100 + value: 44.141999999999996 + - type: map_at_1000 + value: 44.277 + - type: map_at_3 + value: 39.201 + - type: map_at_5 + value: 41.262 + - type: mrr_at_1 + value: 41.488 + - type: mrr_at_10 + value: 49.214999999999996 + - type: mrr_at_100 + value: 50.02799999999999 + - type: mrr_at_1000 + value: 50.075 + - type: mrr_at_3 + value: 46.733000000000004 + - type: mrr_at_5 + value: 48.171 + - type: ndcg_at_1 + value: 41.488 + - type: ndcg_at_10 + value: 48.619 + - type: ndcg_at_100 + value: 53.868 + - type: ndcg_at_1000 + value: 56.027 + - type: ndcg_at_3 + value: 43.765 + - type: ndcg_at_5 + value: 45.974 + - type: precision_at_1 + value: 41.488 + - type: precision_at_10 + value: 9.07 + - type: precision_at_100 + value: 1.4460000000000002 + - type: precision_at_1000 + value: 0.19499999999999998 + - type: precision_at_3 + value: 20.649 + - type: precision_at_5 + value: 14.878 + - type: recall_at_1 + value: 32.943 + - type: recall_at_10 + value: 59.217 + - type: recall_at_100 + value: 81.337 + - type: recall_at_1000 + value: 95.185 + - type: recall_at_3 + value: 44.377 + - type: recall_at_5 + value: 51.088 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackEnglishRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 26.412999999999997 + - type: map_at_10 + value: 34.766999999999996 + - type: map_at_100 + value: 35.774 + - type: map_at_1000 + value: 35.894999999999996 + - type: map_at_3 + value: 31.935000000000002 + - type: map_at_5 + value: 33.661 + - type: mrr_at_1 + value: 33.248 + - type: mrr_at_10 + value: 40.274 + - type: mrr_at_100 + value: 40.92 + - type: mrr_at_1000 + value: 40.977000000000004 + - type: mrr_at_3 + value: 38.004 + - type: mrr_at_5 + value: 39.425 + - type: ndcg_at_1 + value: 33.248 + - type: ndcg_at_10 + value: 39.828 + - type: ndcg_at_100 + value: 43.863 + - type: ndcg_at_1000 + value: 46.228 + - type: ndcg_at_3 + value: 35.643 + - type: ndcg_at_5 + value: 37.851 + - type: precision_at_1 + value: 33.248 + - type: precision_at_10 + value: 7.4079999999999995 + - type: precision_at_100 + value: 1.162 + - type: precision_at_1000 + value: 0.168 + - type: precision_at_3 + value: 16.964000000000002 + - type: precision_at_5 + value: 12.267999999999999 + - type: recall_at_1 + value: 26.412999999999997 + - type: recall_at_10 + value: 48.93 + - type: recall_at_100 + value: 66.437 + - type: recall_at_1000 + value: 81.68900000000001 + - type: recall_at_3 + value: 36.822 + - type: recall_at_5 + value: 42.925000000000004 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackGamingRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 37.07 + - type: map_at_10 + value: 49.051 + - type: map_at_100 + value: 50.13999999999999 + - type: map_at_1000 + value: 50.2 + - type: map_at_3 + value: 46.01 + - type: map_at_5 + value: 47.711 + - type: mrr_at_1 + value: 42.32 + - type: mrr_at_10 + value: 52.32 + - type: mrr_at_100 + value: 53.068000000000005 + - type: mrr_at_1000 + value: 53.09700000000001 + - type: mrr_at_3 + value: 49.864000000000004 + - type: mrr_at_5 + value: 51.312000000000005 + - type: ndcg_at_1 + value: 42.32 + - type: ndcg_at_10 + value: 54.727000000000004 + - type: ndcg_at_100 + value: 59.153 + - type: ndcg_at_1000 + value: 60.373 + - type: ndcg_at_3 + value: 49.478 + - type: ndcg_at_5 + value: 51.998999999999995 + - type: precision_at_1 + value: 42.32 + - type: precision_at_10 + value: 8.802999999999999 + - type: precision_at_100 + value: 1.196 + - type: precision_at_1000 + value: 0.135 + - type: precision_at_3 + value: 22.006 + - type: precision_at_5 + value: 15.072 + - type: recall_at_1 + value: 37.07 + - type: recall_at_10 + value: 68.221 + - type: recall_at_100 + value: 87.22999999999999 + - type: recall_at_1000 + value: 95.929 + - type: recall_at_3 + value: 54.321 + - type: recall_at_5 + value: 60.358000000000004 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackGisRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 23.055 + - type: map_at_10 + value: 31.163999999999998 + - type: map_at_100 + value: 32.213 + - type: map_at_1000 + value: 32.303 + - type: map_at_3 + value: 28.610000000000003 + - type: map_at_5 + value: 30.091 + - type: mrr_at_1 + value: 24.972 + - type: mrr_at_10 + value: 32.981 + - type: mrr_at_100 + value: 33.948 + - type: mrr_at_1000 + value: 34.015 + - type: mrr_at_3 + value: 30.546 + - type: mrr_at_5 + value: 31.959 + - type: ndcg_at_1 + value: 24.972 + - type: ndcg_at_10 + value: 35.806 + - type: ndcg_at_100 + value: 40.991 + - type: ndcg_at_1000 + value: 43.296 + - type: ndcg_at_3 + value: 30.849 + - type: ndcg_at_5 + value: 33.334 + - type: precision_at_1 + value: 24.972 + - type: precision_at_10 + value: 5.571000000000001 + - type: precision_at_100 + value: 0.853 + - type: precision_at_1000 + value: 0.109 + - type: precision_at_3 + value: 12.956999999999999 + - type: precision_at_5 + value: 9.333 + - type: recall_at_1 + value: 23.055 + - type: recall_at_10 + value: 48.301 + - type: recall_at_100 + value: 72.051 + - type: recall_at_1000 + value: 89.408 + - type: recall_at_3 + value: 35.315000000000005 + - type: recall_at_5 + value: 41.031 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackMathematicaRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 14.782 + - type: map_at_10 + value: 21.94 + - type: map_at_100 + value: 23.172 + - type: map_at_1000 + value: 23.302999999999997 + - type: map_at_3 + value: 19.911 + - type: map_at_5 + value: 20.998 + - type: mrr_at_1 + value: 18.407999999999998 + - type: mrr_at_10 + value: 25.936999999999998 + - type: mrr_at_100 + value: 27.035999999999998 + - type: mrr_at_1000 + value: 27.118 + - type: mrr_at_3 + value: 23.983999999999998 + - type: mrr_at_5 + value: 25.141000000000002 + - type: ndcg_at_1 + value: 18.407999999999998 + - type: ndcg_at_10 + value: 26.387 + - type: ndcg_at_100 + value: 32.606 + - type: ndcg_at_1000 + value: 35.744 + - type: ndcg_at_3 + value: 22.686999999999998 + - type: ndcg_at_5 + value: 24.375 + - type: precision_at_1 + value: 18.407999999999998 + - type: precision_at_10 + value: 4.801 + - type: precision_at_100 + value: 0.9299999999999999 + - type: precision_at_1000 + value: 0.134 + - type: precision_at_3 + value: 10.945 + - type: precision_at_5 + value: 7.811 + - type: recall_at_1 + value: 14.782 + - type: recall_at_10 + value: 36.018 + - type: recall_at_100 + value: 63.552 + - type: recall_at_1000 + value: 85.857 + - type: recall_at_3 + value: 25.898 + - type: recall_at_5 + value: 30.081999999999997 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackPhysicsRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 26.369 + - type: map_at_10 + value: 37.704 + - type: map_at_100 + value: 39.018 + - type: map_at_1000 + value: 39.134 + - type: map_at_3 + value: 34.243 + - type: map_at_5 + value: 36.083 + - type: mrr_at_1 + value: 32.916000000000004 + - type: mrr_at_10 + value: 43.488 + - type: mrr_at_100 + value: 44.29 + - type: mrr_at_1000 + value: 44.336999999999996 + - type: mrr_at_3 + value: 40.696 + - type: mrr_at_5 + value: 42.289 + - type: ndcg_at_1 + value: 32.916000000000004 + - type: ndcg_at_10 + value: 44.362 + - type: ndcg_at_100 + value: 49.730999999999995 + - type: ndcg_at_1000 + value: 51.857 + - type: ndcg_at_3 + value: 38.683 + - type: ndcg_at_5 + value: 41.249 + - type: precision_at_1 + value: 32.916000000000004 + - type: precision_at_10 + value: 8.412 + - type: precision_at_100 + value: 1.2970000000000002 + - type: precision_at_1000 + value: 0.166 + - type: precision_at_3 + value: 18.895999999999997 + - type: precision_at_5 + value: 13.550999999999998 + - type: recall_at_1 + value: 26.369 + - type: recall_at_10 + value: 58.464000000000006 + - type: recall_at_100 + value: 80.884 + - type: recall_at_1000 + value: 94.676 + - type: recall_at_3 + value: 42.485 + - type: recall_at_5 + value: 49.262 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackProgrammersRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 23.896 + - type: map_at_10 + value: 33.384 + - type: map_at_100 + value: 34.683 + - type: map_at_1000 + value: 34.807 + - type: map_at_3 + value: 30.724 + - type: map_at_5 + value: 32.339 + - type: mrr_at_1 + value: 29.909000000000002 + - type: mrr_at_10 + value: 38.395 + - type: mrr_at_100 + value: 39.339 + - type: mrr_at_1000 + value: 39.404 + - type: mrr_at_3 + value: 36.339 + - type: mrr_at_5 + value: 37.618 + - type: ndcg_at_1 + value: 29.909000000000002 + - type: ndcg_at_10 + value: 38.688 + - type: ndcg_at_100 + value: 44.399 + - type: ndcg_at_1000 + value: 46.942 + - type: ndcg_at_3 + value: 34.548 + - type: ndcg_at_5 + value: 36.605 + - type: precision_at_1 + value: 29.909000000000002 + - type: precision_at_10 + value: 7.066 + - type: precision_at_100 + value: 1.174 + - type: precision_at_1000 + value: 0.155 + - type: precision_at_3 + value: 16.819 + - type: precision_at_5 + value: 11.872 + - type: recall_at_1 + value: 23.896 + - type: recall_at_10 + value: 49.531 + - type: recall_at_100 + value: 73.977 + - type: recall_at_1000 + value: 91.393 + - type: recall_at_3 + value: 37.53 + - type: recall_at_5 + value: 43.373 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 24.153166666666667 + - type: map_at_10 + value: 32.7705 + - type: map_at_100 + value: 33.93133333333334 + - type: map_at_1000 + value: 34.052499999999995 + - type: map_at_3 + value: 30.158500000000004 + - type: map_at_5 + value: 31.595916666666664 + - type: mrr_at_1 + value: 28.87725 + - type: mrr_at_10 + value: 36.86358333333333 + - type: mrr_at_100 + value: 37.74550000000001 + - type: mrr_at_1000 + value: 37.80916666666666 + - type: mrr_at_3 + value: 34.634499999999996 + - type: mrr_at_5 + value: 35.926750000000006 + - type: ndcg_at_1 + value: 28.87725 + - type: ndcg_at_10 + value: 37.82341666666667 + - type: ndcg_at_100 + value: 42.98408333333333 + - type: ndcg_at_1000 + value: 45.44883333333333 + - type: ndcg_at_3 + value: 33.41875000000001 + - type: ndcg_at_5 + value: 35.45158333333333 + - type: precision_at_1 + value: 28.87725 + - type: precision_at_10 + value: 6.638249999999999 + - type: precision_at_100 + value: 1.0863333333333334 + - type: precision_at_1000 + value: 0.14858333333333335 + - type: precision_at_3 + value: 15.481 + - type: precision_at_5 + value: 10.953916666666668 + - type: recall_at_1 + value: 24.153166666666667 + - type: recall_at_10 + value: 48.796499999999995 + - type: recall_at_100 + value: 71.53716666666666 + - type: recall_at_1000 + value: 88.72158333333333 + - type: recall_at_3 + value: 36.419583333333335 + - type: recall_at_5 + value: 41.735833333333325 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackStatsRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 21.523 + - type: map_at_10 + value: 28.915000000000003 + - type: map_at_100 + value: 29.808 + - type: map_at_1000 + value: 29.910999999999998 + - type: map_at_3 + value: 26.863999999999997 + - type: map_at_5 + value: 27.801 + - type: mrr_at_1 + value: 24.387 + - type: mrr_at_10 + value: 31.703 + - type: mrr_at_100 + value: 32.481 + - type: mrr_at_1000 + value: 32.559 + - type: mrr_at_3 + value: 29.805999999999997 + - type: mrr_at_5 + value: 30.688 + - type: ndcg_at_1 + value: 24.387 + - type: ndcg_at_10 + value: 33.272 + - type: ndcg_at_100 + value: 37.79 + - type: ndcg_at_1000 + value: 40.428 + - type: ndcg_at_3 + value: 29.409000000000002 + - type: ndcg_at_5 + value: 30.813000000000002 + - type: precision_at_1 + value: 24.387 + - type: precision_at_10 + value: 5.337 + - type: precision_at_100 + value: 0.8240000000000001 + - type: precision_at_1000 + value: 0.11299999999999999 + - type: precision_at_3 + value: 13.19 + - type: precision_at_5 + value: 8.926 + - type: recall_at_1 + value: 21.523 + - type: recall_at_10 + value: 44.054 + - type: recall_at_100 + value: 64.80900000000001 + - type: recall_at_1000 + value: 84.265 + - type: recall_at_3 + value: 33.019999999999996 + - type: recall_at_5 + value: 36.561 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackTexRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 15.461 + - type: map_at_10 + value: 21.802 + - type: map_at_100 + value: 22.825 + - type: map_at_1000 + value: 22.95 + - type: map_at_3 + value: 19.79 + - type: map_at_5 + value: 20.828 + - type: mrr_at_1 + value: 18.789 + - type: mrr_at_10 + value: 25.373 + - type: mrr_at_100 + value: 26.269 + - type: mrr_at_1000 + value: 26.355 + - type: mrr_at_3 + value: 23.394000000000002 + - type: mrr_at_5 + value: 24.451999999999998 + - type: ndcg_at_1 + value: 18.789 + - type: ndcg_at_10 + value: 25.948 + - type: ndcg_at_100 + value: 30.926 + - type: ndcg_at_1000 + value: 33.938 + - type: ndcg_at_3 + value: 22.281000000000002 + - type: ndcg_at_5 + value: 23.818 + - type: precision_at_1 + value: 18.789 + - type: precision_at_10 + value: 4.766 + - type: precision_at_100 + value: 0.848 + - type: precision_at_1000 + value: 0.127 + - type: precision_at_3 + value: 10.633 + - type: precision_at_5 + value: 7.6259999999999994 + - type: recall_at_1 + value: 15.461 + - type: recall_at_10 + value: 34.967999999999996 + - type: recall_at_100 + value: 57.25900000000001 + - type: recall_at_1000 + value: 78.738 + - type: recall_at_3 + value: 24.495 + - type: recall_at_5 + value: 28.510999999999996 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackUnixRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 25.165 + - type: map_at_10 + value: 32.66 + - type: map_at_100 + value: 33.842 + - type: map_at_1000 + value: 33.952 + - type: map_at_3 + value: 30.503999999999998 + - type: map_at_5 + value: 31.546000000000003 + - type: mrr_at_1 + value: 29.851 + - type: mrr_at_10 + value: 37.112 + - type: mrr_at_100 + value: 38.057 + - type: mrr_at_1000 + value: 38.119 + - type: mrr_at_3 + value: 35.106 + - type: mrr_at_5 + value: 36.22 + - type: ndcg_at_1 + value: 29.851 + - type: ndcg_at_10 + value: 37.395 + - type: ndcg_at_100 + value: 42.906 + - type: ndcg_at_1000 + value: 45.427 + - type: ndcg_at_3 + value: 33.465 + - type: ndcg_at_5 + value: 35.02 + - type: precision_at_1 + value: 29.851 + - type: precision_at_10 + value: 6.166 + - type: precision_at_100 + value: 1.005 + - type: precision_at_1000 + value: 0.132 + - type: precision_at_3 + value: 15.235999999999999 + - type: precision_at_5 + value: 10.354 + - type: recall_at_1 + value: 25.165 + - type: recall_at_10 + value: 47.439 + - type: recall_at_100 + value: 71.56099999999999 + - type: recall_at_1000 + value: 89.435 + - type: recall_at_3 + value: 36.275 + - type: recall_at_5 + value: 40.435 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackWebmastersRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 25.589000000000002 + - type: map_at_10 + value: 33.729 + - type: map_at_100 + value: 35.306 + - type: map_at_1000 + value: 35.552 + - type: map_at_3 + value: 30.988 + - type: map_at_5 + value: 32.406 + - type: mrr_at_1 + value: 30.830000000000002 + - type: mrr_at_10 + value: 38.446999999999996 + - type: mrr_at_100 + value: 39.478 + - type: mrr_at_1000 + value: 39.544000000000004 + - type: mrr_at_3 + value: 36.034 + - type: mrr_at_5 + value: 37.546 + - type: ndcg_at_1 + value: 30.830000000000002 + - type: ndcg_at_10 + value: 39.22 + - type: ndcg_at_100 + value: 45.004 + - type: ndcg_at_1000 + value: 47.837 + - type: ndcg_at_3 + value: 34.811 + - type: ndcg_at_5 + value: 36.831 + - type: precision_at_1 + value: 30.830000000000002 + - type: precision_at_10 + value: 7.489999999999999 + - type: precision_at_100 + value: 1.534 + - type: precision_at_1000 + value: 0.241 + - type: precision_at_3 + value: 16.14 + - type: precision_at_5 + value: 11.66 + - type: recall_at_1 + value: 25.589000000000002 + - type: recall_at_10 + value: 49.238 + - type: recall_at_100 + value: 74.893 + - type: recall_at_1000 + value: 92.902 + - type: recall_at_3 + value: 36.75 + - type: recall_at_5 + value: 42.256 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackWordpressRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 17.572 + - type: map_at_10 + value: 25.334 + - type: map_at_100 + value: 26.253 + - type: map_at_1000 + value: 26.346000000000004 + - type: map_at_3 + value: 23.122 + - type: map_at_5 + value: 24.425 + - type: mrr_at_1 + value: 19.409000000000002 + - type: mrr_at_10 + value: 27.118 + - type: mrr_at_100 + value: 28.032 + - type: mrr_at_1000 + value: 28.110000000000003 + - type: mrr_at_3 + value: 25.108000000000004 + - type: mrr_at_5 + value: 26.3 + - type: ndcg_at_1 + value: 19.409000000000002 + - type: ndcg_at_10 + value: 29.629 + - type: ndcg_at_100 + value: 34.572 + - type: ndcg_at_1000 + value: 37.289 + - type: ndcg_at_3 + value: 25.406000000000002 + - type: ndcg_at_5 + value: 27.55 + - type: precision_at_1 + value: 19.409000000000002 + - type: precision_at_10 + value: 4.769 + - type: precision_at_100 + value: 0.767 + - type: precision_at_1000 + value: 0.108 + - type: precision_at_3 + value: 11.337 + - type: precision_at_5 + value: 8.096 + - type: recall_at_1 + value: 17.572 + - type: recall_at_10 + value: 41.177 + - type: recall_at_100 + value: 64.456 + - type: recall_at_1000 + value: 85.182 + - type: recall_at_3 + value: 29.747 + - type: recall_at_5 + value: 34.948 + - task: + type: Retrieval + dataset: + type: climate-fever + name: MTEB ClimateFEVER + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 9.264 + - type: map_at_10 + value: 16.09 + - type: map_at_100 + value: 17.717 + - type: map_at_1000 + value: 17.903 + - type: map_at_3 + value: 13.422 + - type: map_at_5 + value: 14.78 + - type: mrr_at_1 + value: 20.326 + - type: mrr_at_10 + value: 31.274 + - type: mrr_at_100 + value: 32.312999999999995 + - type: mrr_at_1000 + value: 32.365 + - type: mrr_at_3 + value: 27.959 + - type: mrr_at_5 + value: 29.877 + - type: ndcg_at_1 + value: 20.326 + - type: ndcg_at_10 + value: 23.358 + - type: ndcg_at_100 + value: 30.36 + - type: ndcg_at_1000 + value: 33.883 + - type: ndcg_at_3 + value: 18.704 + - type: ndcg_at_5 + value: 20.374 + - type: precision_at_1 + value: 20.326 + - type: precision_at_10 + value: 7.303 + - type: precision_at_100 + value: 1.488 + - type: precision_at_1000 + value: 0.214 + - type: precision_at_3 + value: 13.811000000000002 + - type: precision_at_5 + value: 10.84 + - type: recall_at_1 + value: 9.264 + - type: recall_at_10 + value: 29.177999999999997 + - type: recall_at_100 + value: 53.61900000000001 + - type: recall_at_1000 + value: 73.48400000000001 + - type: recall_at_3 + value: 17.738 + - type: recall_at_5 + value: 22.279 + - task: + type: Retrieval + dataset: + type: C-MTEB/CmedqaRetrieval + name: MTEB CmedqaRetrieval + config: default + split: dev + revision: None + metrics: + - type: map_at_1 + value: 14.494000000000002 + - type: map_at_10 + value: 21.37 + - type: map_at_100 + value: 22.741 + - type: map_at_1000 + value: 22.911 + - type: map_at_3 + value: 18.929000000000002 + - type: map_at_5 + value: 20.244 + - type: mrr_at_1 + value: 23.105999999999998 + - type: mrr_at_10 + value: 29.137999999999998 + - type: mrr_at_100 + value: 30.064 + - type: mrr_at_1000 + value: 30.152 + - type: mrr_at_3 + value: 27.119 + - type: mrr_at_5 + value: 28.301 + - type: ndcg_at_1 + value: 23.105999999999998 + - type: ndcg_at_10 + value: 26.182 + - type: ndcg_at_100 + value: 32.396 + - type: ndcg_at_1000 + value: 36.177 + - type: ndcg_at_3 + value: 22.708000000000002 + - type: ndcg_at_5 + value: 24.137 + - type: precision_at_1 + value: 23.105999999999998 + - type: precision_at_10 + value: 6.0040000000000004 + - type: precision_at_100 + value: 1.119 + - type: precision_at_1000 + value: 0.161 + - type: precision_at_3 + value: 13.028 + - type: precision_at_5 + value: 9.557 + - type: recall_at_1 + value: 14.494000000000002 + - type: recall_at_10 + value: 32.910000000000004 + - type: recall_at_100 + value: 59.202999999999996 + - type: recall_at_1000 + value: 85.61 + - type: recall_at_3 + value: 22.397 + - type: recall_at_5 + value: 26.900000000000002 + - task: + type: PairClassification + dataset: + type: C-MTEB/CMNLI + name: MTEB Cmnli + config: default + split: validation + revision: None + metrics: + - type: cos_sim_accuracy + value: 74.91280817799158 + - type: cos_sim_ap + value: 83.32013347926805 + - type: cos_sim_f1 + value: 76.57387580299788 + - type: cos_sim_precision + value: 70.63006122852063 + - type: cos_sim_recall + value: 83.61000701426234 + - type: dot_accuracy + value: 70.5832832230908 + - type: dot_ap + value: 75.9647326130666 + - type: dot_f1 + value: 73.65528072241852 + - type: dot_precision + value: 63.47487734731856 + - type: dot_recall + value: 87.72504091653029 + - type: euclidean_accuracy + value: 74.51593505712569 + - type: euclidean_ap + value: 83.04382773676555 + - type: euclidean_f1 + value: 75.7739770513098 + - type: euclidean_precision + value: 70.5502922797823 + - type: euclidean_recall + value: 81.83306055646482 + - type: manhattan_accuracy + value: 74.73241130487071 + - type: manhattan_ap + value: 83.32768114935021 + - type: manhattan_f1 + value: 76.09116319071167 + - type: manhattan_precision + value: 70.42786069651741 + - type: manhattan_recall + value: 82.74491465980827 + - type: max_accuracy + value: 74.91280817799158 + - type: max_ap + value: 83.32768114935021 + - type: max_f1 + value: 76.57387580299788 + - task: + type: Retrieval + dataset: + type: C-MTEB/CovidRetrieval + name: MTEB CovidRetrieval + config: default + split: dev + revision: None + metrics: + - type: map_at_1 + value: 55.032000000000004 + - type: map_at_10 + value: 63.517 + - type: map_at_100 + value: 64.159 + - type: map_at_1000 + value: 64.17699999999999 + - type: map_at_3 + value: 61.503 + - type: map_at_5 + value: 62.741 + - type: mrr_at_1 + value: 55.111 + - type: mrr_at_10 + value: 63.50900000000001 + - type: mrr_at_100 + value: 64.13499999999999 + - type: mrr_at_1000 + value: 64.153 + - type: mrr_at_3 + value: 61.521 + - type: mrr_at_5 + value: 62.759 + - type: ndcg_at_1 + value: 55.216 + - type: ndcg_at_10 + value: 67.569 + - type: ndcg_at_100 + value: 70.71 + - type: ndcg_at_1000 + value: 71.211 + - type: ndcg_at_3 + value: 63.543000000000006 + - type: ndcg_at_5 + value: 65.718 + - type: precision_at_1 + value: 55.216 + - type: precision_at_10 + value: 8.093 + - type: precision_at_100 + value: 0.96 + - type: precision_at_1000 + value: 0.1 + - type: precision_at_3 + value: 23.253 + - type: precision_at_5 + value: 15.026 + - type: recall_at_1 + value: 55.032000000000004 + - type: recall_at_10 + value: 80.163 + - type: recall_at_100 + value: 94.94200000000001 + - type: recall_at_1000 + value: 98.946 + - type: recall_at_3 + value: 69.231 + - type: recall_at_5 + value: 74.49900000000001 + - task: + type: Retrieval + dataset: + type: dbpedia-entity + name: MTEB DBPedia + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 8.391 + - type: map_at_10 + value: 16.381999999999998 + - type: map_at_100 + value: 21.262 + - type: map_at_1000 + value: 22.461000000000002 + - type: map_at_3 + value: 12.471 + - type: map_at_5 + value: 14.016 + - type: mrr_at_1 + value: 62.25000000000001 + - type: mrr_at_10 + value: 69.64099999999999 + - type: mrr_at_100 + value: 70.114 + - type: mrr_at_1000 + value: 70.128 + - type: mrr_at_3 + value: 67.958 + - type: mrr_at_5 + value: 68.996 + - type: ndcg_at_1 + value: 50.375 + - type: ndcg_at_10 + value: 34.542 + - type: ndcg_at_100 + value: 37.265 + - type: ndcg_at_1000 + value: 44.324000000000005 + - type: ndcg_at_3 + value: 40.113 + - type: ndcg_at_5 + value: 37.177 + - type: precision_at_1 + value: 62.25000000000001 + - type: precision_at_10 + value: 26.05 + - type: precision_at_100 + value: 7.632999999999999 + - type: precision_at_1000 + value: 1.6209999999999998 + - type: precision_at_3 + value: 42.5 + - type: precision_at_5 + value: 35.199999999999996 + - type: recall_at_1 + value: 8.391 + - type: recall_at_10 + value: 21.099 + - type: recall_at_100 + value: 40.886 + - type: recall_at_1000 + value: 63.805 + - type: recall_at_3 + value: 13.766 + - type: recall_at_5 + value: 16.128 + - task: + type: Retrieval + dataset: + type: C-MTEB/DuRetrieval + name: MTEB DuRetrieval + config: default + split: dev + revision: None + metrics: + - type: map_at_1 + value: 21.933 + - type: map_at_10 + value: 65.739 + - type: map_at_100 + value: 69.245 + - type: map_at_1000 + value: 69.33399999999999 + - type: map_at_3 + value: 44.874 + - type: map_at_5 + value: 56.242999999999995 + - type: mrr_at_1 + value: 78.95 + - type: mrr_at_10 + value: 85.37700000000001 + - type: mrr_at_100 + value: 85.474 + - type: mrr_at_1000 + value: 85.481 + - type: mrr_at_3 + value: 84.63300000000001 + - type: mrr_at_5 + value: 85.141 + - type: ndcg_at_1 + value: 78.95 + - type: ndcg_at_10 + value: 75.81599999999999 + - type: ndcg_at_100 + value: 80.42399999999999 + - type: ndcg_at_1000 + value: 81.357 + - type: ndcg_at_3 + value: 73.821 + - type: ndcg_at_5 + value: 72.497 + - type: precision_at_1 + value: 78.95 + - type: precision_at_10 + value: 37.285000000000004 + - type: precision_at_100 + value: 4.589 + - type: precision_at_1000 + value: 0.481 + - type: precision_at_3 + value: 66.333 + - type: precision_at_5 + value: 55.879999999999995 + - type: recall_at_1 + value: 21.933 + - type: recall_at_10 + value: 77.943 + - type: recall_at_100 + value: 92.17 + - type: recall_at_1000 + value: 96.986 + - type: recall_at_3 + value: 48.079 + - type: recall_at_5 + value: 62.65500000000001 + - task: + type: Retrieval + dataset: + type: C-MTEB/EcomRetrieval + name: MTEB EcomRetrieval + config: default + split: dev + revision: None + metrics: + - type: map_at_1 + value: 38.2 + - type: map_at_10 + value: 46.785 + - type: map_at_100 + value: 47.635 + - type: map_at_1000 + value: 47.675 + - type: map_at_3 + value: 44.583 + - type: map_at_5 + value: 45.848 + - type: mrr_at_1 + value: 38.2 + - type: mrr_at_10 + value: 46.785 + - type: mrr_at_100 + value: 47.635 + - type: mrr_at_1000 + value: 47.675 + - type: mrr_at_3 + value: 44.583 + - type: mrr_at_5 + value: 45.848 + - type: ndcg_at_1 + value: 38.2 + - type: ndcg_at_10 + value: 51.282000000000004 + - type: ndcg_at_100 + value: 55.608000000000004 + - type: ndcg_at_1000 + value: 56.726 + - type: ndcg_at_3 + value: 46.763 + - type: ndcg_at_5 + value: 49.035000000000004 + - type: precision_at_1 + value: 38.2 + - type: precision_at_10 + value: 6.550000000000001 + - type: precision_at_100 + value: 0.8619999999999999 + - type: precision_at_1000 + value: 0.095 + - type: precision_at_3 + value: 17.7 + - type: precision_at_5 + value: 11.72 + - type: recall_at_1 + value: 38.2 + - type: recall_at_10 + value: 65.5 + - type: recall_at_100 + value: 86.2 + - type: recall_at_1000 + value: 95.1 + - type: recall_at_3 + value: 53.1 + - type: recall_at_5 + value: 58.599999999999994 + - task: + type: Classification + dataset: + type: mteb/emotion + name: MTEB EmotionClassification + config: default + split: test + revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 + metrics: + - type: accuracy + value: 47.88 + - type: f1 + value: 43.30537129784135 + - task: + type: Retrieval + dataset: + type: fever + name: MTEB FEVER + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 54.423 + - type: map_at_10 + value: 66.136 + - type: map_at_100 + value: 66.557 + - type: map_at_1000 + value: 66.57300000000001 + - type: map_at_3 + value: 64.042 + - type: map_at_5 + value: 65.366 + - type: mrr_at_1 + value: 58.745999999999995 + - type: mrr_at_10 + value: 70.456 + - type: mrr_at_100 + value: 70.801 + - type: mrr_at_1000 + value: 70.809 + - type: mrr_at_3 + value: 68.504 + - type: mrr_at_5 + value: 69.746 + - type: ndcg_at_1 + value: 58.745999999999995 + - type: ndcg_at_10 + value: 71.96000000000001 + - type: ndcg_at_100 + value: 73.83 + - type: ndcg_at_1000 + value: 74.17 + - type: ndcg_at_3 + value: 68.033 + - type: ndcg_at_5 + value: 70.22 + - type: precision_at_1 + value: 58.745999999999995 + - type: precision_at_10 + value: 9.397 + - type: precision_at_100 + value: 1.043 + - type: precision_at_1000 + value: 0.108 + - type: precision_at_3 + value: 27.208 + - type: precision_at_5 + value: 17.561 + - type: recall_at_1 + value: 54.423 + - type: recall_at_10 + value: 85.703 + - type: recall_at_100 + value: 93.989 + - type: recall_at_1000 + value: 96.35000000000001 + - type: recall_at_3 + value: 75.05 + - type: recall_at_5 + value: 80.447 + - task: + type: Retrieval + dataset: + type: fiqa + name: MTEB FiQA2018 + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 16.286 + - type: map_at_10 + value: 27.499000000000002 + - type: map_at_100 + value: 29.176999999999996 + - type: map_at_1000 + value: 29.354999999999997 + - type: map_at_3 + value: 23.684 + - type: map_at_5 + value: 25.544 + - type: mrr_at_1 + value: 32.87 + - type: mrr_at_10 + value: 41.906 + - type: mrr_at_100 + value: 42.739 + - type: mrr_at_1000 + value: 42.78 + - type: mrr_at_3 + value: 38.992 + - type: mrr_at_5 + value: 40.535 + - type: ndcg_at_1 + value: 32.87 + - type: ndcg_at_10 + value: 35.124 + - type: ndcg_at_100 + value: 41.638 + - type: ndcg_at_1000 + value: 44.869 + - type: ndcg_at_3 + value: 30.975 + - type: ndcg_at_5 + value: 32.112 + - type: precision_at_1 + value: 32.87 + - type: precision_at_10 + value: 10.062 + - type: precision_at_100 + value: 1.653 + - type: precision_at_1000 + value: 0.22599999999999998 + - type: precision_at_3 + value: 20.833 + - type: precision_at_5 + value: 15.340000000000002 + - type: recall_at_1 + value: 16.286 + - type: recall_at_10 + value: 42.734 + - type: recall_at_100 + value: 67.582 + - type: recall_at_1000 + value: 86.735 + - type: recall_at_3 + value: 28.438000000000002 + - type: recall_at_5 + value: 33.944 + - task: + type: Retrieval + dataset: + type: hotpotqa + name: MTEB HotpotQA + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 33.606 + - type: map_at_10 + value: 46.085 + - type: map_at_100 + value: 46.796 + - type: map_at_1000 + value: 46.866 + - type: map_at_3 + value: 43.614000000000004 + - type: map_at_5 + value: 45.094 + - type: mrr_at_1 + value: 67.211 + - type: mrr_at_10 + value: 73.447 + - type: mrr_at_100 + value: 73.734 + - type: mrr_at_1000 + value: 73.752 + - type: mrr_at_3 + value: 72.233 + - type: mrr_at_5 + value: 72.982 + - type: ndcg_at_1 + value: 67.211 + - type: ndcg_at_10 + value: 55.125 + - type: ndcg_at_100 + value: 57.904999999999994 + - type: ndcg_at_1000 + value: 59.40800000000001 + - type: ndcg_at_3 + value: 51.283 + - type: ndcg_at_5 + value: 53.32599999999999 + - type: precision_at_1 + value: 67.211 + - type: precision_at_10 + value: 11.198 + - type: precision_at_100 + value: 1.34 + - type: precision_at_1000 + value: 0.154 + - type: precision_at_3 + value: 31.631999999999998 + - type: precision_at_5 + value: 20.591 + - type: recall_at_1 + value: 33.606 + - type: recall_at_10 + value: 55.989 + - type: recall_at_100 + value: 67.01599999999999 + - type: recall_at_1000 + value: 77.076 + - type: recall_at_3 + value: 47.448 + - type: recall_at_5 + value: 51.479 + - task: + type: Classification + dataset: + type: C-MTEB/IFlyTek-classification + name: MTEB IFlyTek + config: default + split: validation + revision: None + metrics: + - type: accuracy + value: 45.02500961908426 + - type: f1 + value: 36.80024928040335 + - task: + type: Classification + dataset: + type: mteb/imdb + name: MTEB ImdbClassification + config: default + split: test + revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 + metrics: + - type: accuracy + value: 77.698 + - type: ap + value: 72.08492726312224 + - type: f1 + value: 77.57721549038352 + - task: + type: Classification + dataset: + type: C-MTEB/JDReview-classification + name: MTEB JDReview + config: default + split: test + revision: None + metrics: + - type: accuracy + value: 83.63977485928706 + - type: ap + value: 48.33680179995013 + - type: f1 + value: 77.42875376726259 + - task: + type: STS + dataset: + type: C-MTEB/LCQMC + name: MTEB LCQMC + config: default + split: test + revision: None + metrics: + - type: cos_sim_pearson + value: 67.71826986847978 + - type: cos_sim_spearman + value: 75.31951271324436 + - type: euclidean_pearson + value: 73.99129929755692 + - type: euclidean_spearman + value: 75.50510874612128 + - type: manhattan_pearson + value: 74.1581557667118 + - type: manhattan_spearman + value: 75.62495446886778 + - task: + type: Retrieval + dataset: + type: C-MTEB/MMarcoRetrieval + name: MTEB MMarcoRetrieval + config: default + split: dev + revision: None + metrics: + - type: map_at_1 + value: 64.305 + - type: map_at_10 + value: 73.286 + - type: map_at_100 + value: 73.661 + - type: map_at_1000 + value: 73.675 + - type: map_at_3 + value: 71.433 + - type: map_at_5 + value: 72.596 + - type: mrr_at_1 + value: 66.562 + - type: mrr_at_10 + value: 73.932 + - type: mrr_at_100 + value: 74.265 + - type: mrr_at_1000 + value: 74.278 + - type: mrr_at_3 + value: 72.333 + - type: mrr_at_5 + value: 73.322 + - type: ndcg_at_1 + value: 66.562 + - type: ndcg_at_10 + value: 76.998 + - type: ndcg_at_100 + value: 78.684 + - type: ndcg_at_1000 + value: 79.038 + - type: ndcg_at_3 + value: 73.491 + - type: ndcg_at_5 + value: 75.436 + - type: precision_at_1 + value: 66.562 + - type: precision_at_10 + value: 9.34 + - type: precision_at_100 + value: 1.018 + - type: precision_at_1000 + value: 0.105 + - type: precision_at_3 + value: 27.683999999999997 + - type: precision_at_5 + value: 17.645 + - type: recall_at_1 + value: 64.305 + - type: recall_at_10 + value: 87.825 + - type: recall_at_100 + value: 95.451 + - type: recall_at_1000 + value: 98.17 + - type: recall_at_3 + value: 78.522 + - type: recall_at_5 + value: 83.146 + - task: + type: Retrieval + dataset: + type: msmarco + name: MTEB MSMARCO + config: default + split: dev + revision: None + metrics: + - type: map_at_1 + value: 21.862000000000002 + - type: map_at_10 + value: 33.635999999999996 + - type: map_at_100 + value: 34.833 + - type: map_at_1000 + value: 34.886 + - type: map_at_3 + value: 29.916999999999998 + - type: map_at_5 + value: 32.042 + - type: mrr_at_1 + value: 22.493 + - type: mrr_at_10 + value: 34.217999999999996 + - type: mrr_at_100 + value: 35.365 + - type: mrr_at_1000 + value: 35.411 + - type: mrr_at_3 + value: 30.585 + - type: mrr_at_5 + value: 32.659 + - type: ndcg_at_1 + value: 22.493 + - type: ndcg_at_10 + value: 40.247 + - type: ndcg_at_100 + value: 46.025 + - type: ndcg_at_1000 + value: 47.343 + - type: ndcg_at_3 + value: 32.696999999999996 + - type: ndcg_at_5 + value: 36.476 + - type: precision_at_1 + value: 22.493 + - type: precision_at_10 + value: 6.334 + - type: precision_at_100 + value: 0.922 + - type: precision_at_1000 + value: 0.104 + - type: precision_at_3 + value: 13.863 + - type: precision_at_5 + value: 10.232 + - type: recall_at_1 + value: 21.862000000000002 + - type: recall_at_10 + value: 60.56700000000001 + - type: recall_at_100 + value: 87.261 + - type: recall_at_1000 + value: 97.365 + - type: recall_at_3 + value: 40.081 + - type: recall_at_5 + value: 49.16 + - task: + type: Classification + dataset: + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (en) + config: en + split: test + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf + metrics: + - type: accuracy + value: 92.34154126766987 + - type: f1 + value: 92.05415284766352 + - task: + type: Classification + dataset: + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (en) + config: en + split: test + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba + metrics: + - type: accuracy + value: 70.63155494756043 + - type: f1 + value: 53.392602505424435 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (en) + config: en + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 70.39340954942837 + - type: f1 + value: 68.85705470713275 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (en) + config: en + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 77.18897108271688 + - type: f1 + value: 77.36699772115247 + - task: + type: Retrieval + dataset: + type: C-MTEB/MedicalRetrieval + name: MTEB MedicalRetrieval + config: default + split: dev + revision: None + metrics: + - type: map_at_1 + value: 40.699999999999996 + - type: map_at_10 + value: 45.304 + - type: map_at_100 + value: 45.862 + - type: map_at_1000 + value: 45.923 + - type: map_at_3 + value: 44.433 + - type: map_at_5 + value: 44.753 + - type: mrr_at_1 + value: 40.8 + - type: mrr_at_10 + value: 45.354 + - type: mrr_at_100 + value: 45.912 + - type: mrr_at_1000 + value: 45.973000000000006 + - type: mrr_at_3 + value: 44.483 + - type: mrr_at_5 + value: 44.803 + - type: ndcg_at_1 + value: 40.699999999999996 + - type: ndcg_at_10 + value: 47.477999999999994 + - type: ndcg_at_100 + value: 50.51 + - type: ndcg_at_1000 + value: 52.367 + - type: ndcg_at_3 + value: 45.609 + - type: ndcg_at_5 + value: 46.186 + - type: precision_at_1 + value: 40.699999999999996 + - type: precision_at_10 + value: 5.43 + - type: precision_at_100 + value: 0.692 + - type: precision_at_1000 + value: 0.084 + - type: precision_at_3 + value: 16.333000000000002 + - type: precision_at_5 + value: 10.08 + - type: recall_at_1 + value: 40.699999999999996 + - type: recall_at_10 + value: 54.300000000000004 + - type: recall_at_100 + value: 69.19999999999999 + - type: recall_at_1000 + value: 84.3 + - type: recall_at_3 + value: 49.0 + - type: recall_at_5 + value: 50.4 + - task: + type: Clustering + dataset: + type: mteb/medrxiv-clustering-p2p + name: MTEB MedrxivClusteringP2P + config: default + split: test + revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 + metrics: + - type: v_measure + value: 31.70883822617504 + - task: + type: Clustering + dataset: + type: mteb/medrxiv-clustering-s2s + name: MTEB MedrxivClusteringS2S + config: default + split: test + revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 + metrics: + - type: v_measure + value: 28.801248513598072 + - task: + type: Reranking + dataset: + type: mteb/mind_small + name: MTEB MindSmallReranking + config: default + split: test + revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 + metrics: + - type: map + value: 30.97227673339198 + - type: mrr + value: 32.03205560232119 + - task: + type: Reranking + dataset: + type: C-MTEB/Mmarco-reranking + name: MTEB MMarcoReranking + config: default + split: dev + revision: None + metrics: + - type: map + value: 25.89977615357687 + - type: mrr + value: 24.192857142857143 + - task: + type: Classification + dataset: + type: C-MTEB/MultilingualSentiment-classification + name: MTEB MultilingualSentiment + config: default + split: validation + revision: None + metrics: + - type: accuracy + value: 67.16666666666666 + - type: f1 + value: 67.15765577091656 + - task: + type: Retrieval + dataset: + type: nfcorpus + name: MTEB NFCorpus + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 5.079000000000001 + - type: map_at_10 + value: 12.04 + - type: map_at_100 + value: 15.375 + - type: map_at_1000 + value: 16.878 + - type: map_at_3 + value: 8.851 + - type: map_at_5 + value: 10.23 + - type: mrr_at_1 + value: 43.963 + - type: mrr_at_10 + value: 52.886 + - type: mrr_at_100 + value: 53.498000000000005 + - type: mrr_at_1000 + value: 53.54 + - type: mrr_at_3 + value: 50.876999999999995 + - type: mrr_at_5 + value: 52.254999999999995 + - type: ndcg_at_1 + value: 42.415000000000006 + - type: ndcg_at_10 + value: 33.660000000000004 + - type: ndcg_at_100 + value: 31.008000000000003 + - type: ndcg_at_1000 + value: 40.016 + - type: ndcg_at_3 + value: 39.329 + - type: ndcg_at_5 + value: 36.687999999999995 + - type: precision_at_1 + value: 43.963 + - type: precision_at_10 + value: 25.356 + - type: precision_at_100 + value: 8.245 + - type: precision_at_1000 + value: 2.106 + - type: precision_at_3 + value: 37.255 + - type: precision_at_5 + value: 31.95 + - type: recall_at_1 + value: 5.079000000000001 + - type: recall_at_10 + value: 15.838 + - type: recall_at_100 + value: 32.159 + - type: recall_at_1000 + value: 64.91799999999999 + - type: recall_at_3 + value: 10.152999999999999 + - type: recall_at_5 + value: 12.4 + - task: + type: Retrieval + dataset: + type: nq + name: MTEB NQ + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 29.605999999999998 + - type: map_at_10 + value: 43.518 + - type: map_at_100 + value: 44.583 + - type: map_at_1000 + value: 44.622 + - type: map_at_3 + value: 39.673 + - type: map_at_5 + value: 41.897 + - type: mrr_at_1 + value: 33.604 + - type: mrr_at_10 + value: 46.156000000000006 + - type: mrr_at_100 + value: 46.974 + - type: mrr_at_1000 + value: 47.002 + - type: mrr_at_3 + value: 42.907000000000004 + - type: mrr_at_5 + value: 44.792 + - type: ndcg_at_1 + value: 33.575 + - type: ndcg_at_10 + value: 50.61600000000001 + - type: ndcg_at_100 + value: 55.129 + - type: ndcg_at_1000 + value: 56.084 + - type: ndcg_at_3 + value: 43.297999999999995 + - type: ndcg_at_5 + value: 46.979 + - type: precision_at_1 + value: 33.575 + - type: precision_at_10 + value: 8.297 + - type: precision_at_100 + value: 1.083 + - type: precision_at_1000 + value: 0.117 + - type: precision_at_3 + value: 19.602 + - type: precision_at_5 + value: 13.934 + - type: recall_at_1 + value: 29.605999999999998 + - type: recall_at_10 + value: 69.718 + - type: recall_at_100 + value: 89.352 + - type: recall_at_1000 + value: 96.543 + - type: recall_at_3 + value: 50.617999999999995 + - type: recall_at_5 + value: 59.031 + - task: + type: PairClassification + dataset: + type: C-MTEB/OCNLI + name: MTEB Ocnli + config: default + split: validation + revision: None + metrics: + - type: cos_sim_accuracy + value: 65.83649160801299 + - type: cos_sim_ap + value: 69.86408265006916 + - type: cos_sim_f1 + value: 70.50709939148074 + - type: cos_sim_precision + value: 57.2463768115942 + - type: cos_sim_recall + value: 91.76346356916578 + - type: dot_accuracy + value: 61.93827828911749 + - type: dot_ap + value: 64.26140500313572 + - type: dot_f1 + value: 68.97081413210446 + - type: dot_precision + value: 54.19432709716355 + - type: dot_recall + value: 94.82576557550159 + - type: euclidean_accuracy + value: 66.32376827287493 + - type: euclidean_ap + value: 70.58216586017075 + - type: euclidean_f1 + value: 71.31782945736435 + - type: euclidean_precision + value: 58.11170212765957 + - type: euclidean_recall + value: 92.29144667370645 + - type: manhattan_accuracy + value: 66.54033567948024 + - type: manhattan_ap + value: 70.88996923294056 + - type: manhattan_f1 + value: 71.45256087321579 + - type: manhattan_precision + value: 59.30313588850174 + - type: manhattan_recall + value: 89.86272439281943 + - type: max_accuracy + value: 66.54033567948024 + - type: max_ap + value: 70.88996923294056 + - type: max_f1 + value: 71.45256087321579 + - task: + type: Classification + dataset: + type: C-MTEB/OnlineShopping-classification + name: MTEB OnlineShopping + config: default + split: test + revision: None + metrics: + - type: accuracy + value: 90.41 + - type: ap + value: 88.15736492425235 + - type: f1 + value: 90.40118324200982 + - task: + type: STS + dataset: + type: C-MTEB/PAWSX + name: MTEB PAWSX + config: default + split: test + revision: None + metrics: + - type: cos_sim_pearson + value: 14.718326697461064 + - type: cos_sim_spearman + value: 17.458017383716168 + - type: euclidean_pearson + value: 19.416710995216608 + - type: euclidean_spearman + value: 17.87886266073602 + - type: manhattan_pearson + value: 19.508696307778063 + - type: manhattan_spearman + value: 18.026398724663487 + - task: + type: STS + dataset: + type: C-MTEB/QBQTC + name: MTEB QBQTC + config: default + split: test + revision: None + metrics: + - type: cos_sim_pearson + value: 31.330102731068386 + - type: cos_sim_spearman + value: 33.69612492132476 + - type: euclidean_pearson + value: 33.83912666711584 + - type: euclidean_spearman + value: 35.58666712573462 + - type: manhattan_pearson + value: 34.257595977157706 + - type: manhattan_spearman + value: 36.08587604692898 + - task: + type: Retrieval + dataset: + type: quora + name: MTEB QuoraRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 70.37 + - type: map_at_10 + value: 84.22699999999999 + - type: map_at_100 + value: 84.871 + - type: map_at_1000 + value: 84.88900000000001 + - type: map_at_3 + value: 81.277 + - type: map_at_5 + value: 83.16799999999999 + - type: mrr_at_1 + value: 80.97 + - type: mrr_at_10 + value: 87.24300000000001 + - type: mrr_at_100 + value: 87.346 + - type: mrr_at_1000 + value: 87.347 + - type: mrr_at_3 + value: 86.258 + - type: mrr_at_5 + value: 86.914 + - type: ndcg_at_1 + value: 81.0 + - type: ndcg_at_10 + value: 88.009 + - type: ndcg_at_100 + value: 89.251 + - type: ndcg_at_1000 + value: 89.374 + - type: ndcg_at_3 + value: 85.169 + - type: ndcg_at_5 + value: 86.75399999999999 + - type: precision_at_1 + value: 81.0 + - type: precision_at_10 + value: 13.343 + - type: precision_at_100 + value: 1.526 + - type: precision_at_1000 + value: 0.157 + - type: precision_at_3 + value: 37.25 + - type: precision_at_5 + value: 24.504 + - type: recall_at_1 + value: 70.37 + - type: recall_at_10 + value: 95.158 + - type: recall_at_100 + value: 99.39 + - type: recall_at_1000 + value: 99.98 + - type: recall_at_3 + value: 86.942 + - type: recall_at_5 + value: 91.446 + - task: + type: Clustering + dataset: + type: mteb/reddit-clustering + name: MTEB RedditClustering + config: default + split: test + revision: 24640382cdbf8abc73003fb0fa6d111a705499eb + metrics: + - type: v_measure + value: 49.71370818375339 + - task: + type: Clustering + dataset: + type: mteb/reddit-clustering-p2p + name: MTEB RedditClusteringP2P + config: default + split: test + revision: 282350215ef01743dc01b456c7f5241fa8937f16 + metrics: + - type: v_measure + value: 55.07451965473589 + - task: + type: Retrieval + dataset: + type: scidocs + name: MTEB SCIDOCS + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 4.508 + - type: map_at_10 + value: 10.825 + - type: map_at_100 + value: 12.598 + - type: map_at_1000 + value: 12.854 + - type: map_at_3 + value: 7.892 + - type: map_at_5 + value: 9.349 + - type: mrr_at_1 + value: 22.2 + - type: mrr_at_10 + value: 32.611000000000004 + - type: mrr_at_100 + value: 33.61 + - type: mrr_at_1000 + value: 33.671 + - type: mrr_at_3 + value: 29.15 + - type: mrr_at_5 + value: 31.225 + - type: ndcg_at_1 + value: 22.2 + - type: ndcg_at_10 + value: 18.502 + - type: ndcg_at_100 + value: 25.424999999999997 + - type: ndcg_at_1000 + value: 30.233999999999998 + - type: ndcg_at_3 + value: 17.711 + - type: ndcg_at_5 + value: 15.501000000000001 + - type: precision_at_1 + value: 22.2 + - type: precision_at_10 + value: 9.49 + - type: precision_at_100 + value: 1.941 + - type: precision_at_1000 + value: 0.31 + - type: precision_at_3 + value: 16.433 + - type: precision_at_5 + value: 13.54 + - type: recall_at_1 + value: 4.508 + - type: recall_at_10 + value: 19.243 + - type: recall_at_100 + value: 39.407 + - type: recall_at_1000 + value: 62.953 + - type: recall_at_3 + value: 9.993 + - type: recall_at_5 + value: 13.733 + - task: + type: STS + dataset: + type: mteb/sickr-sts + name: MTEB SICK-R + config: default + split: test + revision: a6ea5a8cab320b040a23452cc28066d9beae2cee + metrics: + - type: cos_sim_pearson + value: 85.88096352325879 + - type: cos_sim_spearman + value: 80.84882728439892 + - type: euclidean_pearson + value: 82.89512161923362 + - type: euclidean_spearman + value: 80.69723454935396 + - type: manhattan_pearson + value: 82.94365287299226 + - type: manhattan_spearman + value: 80.64700541831023 + - task: + type: STS + dataset: + type: mteb/sts12-sts + name: MTEB STS12 + config: default + split: test + revision: a0d554a64d88156834ff5ae9920b964011b16384 + metrics: + - type: cos_sim_pearson + value: 84.09030569824817 + - type: cos_sim_spearman + value: 76.10288448289813 + - type: euclidean_pearson + value: 82.19317617787483 + - type: euclidean_spearman + value: 78.51206398528993 + - type: manhattan_pearson + value: 82.50688072451729 + - type: manhattan_spearman + value: 78.71694597298867 + - task: + type: STS + dataset: + type: mteb/sts13-sts + name: MTEB STS13 + config: default + split: test + revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca + metrics: + - type: cos_sim_pearson + value: 85.04298066236511 + - type: cos_sim_spearman + value: 85.49051395372348 + - type: euclidean_pearson + value: 85.7369561800059 + - type: euclidean_spearman + value: 86.35626949911497 + - type: manhattan_pearson + value: 85.86766305481635 + - type: manhattan_spearman + value: 86.5115276036124 + - task: + type: STS + dataset: + type: mteb/sts14-sts + name: MTEB STS14 + config: default + split: test + revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 + metrics: + - type: cos_sim_pearson + value: 83.98107748125086 + - type: cos_sim_spearman + value: 80.43502071880916 + - type: euclidean_pearson + value: 82.24603130661005 + - type: euclidean_spearman + value: 80.94302742946145 + - type: manhattan_pearson + value: 82.4215619893203 + - type: manhattan_spearman + value: 81.13824893869541 + - task: + type: STS + dataset: + type: mteb/sts15-sts + name: MTEB STS15 + config: default + split: test + revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 + metrics: + - type: cos_sim_pearson + value: 86.95857345426359 + - type: cos_sim_spearman + value: 87.7540379885978 + - type: euclidean_pearson + value: 87.86433964223119 + - type: euclidean_spearman + value: 88.43585275816753 + - type: manhattan_pearson + value: 87.90915813062988 + - type: manhattan_spearman + value: 88.49038031429657 + - task: + type: STS + dataset: + type: mteb/sts16-sts + name: MTEB STS16 + config: default + split: test + revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 + metrics: + - type: cos_sim_pearson + value: 83.84530028548023 + - type: cos_sim_spearman + value: 85.42197371225963 + - type: euclidean_pearson + value: 84.12042159341938 + - type: euclidean_spearman + value: 84.69864997658445 + - type: manhattan_pearson + value: 84.09772815909784 + - type: manhattan_spearman + value: 84.63986468736967 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (en-en) + config: en-en + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 89.89281017946413 + - type: cos_sim_spearman + value: 89.94783195991867 + - type: euclidean_pearson + value: 89.19342633226815 + - type: euclidean_spearman + value: 88.6692137120815 + - type: manhattan_pearson + value: 89.19006596701496 + - type: manhattan_spearman + value: 88.65041672073397 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (en) + config: en + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 65.05176237336566 + - type: cos_sim_spearman + value: 65.12758602746149 + - type: euclidean_pearson + value: 67.44468889455905 + - type: euclidean_spearman + value: 67.42836832904808 + - type: manhattan_pearson + value: 67.99438187200471 + - type: manhattan_spearman + value: 67.96190936270705 + - task: + type: STS + dataset: + type: C-MTEB/STSB + name: MTEB STSB + config: default + split: test + revision: None + metrics: + - type: cos_sim_pearson + value: 81.36171514729287 + - type: cos_sim_spearman + value: 81.51752389848613 + - type: euclidean_pearson + value: 81.14136234145765 + - type: euclidean_spearman + value: 81.27609983297867 + - type: manhattan_pearson + value: 81.44966268348165 + - type: manhattan_spearman + value: 81.53484018091312 + - task: + type: STS + dataset: + type: mteb/stsbenchmark-sts + name: MTEB STSBenchmark + config: default + split: test + revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 + metrics: + - type: cos_sim_pearson + value: 86.92195724268996 + - type: cos_sim_spearman + value: 87.70682082313391 + - type: euclidean_pearson + value: 86.24220109166684 + - type: euclidean_spearman + value: 86.51998671092596 + - type: manhattan_pearson + value: 86.17577571663554 + - type: manhattan_spearman + value: 86.45961101071687 + - task: + type: Reranking + dataset: + type: mteb/scidocs-reranking + name: MTEB SciDocsRR + config: default + split: test + revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab + metrics: + - type: map + value: 78.62106635785725 + - type: mrr + value: 93.84658279266121 + - task: + type: Retrieval + dataset: + type: scifact + name: MTEB SciFact + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 53.761 + - type: map_at_10 + value: 64.56 + - type: map_at_100 + value: 65.243 + - type: map_at_1000 + value: 65.269 + - type: map_at_3 + value: 62.156 + - type: map_at_5 + value: 63.55 + - type: mrr_at_1 + value: 56.667 + - type: mrr_at_10 + value: 66.084 + - type: mrr_at_100 + value: 66.58500000000001 + - type: mrr_at_1000 + value: 66.61 + - type: mrr_at_3 + value: 64.333 + - type: mrr_at_5 + value: 65.3 + - type: ndcg_at_1 + value: 56.667 + - type: ndcg_at_10 + value: 69.43 + - type: ndcg_at_100 + value: 72.031 + - type: ndcg_at_1000 + value: 72.75 + - type: ndcg_at_3 + value: 65.282 + - type: ndcg_at_5 + value: 67.24900000000001 + - type: precision_at_1 + value: 56.667 + - type: precision_at_10 + value: 9.3 + - type: precision_at_100 + value: 1.0670000000000002 + - type: precision_at_1000 + value: 0.11199999999999999 + - type: precision_at_3 + value: 25.778000000000002 + - type: precision_at_5 + value: 16.866999999999997 + - type: recall_at_1 + value: 53.761 + - type: recall_at_10 + value: 82.678 + - type: recall_at_100 + value: 93.667 + - type: recall_at_1000 + value: 99.333 + - type: recall_at_3 + value: 71.578 + - type: recall_at_5 + value: 76.25 + - task: + type: PairClassification + dataset: + type: mteb/sprintduplicatequestions-pairclassification + name: MTEB SprintDuplicateQuestions + config: default + split: test + revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 + metrics: + - type: cos_sim_accuracy + value: 99.80594059405941 + - type: cos_sim_ap + value: 95.35711574476811 + - type: cos_sim_f1 + value: 90.12096774193547 + - type: cos_sim_precision + value: 90.85365853658537 + - type: cos_sim_recall + value: 89.4 + - type: dot_accuracy + value: 99.76732673267327 + - type: dot_ap + value: 93.20624501431367 + - type: dot_f1 + value: 87.74126238914971 + - type: dot_precision + value: 91.71210468920393 + - type: dot_recall + value: 84.1 + - type: euclidean_accuracy + value: 99.80594059405941 + - type: euclidean_ap + value: 95.35758863966429 + - type: euclidean_f1 + value: 90.15075376884421 + - type: euclidean_precision + value: 90.6060606060606 + - type: euclidean_recall + value: 89.7 + - type: manhattan_accuracy + value: 99.80990099009901 + - type: manhattan_ap + value: 95.48335466728275 + - type: manhattan_f1 + value: 90.2672718103883 + - type: manhattan_precision + value: 91.04781281790437 + - type: manhattan_recall + value: 89.5 + - type: max_accuracy + value: 99.80990099009901 + - type: max_ap + value: 95.48335466728275 + - type: max_f1 + value: 90.2672718103883 + - task: + type: Clustering + dataset: + type: mteb/stackexchange-clustering + name: MTEB StackExchangeClustering + config: default + split: test + revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 + metrics: + - type: v_measure + value: 59.422562431402845 + - task: + type: Clustering + dataset: + type: mteb/stackexchange-clustering-p2p + name: MTEB StackExchangeClusteringP2P + config: default + split: test + revision: 815ca46b2622cec33ccafc3735d572c266efdb44 + metrics: + - type: v_measure + value: 31.695493629721373 + - task: + type: Reranking + dataset: + type: mteb/stackoverflowdupquestions-reranking + name: MTEB StackOverflowDupQuestions + config: default + split: test + revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 + metrics: + - type: map + value: 50.070077950465965 + - type: mrr + value: 50.72293311263899 + - task: + type: Summarization + dataset: + type: mteb/summeval + name: MTEB SummEval + config: default + split: test + revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c + metrics: + - type: cos_sim_pearson + value: 30.59608436984981 + - type: cos_sim_spearman + value: 30.617289383193103 + - type: dot_pearson + value: 30.78715584903813 + - type: dot_spearman + value: 31.269245492805283 + - task: + type: Reranking + dataset: + type: C-MTEB/T2Reranking + name: MTEB T2Reranking + config: default + split: dev + revision: None + metrics: + - type: map + value: 66.49332760690612 + - type: mrr + value: 76.52668294806075 + - task: + type: Retrieval + dataset: + type: C-MTEB/T2Retrieval + name: MTEB T2Retrieval + config: default + split: dev + revision: None + metrics: + - type: map_at_1 + value: 24.607 + - type: map_at_10 + value: 67.009 + - type: map_at_100 + value: 70.838 + - type: map_at_1000 + value: 70.954 + - type: map_at_3 + value: 47.573 + - type: map_at_5 + value: 58.10999999999999 + - type: mrr_at_1 + value: 84.333 + - type: mrr_at_10 + value: 87.822 + - type: mrr_at_100 + value: 87.969 + - type: mrr_at_1000 + value: 87.97500000000001 + - type: mrr_at_3 + value: 87.16000000000001 + - type: mrr_at_5 + value: 87.587 + - type: ndcg_at_1 + value: 84.333 + - type: ndcg_at_10 + value: 76.303 + - type: ndcg_at_100 + value: 81.05499999999999 + - type: ndcg_at_1000 + value: 82.218 + - type: ndcg_at_3 + value: 78.691 + - type: ndcg_at_5 + value: 76.66 + - type: precision_at_1 + value: 84.333 + - type: precision_at_10 + value: 38.019999999999996 + - type: precision_at_100 + value: 4.7669999999999995 + - type: precision_at_1000 + value: 0.505 + - type: precision_at_3 + value: 68.939 + - type: precision_at_5 + value: 57.306999999999995 + - type: recall_at_1 + value: 24.607 + - type: recall_at_10 + value: 74.971 + - type: recall_at_100 + value: 90.108 + - type: recall_at_1000 + value: 95.917 + - type: recall_at_3 + value: 49.586000000000006 + - type: recall_at_5 + value: 62.232 + - task: + type: Classification + dataset: + type: C-MTEB/TNews-classification + name: MTEB TNews + config: default + split: validation + revision: None + metrics: + - type: accuracy + value: 47.702 + - type: f1 + value: 46.274469606672426 + - task: + type: Retrieval + dataset: + type: trec-covid + name: MTEB TRECCOVID + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 0.252 + - type: map_at_10 + value: 2.178 + - type: map_at_100 + value: 12.781999999999998 + - type: map_at_1000 + value: 29.494999999999997 + - type: map_at_3 + value: 0.73 + - type: map_at_5 + value: 1.169 + - type: mrr_at_1 + value: 94.0 + - type: mrr_at_10 + value: 97.0 + - type: mrr_at_100 + value: 97.0 + - type: mrr_at_1000 + value: 97.0 + - type: mrr_at_3 + value: 97.0 + - type: mrr_at_5 + value: 97.0 + - type: ndcg_at_1 + value: 88.0 + - type: ndcg_at_10 + value: 83.21 + - type: ndcg_at_100 + value: 63.31 + - type: ndcg_at_1000 + value: 54.734 + - type: ndcg_at_3 + value: 87.408 + - type: ndcg_at_5 + value: 86.20100000000001 + - type: precision_at_1 + value: 94.0 + - type: precision_at_10 + value: 88.2 + - type: precision_at_100 + value: 64.68 + - type: precision_at_1000 + value: 23.966 + - type: precision_at_3 + value: 93.333 + - type: precision_at_5 + value: 91.60000000000001 + - type: recall_at_1 + value: 0.252 + - type: recall_at_10 + value: 2.307 + - type: recall_at_100 + value: 15.703 + - type: recall_at_1000 + value: 51.111 + - type: recall_at_3 + value: 0.749 + - type: recall_at_5 + value: 1.212 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (sqi-eng) + config: sqi-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 16.8 + - type: f1 + value: 13.168299935527422 + - type: precision + value: 12.209559281760876 + - type: recall + value: 16.8 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (fry-eng) + config: fry-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 35.83815028901734 + - type: f1 + value: 29.0852500101055 + - type: precision + value: 26.965317919075147 + - type: recall + value: 35.83815028901734 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (kur-eng) + config: kur-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 15.121951219512194 + - type: f1 + value: 11.844149203614325 + - type: precision + value: 11.042929292929294 + - type: recall + value: 15.121951219512194 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tur-eng) + config: tur-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 9.9 + - type: f1 + value: 7.1396348187007215 + - type: precision + value: 6.501835713997978 + - type: recall + value: 9.9 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (deu-eng) + config: deu-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 76.6 + - type: f1 + value: 72.73241758241758 + - type: precision + value: 71.18867647058823 + - type: recall + value: 76.6 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (nld-eng) + config: nld-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 42.0 + - type: f1 + value: 36.81003102453103 + - type: precision + value: 35.19870269535562 + - type: recall + value: 42.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ron-eng) + config: ron-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 35.3 + - type: f1 + value: 30.353777056277053 + - type: precision + value: 28.773956778515604 + - type: recall + value: 35.3 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ang-eng) + config: ang-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 35.82089552238806 + - type: f1 + value: 27.44136460554371 + - type: precision + value: 24.340796019900495 + - type: recall + value: 35.82089552238806 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ido-eng) + config: ido-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 51.800000000000004 + - type: f1 + value: 45.82491836793846 + - type: precision + value: 43.729303094622864 + - type: recall + value: 51.800000000000004 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (jav-eng) + config: jav-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 25.853658536585368 + - type: f1 + value: 19.79869362796192 + - type: precision + value: 18.250680214094846 + - type: recall + value: 25.853658536585368 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (isl-eng) + config: isl-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 9.0 + - type: f1 + value: 6.926590762281661 + - type: precision + value: 6.507185696775364 + - type: recall + value: 9.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (slv-eng) + config: slv-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 14.33778857837181 + - type: f1 + value: 10.888963524130242 + - type: precision + value: 10.189272116928368 + - type: recall + value: 14.33778857837181 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (cym-eng) + config: cym-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 11.304347826086957 + - type: f1 + value: 8.459121175343064 + - type: precision + value: 7.7218644669759975 + - type: recall + value: 11.304347826086957 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (kaz-eng) + config: kaz-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 8.521739130434783 + - type: f1 + value: 6.751744703151353 + - type: precision + value: 6.387004921960017 + - type: recall + value: 8.521739130434783 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (est-eng) + config: est-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 7.3 + - type: f1 + value: 5.626766011766011 + - type: precision + value: 5.1270385799923 + - type: recall + value: 7.3 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (heb-eng) + config: heb-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 3.2 + - type: f1 + value: 1.91950282507703 + - type: precision + value: 1.6684431360304504 + - type: recall + value: 3.2 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (gla-eng) + config: gla-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 5.790108564535585 + - type: f1 + value: 4.128499324411468 + - type: precision + value: 3.8151453928788914 + - type: recall + value: 5.790108564535585 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (mar-eng) + config: mar-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 70.3 + - type: f1 + value: 65.18318181818181 + - type: precision + value: 63.126911976911984 + - type: recall + value: 70.3 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (lat-eng) + config: lat-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 45.300000000000004 + - type: f1 + value: 38.339152873270514 + - type: precision + value: 36.130903304212126 + - type: recall + value: 45.300000000000004 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (bel-eng) + config: bel-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 16.0 + - type: f1 + value: 12.172850459161385 + - type: precision + value: 11.27855570316309 + - type: recall + value: 16.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (pms-eng) + config: pms-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 37.714285714285715 + - type: f1 + value: 32.188793178089945 + - type: precision + value: 30.457500778089013 + - type: recall + value: 37.714285714285715 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (gle-eng) + config: gle-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 6.5 + - type: f1 + value: 4.528544131928126 + - type: precision + value: 4.171387799947767 + - type: recall + value: 6.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (pes-eng) + config: pes-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 21.0 + - type: f1 + value: 17.006564035803166 + - type: precision + value: 15.844832112332114 + - type: recall + value: 21.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (nob-eng) + config: nob-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 25.5 + - type: f1 + value: 22.79430820164996 + - type: precision + value: 21.938476924594045 + - type: recall + value: 25.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (bul-eng) + config: bul-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 33.7 + - type: f1 + value: 26.898922166422164 + - type: precision + value: 24.939117884031678 + - type: recall + value: 33.7 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (cbk-eng) + config: cbk-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 69.0 + - type: f1 + value: 63.68992285492286 + - type: precision + value: 61.72837301587302 + - type: recall + value: 69.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (hun-eng) + config: hun-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 7.3999999999999995 + - type: f1 + value: 5.5655686223658565 + - type: precision + value: 5.119921502146487 + - type: recall + value: 7.3999999999999995 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (uig-eng) + config: uig-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 1.5 + - type: f1 + value: 1.001208686507139 + - type: precision + value: 0.9683730903243098 + - type: recall + value: 1.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (rus-eng) + config: rus-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 69.0 + - type: f1 + value: 62.61056277056276 + - type: precision + value: 59.96357142857143 + - type: recall + value: 69.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (spa-eng) + config: spa-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 98.3 + - type: f1 + value: 97.76666666666668 + - type: precision + value: 97.51666666666668 + - type: recall + value: 98.3 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (hye-eng) + config: hye-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 2.0215633423180592 + - type: f1 + value: 1.5634923413129036 + - type: precision + value: 1.4895885785373653 + - type: recall + value: 2.0215633423180592 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tel-eng) + config: tel-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 83.33333333333334 + - type: f1 + value: 79.3019943019943 + - type: precision + value: 77.45726495726495 + - type: recall + value: 83.33333333333334 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (afr-eng) + config: afr-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 23.400000000000002 + - type: f1 + value: 18.655079988631996 + - type: precision + value: 17.338269096494905 + - type: recall + value: 23.400000000000002 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (mon-eng) + config: mon-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 6.363636363636363 + - type: f1 + value: 4.48376251469035 + - type: precision + value: 4.071778641679957 + - type: recall + value: 6.363636363636363 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (arz-eng) + config: arz-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 77.56813417190776 + - type: f1 + value: 73.16561844863732 + - type: precision + value: 71.3440484509667 + - type: recall + value: 77.56813417190776 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (hrv-eng) + config: hrv-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 17.299999999999997 + - type: f1 + value: 13.693204564375854 + - type: precision + value: 12.830651358081276 + - type: recall + value: 17.299999999999997 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (nov-eng) + config: nov-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 59.92217898832685 + - type: f1 + value: 53.29591938541354 + - type: precision + value: 50.58736335000926 + - type: recall + value: 59.92217898832685 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (gsw-eng) + config: gsw-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 25.64102564102564 + - type: f1 + value: 19.31404777558624 + - type: precision + value: 17.413105413105416 + - type: recall + value: 25.64102564102564 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (nds-eng) + config: nds-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 29.7 + - type: f1 + value: 24.44977050316952 + - type: precision + value: 22.798075396825396 + - type: recall + value: 29.7 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ukr-eng) + config: ukr-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 32.2 + - type: f1 + value: 25.423187804627435 + - type: precision + value: 23.404003309492442 + - type: recall + value: 32.2 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (uzb-eng) + config: uzb-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 9.11214953271028 + - type: f1 + value: 5.910063827286792 + - type: precision + value: 5.296401380795872 + - type: recall + value: 9.11214953271028 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (lit-eng) + config: lit-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 7.199999999999999 + - type: f1 + value: 5.816726797396153 + - type: precision + value: 5.508698718788661 + - type: recall + value: 7.199999999999999 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ina-eng) + config: ina-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 87.2 + - type: f1 + value: 83.88333333333333 + - type: precision + value: 82.42833333333333 + - type: recall + value: 87.2 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (lfn-eng) + config: lfn-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 53.7 + - type: f1 + value: 48.25312435500516 + - type: precision + value: 46.34107401656314 + - type: recall + value: 53.7 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (zsm-eng) + config: zsm-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 88.1 + - type: f1 + value: 85.21690476190476 + - type: precision + value: 83.96761904761905 + - type: recall + value: 88.1 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ita-eng) + config: ita-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 78.10000000000001 + - type: f1 + value: 73.38746031746032 + - type: precision + value: 71.47583333333334 + - type: recall + value: 78.10000000000001 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (cmn-eng) + config: cmn-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 96.1 + - type: f1 + value: 95.08333333333333 + - type: precision + value: 94.58333333333334 + - type: recall + value: 96.1 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (lvs-eng) + config: lvs-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 9.0 + - type: f1 + value: 6.952605595133894 + - type: precision + value: 6.457724621713984 + - type: recall + value: 9.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (glg-eng) + config: glg-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 84.7 + - type: f1 + value: 80.97880952380953 + - type: precision + value: 79.36428571428571 + - type: recall + value: 84.7 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ceb-eng) + config: ceb-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 10.5 + - type: f1 + value: 8.146458694813958 + - type: precision + value: 7.618942433110826 + - type: recall + value: 10.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (bre-eng) + config: bre-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 8.4 + - type: f1 + value: 6.144921607886653 + - type: precision + value: 5.5261043562899586 + - type: recall + value: 8.4 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ben-eng) + config: ben-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 84.39999999999999 + - type: f1 + value: 80.65333333333334 + - type: precision + value: 78.97833333333332 + - type: recall + value: 84.39999999999999 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (swg-eng) + config: swg-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 28.57142857142857 + - type: f1 + value: 22.767379679144387 + - type: precision + value: 21.2016369047619 + - type: recall + value: 28.57142857142857 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (arq-eng) + config: arq-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 34.24807903402854 + - type: f1 + value: 29.241572730305222 + - type: precision + value: 27.6428310072657 + - type: recall + value: 34.24807903402854 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (kab-eng) + config: kab-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 2.9000000000000004 + - type: f1 + value: 1.9156734696693711 + - type: precision + value: 1.7528460881307182 + - type: recall + value: 2.9000000000000004 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (fra-eng) + config: fra-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 94.89999999999999 + - type: f1 + value: 93.53333333333332 + - type: precision + value: 92.90666666666667 + - type: recall + value: 94.89999999999999 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (por-eng) + config: por-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 95.0 + - type: f1 + value: 93.61666666666666 + - type: precision + value: 92.93333333333332 + - type: recall + value: 95.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tat-eng) + config: tat-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 6.3 + - type: f1 + value: 4.920070356472795 + - type: precision + value: 4.565811270125224 + - type: recall + value: 6.3 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (oci-eng) + config: oci-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 47.4 + - type: f1 + value: 41.08392857142857 + - type: precision + value: 38.999704968944094 + - type: recall + value: 47.4 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (pol-eng) + config: pol-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 18.2 + - type: f1 + value: 14.826165036734295 + - type: precision + value: 13.988559330454489 + - type: recall + value: 18.2 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (war-eng) + config: war-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 13.3 + - type: f1 + value: 10.73451225789461 + - type: precision + value: 10.06524508030025 + - type: recall + value: 13.3 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (aze-eng) + config: aze-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 9.3 + - type: f1 + value: 7.613044370901514 + - type: precision + value: 7.184100384035204 + - type: recall + value: 9.3 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (vie-eng) + config: vie-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 97.0 + - type: f1 + value: 96.05 + - type: precision + value: 95.58333333333334 + - type: recall + value: 97.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (nno-eng) + config: nno-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 19.8 + - type: f1 + value: 16.070523504273503 + - type: precision + value: 14.848185626325227 + - type: recall + value: 19.8 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (cha-eng) + config: cha-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 29.1970802919708 + - type: f1 + value: 22.579707397225647 + - type: precision + value: 20.792945550165477 + - type: recall + value: 29.1970802919708 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (mhr-eng) + config: mhr-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 4.3 + - type: f1 + value: 2.884495496452018 + - type: precision + value: 2.6280916815877506 + - type: recall + value: 4.3 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (dan-eng) + config: dan-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 28.7 + - type: f1 + value: 24.9056519214062 + - type: precision + value: 23.800155414494334 + - type: recall + value: 28.7 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ell-eng) + config: ell-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 9.5 + - type: f1 + value: 6.723431537130878 + - type: precision + value: 6.078266616597544 + - type: recall + value: 9.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (amh-eng) + config: amh-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 1.7857142857142856 + - type: f1 + value: 0.4579590594653929 + - type: precision + value: 0.32939943654229364 + - type: recall + value: 1.7857142857142856 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (pam-eng) + config: pam-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 9.1 + - type: f1 + value: 7.1794182614770845 + - type: precision + value: 6.81138018671376 + - type: recall + value: 9.1 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (hsb-eng) + config: hsb-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 15.113871635610765 + - type: f1 + value: 12.353104530336957 + - type: precision + value: 11.66106754766342 + - type: recall + value: 15.113871635610765 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (srp-eng) + config: srp-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 18.4 + - type: f1 + value: 15.091645001025805 + - type: precision + value: 14.200823959052217 + - type: recall + value: 18.4 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (epo-eng) + config: epo-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 33.2 + - type: f1 + value: 28.066634199134192 + - type: precision + value: 26.54372717117398 + - type: recall + value: 33.2 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (kzj-eng) + config: kzj-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 7.6 + - type: f1 + value: 5.992580343865051 + - type: precision + value: 5.7409125738839055 + - type: recall + value: 7.6 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (awa-eng) + config: awa-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 52.81385281385281 + - type: f1 + value: 46.86834810211434 + - type: precision + value: 45.13687899402185 + - type: recall + value: 52.81385281385281 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (fao-eng) + config: fao-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 16.030534351145036 + - type: f1 + value: 12.902313597194603 + - type: precision + value: 12.19757977391565 + - type: recall + value: 16.030534351145036 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (mal-eng) + config: mal-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 94.75982532751091 + - type: f1 + value: 93.11984473556527 + - type: precision + value: 92.3216885007278 + - type: recall + value: 94.75982532751091 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ile-eng) + config: ile-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 70.19999999999999 + - type: f1 + value: 64.41237595737596 + - type: precision + value: 62.074285714285715 + - type: recall + value: 70.19999999999999 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (bos-eng) + config: bos-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 19.2090395480226 + - type: f1 + value: 14.986259497894084 + - type: precision + value: 14.08083152750014 + - type: recall + value: 19.2090395480226 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (cor-eng) + config: cor-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 5.800000000000001 + - type: f1 + value: 4.004811414639001 + - type: precision + value: 3.611296721493974 + - type: recall + value: 5.800000000000001 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (cat-eng) + config: cat-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 93.10000000000001 + - type: f1 + value: 91.17333333333335 + - type: precision + value: 90.27833333333334 + - type: recall + value: 93.10000000000001 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (eus-eng) + config: eus-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 68.2 + - type: f1 + value: 63.805870279146134 + - type: precision + value: 62.064924029458915 + - type: recall + value: 68.2 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (yue-eng) + config: yue-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 88.9 + - type: f1 + value: 86.38250000000001 + - type: precision + value: 85.345 + - type: recall + value: 88.9 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (swe-eng) + config: swe-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 26.3 + - type: f1 + value: 21.72601907540825 + - type: precision + value: 20.3161132602622 + - type: recall + value: 26.3 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (dtp-eng) + config: dtp-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 6.6000000000000005 + - type: f1 + value: 5.4107919446503585 + - type: precision + value: 5.143205186348676 + - type: recall + value: 6.6000000000000005 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (kat-eng) + config: kat-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 1.2064343163538873 + - type: f1 + value: 0.7118331023204635 + - type: precision + value: 0.6930197065411955 + - type: recall + value: 1.2064343163538873 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (jpn-eng) + config: jpn-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 78.0 + - type: f1 + value: 73.95134920634919 + - type: precision + value: 72.3770634920635 + - type: recall + value: 78.0 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (csb-eng) + config: csb-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 12.648221343873518 + - type: f1 + value: 10.259994816302727 + - type: precision + value: 9.677206851119895 + - type: recall + value: 12.648221343873518 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (xho-eng) + config: xho-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 10.56338028169014 + - type: f1 + value: 7.792644757433489 + - type: precision + value: 7.299087316692951 + - type: recall + value: 10.56338028169014 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (orv-eng) + config: orv-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 8.1437125748503 + - type: f1 + value: 5.6113303405098724 + - type: precision + value: 5.156075980223929 + - type: recall + value: 8.1437125748503 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ind-eng) + config: ind-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 92.5 + - type: f1 + value: 90.53999999999999 + - type: precision + value: 89.64500000000001 + - type: recall + value: 92.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tuk-eng) + config: tuk-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 8.374384236453201 + - type: f1 + value: 5.831645092728836 + - type: precision + value: 5.241568776051535 + - type: recall + value: 8.374384236453201 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (max-eng) + config: max-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 45.42253521126761 + - type: f1 + value: 40.878561970111264 + - type: precision + value: 39.52681669728516 + - type: recall + value: 45.42253521126761 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (swh-eng) + config: swh-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 32.05128205128205 + - type: f1 + value: 25.433010420698523 + - type: precision + value: 23.545685308843208 + - type: recall + value: 32.05128205128205 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (hin-eng) + config: hin-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 94.6 + - type: f1 + value: 92.86666666666666 + - type: precision + value: 92.01666666666667 + - type: recall + value: 94.6 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (dsb-eng) + config: dsb-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 14.822546972860126 + - type: f1 + value: 12.439321820122155 + - type: precision + value: 11.940341857811413 + - type: recall + value: 14.822546972860126 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ber-eng) + config: ber-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 6.7 + - type: f1 + value: 5.534443298607457 + - type: precision + value: 5.299107273391812 + - type: recall + value: 6.7 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tam-eng) + config: tam-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 87.94788273615634 + - type: f1 + value: 84.65798045602605 + - type: precision + value: 83.2084690553746 + - type: recall + value: 87.94788273615634 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (slk-eng) + config: slk-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 13.8 + - type: f1 + value: 11.356912127897372 + - type: precision + value: 10.778191051205624 + - type: recall + value: 13.8 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tgl-eng) + config: tgl-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 13.700000000000001 + - type: f1 + value: 10.74774895608627 + - type: precision + value: 9.966243757837463 + - type: recall + value: 13.700000000000001 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ast-eng) + config: ast-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 76.37795275590551 + - type: f1 + value: 71.24671916010499 + - type: precision + value: 69.20697412823397 + - type: recall + value: 76.37795275590551 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (mkd-eng) + config: mkd-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 18.099999999999998 + - type: f1 + value: 13.934122253809159 + - type: precision + value: 12.815974391105971 + - type: recall + value: 18.099999999999998 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (khm-eng) + config: khm-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 0.6925207756232686 + - type: f1 + value: 0.08966600365830146 + - type: precision + value: 0.05066184676394412 + - type: recall + value: 0.6925207756232686 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ces-eng) + config: ces-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 11.1 + - type: f1 + value: 8.28646043238052 + - type: precision + value: 7.686198801198802 + - type: recall + value: 11.1 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tzl-eng) + config: tzl-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 38.46153846153847 + - type: f1 + value: 31.640899949723472 + - type: precision + value: 29.298878205128204 + - type: recall + value: 38.46153846153847 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (urd-eng) + config: urd-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 81.2 + - type: f1 + value: 76.77103174603175 + - type: precision + value: 74.96511904761905 + - type: recall + value: 81.2 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (ara-eng) + config: ara-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 90.60000000000001 + - type: f1 + value: 88.20666666666665 + - type: precision + value: 87.14833333333334 + - type: recall + value: 90.60000000000001 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (kor-eng) + config: kor-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 35.699999999999996 + - type: f1 + value: 29.159127620745267 + - type: precision + value: 27.109529030910608 + - type: recall + value: 35.699999999999996 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (yid-eng) + config: yid-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 0.9433962264150944 + - type: f1 + value: 0.28088681664921333 + - type: precision + value: 0.22694150916099465 + - type: recall + value: 0.9433962264150944 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (fin-eng) + config: fin-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 7.5 + - type: f1 + value: 5.825362182391272 + - type: precision + value: 5.526187577939453 + - type: recall + value: 7.5 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (tha-eng) + config: tha-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 4.197080291970803 + - type: f1 + value: 3.079215618580677 + - type: precision + value: 2.8501768792419 + - type: recall + value: 4.197080291970803 + - task: + type: BitextMining + dataset: + type: mteb/tatoeba-bitext-mining + name: MTEB Tatoeba (wuu-eng) + config: wuu-eng + split: test + revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 + metrics: + - type: accuracy + value: 87.9 + - type: f1 + value: 84.60499999999999 + - type: precision + value: 83.11428571428571 + - type: recall + value: 87.9 + - task: + type: Clustering + dataset: + type: C-MTEB/ThuNewsClusteringP2P + name: MTEB ThuNewsClusteringP2P + config: default + split: test + revision: None + metrics: + - type: v_measure + value: 50.23655676494653 + - task: + type: Clustering + dataset: + type: C-MTEB/ThuNewsClusteringS2S + name: MTEB ThuNewsClusteringS2S + config: default + split: test + revision: None + metrics: + - type: v_measure + value: 49.54033078256682 + - task: + type: Retrieval + dataset: + type: webis-touche2020 + name: MTEB Touche2020 + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 2.299 + - type: map_at_10 + value: 9.232999999999999 + - type: map_at_100 + value: 15.156 + - type: map_at_1000 + value: 16.63 + - type: map_at_3 + value: 4.2250000000000005 + - type: map_at_5 + value: 6.078 + - type: mrr_at_1 + value: 30.612000000000002 + - type: mrr_at_10 + value: 45.158 + - type: mrr_at_100 + value: 45.9 + - type: mrr_at_1000 + value: 45.910000000000004 + - type: mrr_at_3 + value: 39.456 + - type: mrr_at_5 + value: 42.925000000000004 + - type: ndcg_at_1 + value: 29.592000000000002 + - type: ndcg_at_10 + value: 25.166 + - type: ndcg_at_100 + value: 35.35 + - type: ndcg_at_1000 + value: 46.67 + - type: ndcg_at_3 + value: 24.545 + - type: ndcg_at_5 + value: 25.112000000000002 + - type: precision_at_1 + value: 30.612000000000002 + - type: precision_at_10 + value: 23.673 + - type: precision_at_100 + value: 7.428999999999999 + - type: precision_at_1000 + value: 1.482 + - type: precision_at_3 + value: 23.810000000000002 + - type: precision_at_5 + value: 25.306 + - type: recall_at_1 + value: 2.299 + - type: recall_at_10 + value: 16.801 + - type: recall_at_100 + value: 45.506 + - type: recall_at_1000 + value: 79.985 + - type: recall_at_3 + value: 5.069 + - type: recall_at_5 + value: 8.863999999999999 + - task: + type: Classification + dataset: + type: mteb/toxic_conversations_50k + name: MTEB ToxicConversationsClassification + config: default + split: test + revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c + metrics: + - type: accuracy + value: 72.1314 + - type: ap + value: 14.605968497007712 + - type: f1 + value: 55.37284214772282 + - task: + type: Classification + dataset: + type: mteb/tweet_sentiment_extraction + name: MTEB TweetSentimentExtractionClassification + config: default + split: test + revision: d604517c81ca91fe16a244d1248fc021f9ecee7a + metrics: + - type: accuracy + value: 61.044142614601014 + - type: f1 + value: 61.30028928459138 + - task: + type: Clustering + dataset: + type: mteb/twentynewsgroups-clustering + name: MTEB TwentyNewsgroupsClustering + config: default + split: test + revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 + metrics: + - type: v_measure + value: 41.28707371610032 + - task: + type: PairClassification + dataset: + type: mteb/twittersemeval2015-pairclassification + name: MTEB TwitterSemEval2015 + config: default + split: test + revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 + metrics: + - type: cos_sim_accuracy + value: 85.09864695714371 + - type: cos_sim_ap + value: 70.63738634684302 + - type: cos_sim_f1 + value: 66.12903225806453 + - type: cos_sim_precision + value: 64.22178020885131 + - type: cos_sim_recall + value: 68.15303430079156 + - type: dot_accuracy + value: 83.59063002920665 + - type: dot_ap + value: 66.68356189934075 + - type: dot_f1 + value: 63.27201851626264 + - type: dot_precision + value: 58.76895225164064 + - type: dot_recall + value: 68.52242744063325 + - type: euclidean_accuracy + value: 85.027120462538 + - type: euclidean_ap + value: 69.99328290454234 + - type: euclidean_f1 + value: 65.23797657612758 + - type: euclidean_precision + value: 61.803588290840416 + - type: euclidean_recall + value: 69.07651715039577 + - type: manhattan_accuracy + value: 85.02115992132086 + - type: manhattan_ap + value: 69.91284274429754 + - type: manhattan_f1 + value: 65.19297407097623 + - type: manhattan_precision + value: 59.5763267088884 + - type: manhattan_recall + value: 71.97889182058047 + - type: max_accuracy + value: 85.09864695714371 + - type: max_ap + value: 70.63738634684302 + - type: max_f1 + value: 66.12903225806453 + - task: + type: PairClassification + dataset: + type: mteb/twitterurlcorpus-pairclassification + name: MTEB TwitterURLCorpus + config: default + split: test + revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf + metrics: + - type: cos_sim_accuracy + value: 89.119804400978 + - type: cos_sim_ap + value: 86.1777422918812 + - type: cos_sim_f1 + value: 78.57841293719444 + - type: cos_sim_precision + value: 76.80488163505366 + - type: cos_sim_recall + value: 80.4357868801971 + - type: dot_accuracy + value: 88.86366282454303 + - type: dot_ap + value: 84.1891332504211 + - type: dot_f1 + value: 78.31691507672025 + - type: dot_precision + value: 74.67700258397933 + - type: dot_recall + value: 82.32984293193716 + - type: euclidean_accuracy + value: 88.74141343578997 + - type: euclidean_ap + value: 85.60421594792011 + - type: euclidean_f1 + value: 77.79556879538262 + - type: euclidean_precision + value: 75.32991995384727 + - type: euclidean_recall + value: 80.42808746535263 + - type: manhattan_accuracy + value: 88.7782822990647 + - type: manhattan_ap + value: 85.61374819166252 + - type: manhattan_f1 + value: 77.78237795927583 + - type: manhattan_precision + value: 76.08423532876813 + - type: manhattan_recall + value: 79.55805358792732 + - type: max_accuracy + value: 89.119804400978 + - type: max_ap + value: 86.1777422918812 + - type: max_f1 + value: 78.57841293719444 + - task: + type: Retrieval + dataset: + type: C-MTEB/VideoRetrieval + name: MTEB VideoRetrieval + config: default + split: dev + revision: None + metrics: + - type: map_at_1 + value: 41.8 + - type: map_at_10 + value: 51.456999999999994 + - type: map_at_100 + value: 52.107000000000006 + - type: map_at_1000 + value: 52.141999999999996 + - type: map_at_3 + value: 48.717 + - type: map_at_5 + value: 50.452 + - type: mrr_at_1 + value: 41.8 + - type: mrr_at_10 + value: 51.441 + - type: mrr_at_100 + value: 52.091 + - type: mrr_at_1000 + value: 52.125 + - type: mrr_at_3 + value: 48.699999999999996 + - type: mrr_at_5 + value: 50.434999999999995 + - type: ndcg_at_1 + value: 41.8 + - type: ndcg_at_10 + value: 56.537000000000006 + - type: ndcg_at_100 + value: 59.901 + - type: ndcg_at_1000 + value: 60.889 + - type: ndcg_at_3 + value: 51.019999999999996 + - type: ndcg_at_5 + value: 54.106 + - type: precision_at_1 + value: 41.8 + - type: precision_at_10 + value: 7.26 + - type: precision_at_100 + value: 0.8880000000000001 + - type: precision_at_1000 + value: 0.097 + - type: precision_at_3 + value: 19.233 + - type: precision_at_5 + value: 13.020000000000001 + - type: recall_at_1 + value: 41.8 + - type: recall_at_10 + value: 72.6 + - type: recall_at_100 + value: 88.8 + - type: recall_at_1000 + value: 96.7 + - type: recall_at_3 + value: 57.699999999999996 + - type: recall_at_5 + value: 65.10000000000001 + - task: + type: Classification + dataset: + type: C-MTEB/waimai-classification + name: MTEB Waimai + config: default + split: test + revision: None + metrics: + - type: accuracy + value: 84.07 + - type: ap + value: 65.23766736490957 + - type: f1 + value: 82.17794239849368 --- + +# Model Card for udever-bloom + + + +`udever-bloom-3b` is finetuned from [bigscience/bloom-3b](https://huggingface.co/bigscience/bloom-3b) via [BitFit](https://aclanthology.org/2022.acl-short.1/) on MS MARCO Passage Ranking, SNLI and MultiNLI data. +It is a universal embedding model across tasks, natural and programming languages. +(From the technical view, `udever` is merely with some minor improvements to `sgpt-bloom`) + +
+ + +## Model Details + +### Model Description + +- **Developed by:** Alibaba Group +- **Model type:** Transformer-based Language Model (decoder-only) +- **Language(s) (NLP):** Multiple; see [bloom training data](https://huggingface.co/bigscience/bloom-3b#training-data) +- **Finetuned from model :** [bigscience/bloom-3b](https://huggingface.co/bigscience/bloom-3b) + +### Model Sources + + + +- **Repository:** [github.com/izhx/uni-rep](https://github.com/izhx/uni-rep) +- **Paper :** [Language Models are Universal Embedders](https://arxiv.org/pdf/2310.08232.pdf) +- **Training Date :** 2023-06 + + + +## How to Get Started with the Model + +Use the code below to get started with the model. + +```python +import torch +from transformers import AutoTokenizer, BloomModel + +tokenizer = AutoTokenizer.from_pretrained('izhx/udever-bloom-3b') +model = BloomModel.from_pretrained('izhx/udever-bloom-3b') + +boq, eoq, bod, eod = '[BOQ]', '[EOQ]', '[BOD]', '[EOD]' +eoq_id, eod_id = tokenizer.convert_tokens_to_ids([eoq, eod]) + +if tokenizer.padding_side != 'left': + print('!!!', tokenizer.padding_side) + tokenizer.padding_side = 'left' + + +def encode(texts: list, is_query: bool = True, max_length=300): + bos = boq if is_query else bod + eos_id = eoq_id if is_query else eod_id + texts = [bos + t for t in texts] + encoding = tokenizer( + texts, truncation=True, max_length=max_length - 1, padding=True + ) + for ids, mask in zip(encoding['input_ids'], encoding['attention_mask']): + ids.append(eos_id) + mask.append(1) + inputs = tokenizer.pad(encoding, return_tensors='pt') + with torch.inference_mode(): + outputs = model(**inputs) + embeds = outputs.last_hidden_state[:, -1] + return embeds + +encode(['I am Bert', 'You are Elmo']) + +``` + +## Training Details + +### Training Data + + + +- MS MARCO Passage Ranking, retrieved by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86) +- SNLI and MultiNLI (https://sbert.net/datasets/AllNLI.tsv.gz) + + +### Training Procedure + + + +#### Preprocessing + +MS MARCO hard negatives provided by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86). +Negatives for SNLI and MultiNLI are randomly sampled. + + +#### Training Hyperparameters + +- **Training regime:** tf32, BitFit +- **Batch size:** 1024 +- **Epochs:** 3 +- **Optimizer:** AdamW +- **Learning rate:** 1e-4 +- **Scheduler:** constant with warmup. +- **Warmup:** 0.25 epoch + + +## Evaluation + +### Table 1: Massive Text Embedding Benchmark [MTEB](https://huggingface.co/spaces/mteb/leaderboard) + +| MTEB | Avg. | Class. | Clust. | PairClass. | Rerank. | Retr. | STS | Summ. | +|-----------------------------|--------------|--------------|--------------|--------------|--------------|--------------|--------------|--------| +| #Datasets ➡️ | 56 | 12 | 11 | 3 | 4 | 15 | 10 | 1 | +|| +| bge-large-en-v1.5 | **64.23** | **75.97** | 46.08| **87.12** | **60.03** | **54.29** | 83.11| 31.61 | +| bge-base-en-v1.5 | 63.55| 75.53| 45.77| 86.55| 58.86| 53.25| 82.4| 31.07 | +| gte-large | 63.13| 73.33| **46.84** | 85| 59.13| 52.22| **83.35** | 31.66 | +| gte-base | 62.39| 73.01| 46.2| 84.57| 58.61| 51.14| 82.3| 31.17 | +| e5-large-v2 | 62.25| 75.24| 44.49| 86.03| 56.61| 50.56| 82.05| 30.19 | +| instructor-xl | 61.79| 73.12| 44.74| 86.62| 57.29| 49.26| 83.06| 32.32 | +| instructor-large | 61.59| 73.86| 45.29| 85.89| 57.54| 47.57| 83.15| 31.84 | +| e5-base-v2 | 61.5 | 73.84| 43.8| 85.73| 55.91| 50.29| 81.05| 30.28 | +| e5-large | 61.42| 73.14| 43.33| 85.94| 56.53| 49.99| 82.06| 30.97 | +| text-embedding-ada-002 (OpenAI API) | 60.99| 70.93| 45.9 | 84.89| 56.32| 49.25| 80.97| 30.8 | +| e5-base | 60.44| 72.63| 42.11| 85.09| 55.7 | 48.75| 80.96| 31.01 | +| SGPT-5.8B-msmarco | 58.93| 68.13| 40.34| 82 | 56.56| 50.25| 78.1 | 31.46 | +| sgpt-bloom-7b1-msmarco | 57.59| 66.19| 38.93| 81.9 | 55.65| 48.22| 77.74| **33.6** | +|| +| Udever-bloom-560m | 55.80| 68.04| 36.89| 81.05| 52.60| 41.19| 79.93| 32.06 | +| Udever-bloom-1b1 | 58.28| 70.18| 39.11| 83.11| 54.28| 45.27| 81.52| 31.10 | +| Udever-bloom-3b | 59.86| 71.91| 40.74| 84.06| 54.90| 47.67| 82.37| 30.62 | +| Udever-bloom-7b1 | 60.63 | 72.13| 40.81| 85.40| 55.91| 49.34| 83.01| 30.97 | + + +### Table 2: [CodeSearchNet](https://github.com/github/CodeSearchNet) + +| CodeSearchNet | Go | Ruby | Python | Java | JS | PHP | Avg. | +|-|-|-|-|-|-|-|-| +| CodeBERT | 69.3 | 70.6 | 84.0 | 86.8 | 74.8 | 70.6 | 76.0 | +| GraphCodeBERT | 84.1 | 73.2 | 87.9 | 75.7 | 71.1 | 72.5 | 77.4 | +| cpt-code S | **97.7** | **86.3** | 99.8 | 94.0 | 86.0 | 96.7 | 93.4 | +| cpt-code M | 97.5 | 85.5 | **99.9** | **94.4** | **86.5** | **97.2** | **93.5** | +| sgpt-bloom-7b1-msmarco | 76.79 | 69.25 | 95.68 | 77.93 | 70.35 | 73.45 | 77.24 | +|| +| Udever-bloom-560m | 75.38 | 66.67 | 96.23 | 78.99 | 69.39 | 73.69 | 76.73 | +| Udever-bloom-1b1 | 78.76 | 72.85 | 97.67 | 82.77 | 74.38 | 78.97 | 80.90 | +| Udever-bloom-3b | 80.63 | 75.40 | 98.02 | 83.88 | 76.18 | 79.67 | 82.29 | +| Udever-bloom-7b1 | 79.37 | 76.59 | 98.38 | 84.68 | 77.49 | 80.03 | 82.76 | + + +### Table 3: Chinese multi-domain retrieval [Multi-cpr](https://dl.acm.org/doi/10.1145/3477495.3531736) + +| | | |E-commerce | | Entertainment video | | Medical | | +|--|--|--|--|--|--|--|--|--| +| Model | Train | Backbone | MRR@10 | Recall@1k | MRR@10 | Recall@1k | MRR@10 | Recall@1k | +|| +| BM25 | - | - | 0.225 | 0.815 | 0.225 | 0.780 | 0.187 | 0.482 | +| Doc2Query | - | - | 0.239 | 0.826 | 0.238 | 0.794 | 0.210 | 0.505 | +| DPR-1 | In-Domain | BERT | 0.270 | 0.921 | 0.254 | 0.934 | 0.327 | 0.747 | +| DPR-2 | In-Domain | BERT-CT | 0.289 | **0.926** | 0.263 | **0.935** | 0.339 | **0.769** | +| text-embedding-ada-002 | General | GPT | 0.183 | 0.825 | 0.159 | 0.786 | 0.245 | 0.593 | +| sgpt-bloom-7b1-msmarco | General | BLOOM | 0.242 | 0.840 | 0.227 | 0.829 | 0.311 | 0.675 | +|| + | Udever-bloom-560m | General | BLOOM | 0.156 | 0.802 | 0.149 | 0.749 | 0.245 | 0.571 | + | Udever-bloom-1b1 | General | BLOOM | 0.244 | 0.863 | 0.208 | 0.815 | 0.241 | 0.557 | + | Udever-bloom-3b | General | BLOOM | 0.267 | 0.871 | 0.228 | 0.836 | 0.288 | 0.619 | + | Udever-bloom-7b1 | General | BLOOM | **0.296** | 0.889 | **0.267** | 0.907 | **0.343** | 0.705 | + +#### More results refer to [paper](https://arxiv.org/pdf/2310.08232.pdf) section 3. + + + +## Technical Specifications + +### Model Architecture and Objective + +- Model: [bigscience/bloom-3b](https://huggingface.co/bigscience/bloom-3b). +- Objective: Constrastive loss with hard negatives (refer to [paper](https://arxiv.org/pdf/2310.08232.pdf) section 2.2). + + +### Compute Infrastructure + +- Nvidia A100 SXM4 80GB. +- torch 2.0.0, transformers 4.29.2. + + +## Citation + +**BibTeX:** + +```BibTeX +@article{zhang2023language, + title={Language Models are Universal Embedders}, + author={Zhang, Xin and Li, Zehan and Zhang, Yanzhao and Long, Dingkun and Xie, Pengjun and Zhang, Meishan and Zhang, Min}, + journal={arXiv preprint arXiv:2310.08232}, + year={2023} +} +```