metadata
license: bigscience-bloom-rail-1.0
language:
- ak
- ar
- as
- bm
- bn
- ca
- code
- en
- es
- eu
- fon
- fr
- gu
- hi
- id
- ig
- ki
- kn
- lg
- ln
- ml
- mr
- ne
- nso
- ny
- or
- pa
- pt
- rn
- rw
- sn
- st
- sw
- ta
- te
- tn
- ts
- tum
- tw
- ur
- vi
- wo
- xh
- yo
- zh
- zhs
- zht
- zu
tags:
- mteb
model-index:
- name: udever-bloom-560m
results:
- task:
type: STS
dataset:
type: C-MTEB/AFQMC
name: MTEB AFQMC
config: default
split: validation
revision: None
metrics:
- type: cos_sim_pearson
value: 25.170024237678657
- type: cos_sim_spearman
value: 25.32025098111752
- type: euclidean_pearson
value: 25.34284673812859
- type: euclidean_spearman
value: 25.52812937004611
- type: manhattan_pearson
value: 25.734179522960822
- type: manhattan_spearman
value: 25.92247507041032
- task:
type: STS
dataset:
type: C-MTEB/ATEC
name: MTEB ATEC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 32.3359541791282
- type: cos_sim_spearman
value: 33.45815274836323
- type: euclidean_pearson
value: 35.14748229440635
- type: euclidean_spearman
value: 33.377829932851334
- type: manhattan_pearson
value: 35.359130773295625
- type: manhattan_spearman
value: 33.524469762932426
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (en)
config: en
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
value: 72.35820895522389
- type: ap
value: 35.45566303125099
- type: f1
value: 66.49474786522534
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (de)
config: de
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
value: 66.423982869379
- type: ap
value: 78.32781372746805
- type: f1
value: 64.24959400774807
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (en-ext)
config: en-ext
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
value: 73.65817091454274
- type: ap
value: 21.73416645163647
- type: f1
value: 60.52120070712094
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (ja)
config: ja
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
value: 56.86295503211991
- type: ap
value: 12.906256075113513
- type: f1
value: 46.68625513679152
- task:
type: Classification
dataset:
type: mteb/amazon_polarity
name: MTEB AmazonPolarityClassification
config: default
split: test
revision: e2d317d38cd51312af73b3d32a06d1a08b442046
metrics:
- type: accuracy
value: 83.8095
- type: ap
value: 78.5195717101614
- type: f1
value: 83.74169093676316
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (en)
config: en
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 38.97
- type: f1
value: 38.57853211177342
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (de)
config: de
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 26.846000000000004
- type: f1
value: 26.473886891677306
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (es)
config: es
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 38.974
- type: f1
value: 38.31719230291287
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (fr)
config: fr
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 38.38799999999999
- type: f1
value: 37.53319978613875
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (ja)
config: ja
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 28.311999999999998
- type: f1
value: 27.988313617729755
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (zh)
config: zh
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 35.704
- type: f1
value: 34.863182924437254
- task:
type: Retrieval
dataset:
type: arguana
name: MTEB ArguAna
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 21.053
- type: map_at_10
value: 35.811
- type: map_at_100
value: 37.035000000000004
- type: map_at_1000
value: 37.055
- type: map_at_3
value: 30.666
- type: map_at_5
value: 33.525
- type: mrr_at_1
value: 21.266
- type: mrr_at_10
value: 35.906
- type: mrr_at_100
value: 37.122
- type: mrr_at_1000
value: 37.141999999999996
- type: mrr_at_3
value: 30.714000000000002
- type: mrr_at_5
value: 33.576
- type: ndcg_at_1
value: 21.053
- type: ndcg_at_10
value: 44.545
- type: ndcg_at_100
value: 49.844
- type: ndcg_at_1000
value: 50.298
- type: ndcg_at_3
value: 33.889
- type: ndcg_at_5
value: 39.059
- type: precision_at_1
value: 21.053
- type: precision_at_10
value: 7.269
- type: precision_at_100
value: 0.96
- type: precision_at_1000
value: 0.099
- type: precision_at_3
value: 14.414
- type: precision_at_5
value: 11.166
- type: recall_at_1
value: 21.053
- type: recall_at_10
value: 72.688
- type: recall_at_100
value: 96.017
- type: recall_at_1000
value: 99.431
- type: recall_at_3
value: 43.242999999999995
- type: recall_at_5
value: 55.832
- task:
type: Clustering
dataset:
type: mteb/arxiv-clustering-p2p
name: MTEB ArxivClusteringP2P
config: default
split: test
revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d
metrics:
- type: v_measure
value: 40.26646269393896
- task:
type: Clustering
dataset:
type: mteb/arxiv-clustering-s2s
name: MTEB ArxivClusteringS2S
config: default
split: test
revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53
metrics:
- type: v_measure
value: 32.00218289816601
- task:
type: Reranking
dataset:
type: mteb/askubuntudupquestions-reranking
name: MTEB AskUbuntuDupQuestions
config: default
split: test
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54
metrics:
- type: map
value: 57.381567373603424
- type: mrr
value: 70.09431473420392
- task:
type: STS
dataset:
type: mteb/biosses-sts
name: MTEB BIOSSES
config: default
split: test
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a
metrics:
- type: cos_sim_pearson
value: 87.14803223261677
- type: cos_sim_spearman
value: 84.43626128689064
- type: euclidean_pearson
value: 85.03130036472703
- type: euclidean_spearman
value: 84.05974668365359
- type: manhattan_pearson
value: 85.59339889467545
- type: manhattan_spearman
value: 83.86938090025696
- task:
type: STS
dataset:
type: C-MTEB/BQ
name: MTEB BQ
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 44.19468290937555
- type: cos_sim_spearman
value: 43.93025426799595
- type: euclidean_pearson
value: 45.273900549350735
- type: euclidean_spearman
value: 45.07419415738924
- type: manhattan_pearson
value: 45.469211385235376
- type: manhattan_spearman
value: 45.27440191151001
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (de-en)
config: de-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 11.440501043841337
- type: f1
value: 11.295895880968951
- type: precision
value: 11.237446950317073
- type: recall
value: 11.440501043841337
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (fr-en)
config: fr-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 96.53312788906008
- type: f1
value: 96.18093770636143
- type: precision
value: 96.00667693888035
- type: recall
value: 96.53312788906008
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (ru-en)
config: ru-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 1.6972635954277795
- type: f1
value: 1.5885146938143124
- type: precision
value: 1.5581125970067466
- type: recall
value: 1.6972635954277795
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (zh-en)
config: zh-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 96.31384939441811
- type: f1
value: 96.15587151132175
- type: precision
value: 96.07688256977357
- type: recall
value: 96.31384939441811
- task:
type: Classification
dataset:
type: mteb/banking77
name: MTEB Banking77Classification
config: default
split: test
revision: 0fd18e25b25c072e09e0d92ab615fda904d66300
metrics:
- type: accuracy
value: 80.97402597402598
- type: f1
value: 80.88177660652944
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-p2p
name: MTEB BiorxivClusteringP2P
config: default
split: test
revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40
metrics:
- type: v_measure
value: 33.266950159712465
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-s2s
name: MTEB BiorxivClusteringS2S
config: default
split: test
revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908
metrics:
- type: v_measure
value: 28.65092446021672
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringP2P
name: MTEB CLSClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 35.21075820650184
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringS2S
name: MTEB CLSClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 35.121931960714484
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv1-reranking
name: MTEB CMedQAv1
config: default
split: test
revision: None
metrics:
- type: map
value: 63.41256934884578
- type: mrr
value: 68.6492857142857
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv2-reranking
name: MTEB CMedQAv2
config: default
split: test
revision: None
metrics:
- type: map
value: 63.663067375541104
- type: mrr
value: 68.92075396825396
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackAndroidRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 24.997
- type: map_at_10
value: 35.477
- type: map_at_100
value: 36.722
- type: map_at_1000
value: 36.849
- type: map_at_3
value: 32.083
- type: map_at_5
value: 33.884
- type: mrr_at_1
value: 32.046
- type: mrr_at_10
value: 41.455999999999996
- type: mrr_at_100
value: 42.214
- type: mrr_at_1000
value: 42.268
- type: mrr_at_3
value: 38.722
- type: mrr_at_5
value: 40.266999999999996
- type: ndcg_at_1
value: 32.046
- type: ndcg_at_10
value: 41.705999999999996
- type: ndcg_at_100
value: 46.695
- type: ndcg_at_1000
value: 49.128
- type: ndcg_at_3
value: 36.6
- type: ndcg_at_5
value: 38.725
- type: precision_at_1
value: 32.046
- type: precision_at_10
value: 8.197000000000001
- type: precision_at_100
value: 1.323
- type: precision_at_1000
value: 0.183
- type: precision_at_3
value: 18.073
- type: precision_at_5
value: 13.047
- type: recall_at_1
value: 24.997
- type: recall_at_10
value: 54.013
- type: recall_at_100
value: 75.29400000000001
- type: recall_at_1000
value: 91.611
- type: recall_at_3
value: 38.627
- type: recall_at_5
value: 45.019999999999996
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackEnglishRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 23.194
- type: map_at_10
value: 30.076000000000004
- type: map_at_100
value: 31
- type: map_at_1000
value: 31.125999999999998
- type: map_at_3
value: 28.137
- type: map_at_5
value: 29.206
- type: mrr_at_1
value: 28.535
- type: mrr_at_10
value: 34.833999999999996
- type: mrr_at_100
value: 35.504999999999995
- type: mrr_at_1000
value: 35.57
- type: mrr_at_3
value: 33.089
- type: mrr_at_5
value: 34.115
- type: ndcg_at_1
value: 28.535
- type: ndcg_at_10
value: 34.285
- type: ndcg_at_100
value: 38.286
- type: ndcg_at_1000
value: 41.007
- type: ndcg_at_3
value: 31.395
- type: ndcg_at_5
value: 32.687
- type: precision_at_1
value: 28.535
- type: precision_at_10
value: 6.166
- type: precision_at_100
value: 1.042
- type: precision_at_1000
value: 0.155
- type: precision_at_3
value: 14.862
- type: precision_at_5
value: 10.331
- type: recall_at_1
value: 23.194
- type: recall_at_10
value: 41.648
- type: recall_at_100
value: 58.999
- type: recall_at_1000
value: 77.46300000000001
- type: recall_at_3
value: 32.931
- type: recall_at_5
value: 36.736999999999995
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGamingRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 31.899
- type: map_at_10
value: 42.657000000000004
- type: map_at_100
value: 43.717
- type: map_at_1000
value: 43.79
- type: map_at_3
value: 39.635
- type: map_at_5
value: 41.538000000000004
- type: mrr_at_1
value: 36.864999999999995
- type: mrr_at_10
value: 46.137
- type: mrr_at_100
value: 46.946
- type: mrr_at_1000
value: 46.986
- type: mrr_at_3
value: 43.469
- type: mrr_at_5
value: 45.262
- type: ndcg_at_1
value: 36.864999999999995
- type: ndcg_at_10
value: 48.164
- type: ndcg_at_100
value: 52.769999999999996
- type: ndcg_at_1000
value: 54.393
- type: ndcg_at_3
value: 42.887
- type: ndcg_at_5
value: 45.871
- type: precision_at_1
value: 36.864999999999995
- type: precision_at_10
value: 7.843
- type: precision_at_100
value: 1.102
- type: precision_at_1000
value: 0.13
- type: precision_at_3
value: 19.352
- type: precision_at_5
value: 13.618
- type: recall_at_1
value: 31.899
- type: recall_at_10
value: 61.131
- type: recall_at_100
value: 81.504
- type: recall_at_1000
value: 93.146
- type: recall_at_3
value: 46.971000000000004
- type: recall_at_5
value: 54.42399999999999
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGisRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 17.621000000000002
- type: map_at_10
value: 23.621
- type: map_at_100
value: 24.636
- type: map_at_1000
value: 24.739
- type: map_at_3
value: 21.623
- type: map_at_5
value: 22.511
- type: mrr_at_1
value: 19.096
- type: mrr_at_10
value: 25.288
- type: mrr_at_100
value: 26.238
- type: mrr_at_1000
value: 26.314
- type: mrr_at_3
value: 23.202
- type: mrr_at_5
value: 24.213
- type: ndcg_at_1
value: 19.096
- type: ndcg_at_10
value: 27.529999999999998
- type: ndcg_at_100
value: 32.763
- type: ndcg_at_1000
value: 35.538
- type: ndcg_at_3
value: 23.362
- type: ndcg_at_5
value: 24.961
- type: precision_at_1
value: 19.096
- type: precision_at_10
value: 4.417999999999999
- type: precision_at_100
value: 0.739
- type: precision_at_1000
value: 0.10300000000000001
- type: precision_at_3
value: 9.981
- type: precision_at_5
value: 6.959999999999999
- type: recall_at_1
value: 17.621000000000002
- type: recall_at_10
value: 38.079
- type: recall_at_100
value: 62.499
- type: recall_at_1000
value: 83.783
- type: recall_at_3
value: 26.687
- type: recall_at_5
value: 30.459000000000003
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackMathematicaRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 11.019
- type: map_at_10
value: 15.869
- type: map_at_100
value: 17.078
- type: map_at_1000
value: 17.205000000000002
- type: map_at_3
value: 13.794
- type: map_at_5
value: 14.814
- type: mrr_at_1
value: 13.930000000000001
- type: mrr_at_10
value: 19.172
- type: mrr_at_100
value: 20.325
- type: mrr_at_1000
value: 20.415
- type: mrr_at_3
value: 17.122999999999998
- type: mrr_at_5
value: 18.124000000000002
- type: ndcg_at_1
value: 13.930000000000001
- type: ndcg_at_10
value: 19.646
- type: ndcg_at_100
value: 25.684
- type: ndcg_at_1000
value: 29.14
- type: ndcg_at_3
value: 15.614
- type: ndcg_at_5
value: 17.247
- type: precision_at_1
value: 13.930000000000001
- type: precision_at_10
value: 3.868
- type: precision_at_100
value: 0.8
- type: precision_at_1000
value: 0.125
- type: precision_at_3
value: 7.420999999999999
- type: precision_at_5
value: 5.672
- type: recall_at_1
value: 11.019
- type: recall_at_10
value: 28.116000000000003
- type: recall_at_100
value: 54.794
- type: recall_at_1000
value: 79.838
- type: recall_at_3
value: 17.124
- type: recall_at_5
value: 21.086
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackPhysicsRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 24.791
- type: map_at_10
value: 33.442
- type: map_at_100
value: 34.719
- type: map_at_1000
value: 34.849000000000004
- type: map_at_3
value: 30.885
- type: map_at_5
value: 32.245000000000005
- type: mrr_at_1
value: 30.606
- type: mrr_at_10
value: 38.922000000000004
- type: mrr_at_100
value: 39.822
- type: mrr_at_1000
value: 39.881
- type: mrr_at_3
value: 36.622
- type: mrr_at_5
value: 37.907000000000004
- type: ndcg_at_1
value: 30.606
- type: ndcg_at_10
value: 38.867000000000004
- type: ndcg_at_100
value: 44.364
- type: ndcg_at_1000
value: 47.073
- type: ndcg_at_3
value: 34.63
- type: ndcg_at_5
value: 36.479
- type: precision_at_1
value: 30.606
- type: precision_at_10
value: 7.0360000000000005
- type: precision_at_100
value: 1.174
- type: precision_at_1000
value: 0.16
- type: precision_at_3
value: 16.522000000000002
- type: precision_at_5
value: 11.588
- type: recall_at_1
value: 24.791
- type: recall_at_10
value: 49.736000000000004
- type: recall_at_100
value: 72.67099999999999
- type: recall_at_1000
value: 91.29599999999999
- type: recall_at_3
value: 37.345
- type: recall_at_5
value: 42.400999999999996
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackProgrammersRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 20.669999999999998
- type: map_at_10
value: 28.605000000000004
- type: map_at_100
value: 29.769000000000002
- type: map_at_1000
value: 29.881999999999998
- type: map_at_3
value: 25.886
- type: map_at_5
value: 27.317999999999998
- type: mrr_at_1
value: 25.457
- type: mrr_at_10
value: 33.423
- type: mrr_at_100
value: 34.269
- type: mrr_at_1000
value: 34.336
- type: mrr_at_3
value: 30.974
- type: mrr_at_5
value: 32.23
- type: ndcg_at_1
value: 25.457
- type: ndcg_at_10
value: 33.785
- type: ndcg_at_100
value: 39.145
- type: ndcg_at_1000
value: 41.772
- type: ndcg_at_3
value: 29.014
- type: ndcg_at_5
value: 31.019999999999996
- type: precision_at_1
value: 25.457
- type: precision_at_10
value: 6.2330000000000005
- type: precision_at_100
value: 1.045
- type: precision_at_1000
value: 0.145
- type: precision_at_3
value: 13.813
- type: precision_at_5
value: 9.863
- type: recall_at_1
value: 20.669999999999998
- type: recall_at_10
value: 44.651
- type: recall_at_100
value: 68.037
- type: recall_at_1000
value: 86.282
- type: recall_at_3
value: 31.381999999999998
- type: recall_at_5
value: 36.778
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 19.796583333333338
- type: map_at_10
value: 26.900166666666664
- type: map_at_100
value: 27.956583333333334
- type: map_at_1000
value: 28.08083333333333
- type: map_at_3
value: 24.598416666666665
- type: map_at_5
value: 25.81791666666667
- type: mrr_at_1
value: 23.68591666666667
- type: mrr_at_10
value: 30.65558333333333
- type: mrr_at_100
value: 31.503583333333335
- type: mrr_at_1000
value: 31.576083333333333
- type: mrr_at_3
value: 28.50525
- type: mrr_at_5
value: 29.690666666666665
- type: ndcg_at_1
value: 23.68591666666667
- type: ndcg_at_10
value: 31.425000000000004
- type: ndcg_at_100
value: 36.34316666666666
- type: ndcg_at_1000
value: 39.164249999999996
- type: ndcg_at_3
value: 27.330083333333338
- type: ndcg_at_5
value: 29.14408333333333
- type: precision_at_1
value: 23.68591666666667
- type: precision_at_10
value: 5.5862500000000015
- type: precision_at_100
value: 0.9571666666666666
- type: precision_at_1000
value: 0.13866666666666666
- type: precision_at_3
value: 12.663499999999999
- type: precision_at_5
value: 9.035333333333332
- type: recall_at_1
value: 19.796583333333338
- type: recall_at_10
value: 41.289416666666675
- type: recall_at_100
value: 63.251250000000006
- type: recall_at_1000
value: 83.4515
- type: recall_at_3
value: 29.727916666666665
- type: recall_at_5
value: 34.45824999999999
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackStatsRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 16.121
- type: map_at_10
value: 22.104
- type: map_at_100
value: 23.003
- type: map_at_1000
value: 23.108
- type: map_at_3
value: 20.233
- type: map_at_5
value: 21.186
- type: mrr_at_1
value: 18.865000000000002
- type: mrr_at_10
value: 24.951
- type: mrr_at_100
value: 25.779000000000003
- type: mrr_at_1000
value: 25.863999999999997
- type: mrr_at_3
value: 23.083000000000002
- type: mrr_at_5
value: 24.049
- type: ndcg_at_1
value: 18.865000000000002
- type: ndcg_at_10
value: 26.031
- type: ndcg_at_100
value: 30.589
- type: ndcg_at_1000
value: 33.565
- type: ndcg_at_3
value: 22.369
- type: ndcg_at_5
value: 23.932000000000002
- type: precision_at_1
value: 18.865000000000002
- type: precision_at_10
value: 4.324999999999999
- type: precision_at_100
value: 0.722
- type: precision_at_1000
value: 0.104
- type: precision_at_3
value: 10.072000000000001
- type: precision_at_5
value: 7.086
- type: recall_at_1
value: 16.121
- type: recall_at_10
value: 35.577
- type: recall_at_100
value: 56.298
- type: recall_at_1000
value: 79.089
- type: recall_at_3
value: 25.239
- type: recall_at_5
value: 29.242
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackTexRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 10.968
- type: map_at_10
value: 15.639
- type: map_at_100
value: 16.459
- type: map_at_1000
value: 16.584
- type: map_at_3
value: 14.127
- type: map_at_5
value: 14.911
- type: mrr_at_1
value: 13.73
- type: mrr_at_10
value: 18.822
- type: mrr_at_100
value: 19.592000000000002
- type: mrr_at_1000
value: 19.683999999999997
- type: mrr_at_3
value: 17.223
- type: mrr_at_5
value: 18.082
- type: ndcg_at_1
value: 13.73
- type: ndcg_at_10
value: 18.881999999999998
- type: ndcg_at_100
value: 23.182
- type: ndcg_at_1000
value: 26.479000000000003
- type: ndcg_at_3
value: 16.067999999999998
- type: ndcg_at_5
value: 17.265
- type: precision_at_1
value: 13.73
- type: precision_at_10
value: 3.544
- type: precision_at_100
value: 0.679
- type: precision_at_1000
value: 0.11199999999999999
- type: precision_at_3
value: 7.674
- type: precision_at_5
value: 5.561
- type: recall_at_1
value: 10.968
- type: recall_at_10
value: 25.596000000000004
- type: recall_at_100
value: 45.411
- type: recall_at_1000
value: 69.555
- type: recall_at_3
value: 17.582
- type: recall_at_5
value: 20.785
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackUnixRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 20.886
- type: map_at_10
value: 27.029999999999998
- type: map_at_100
value: 27.968
- type: map_at_1000
value: 28.108
- type: map_at_3
value: 25.001
- type: map_at_5
value: 26.185000000000002
- type: mrr_at_1
value: 24.067
- type: mrr_at_10
value: 30.756
- type: mrr_at_100
value: 31.593
- type: mrr_at_1000
value: 31.685999999999996
- type: mrr_at_3
value: 28.793999999999997
- type: mrr_at_5
value: 29.997
- type: ndcg_at_1
value: 24.067
- type: ndcg_at_10
value: 31.095
- type: ndcg_at_100
value: 35.893
- type: ndcg_at_1000
value: 39.158
- type: ndcg_at_3
value: 27.321
- type: ndcg_at_5
value: 29.247
- type: precision_at_1
value: 24.067
- type: precision_at_10
value: 5.103
- type: precision_at_100
value: 0.8460000000000001
- type: precision_at_1000
value: 0.125
- type: precision_at_3
value: 12.065
- type: precision_at_5
value: 8.601
- type: recall_at_1
value: 20.886
- type: recall_at_10
value: 39.797
- type: recall_at_100
value: 61.399
- type: recall_at_1000
value: 84.555
- type: recall_at_3
value: 29.721999999999998
- type: recall_at_5
value: 34.455999999999996
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackWebmastersRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 21.394
- type: map_at_10
value: 28.303
- type: map_at_100
value: 29.726000000000003
- type: map_at_1000
value: 29.955
- type: map_at_3
value: 25.705
- type: map_at_5
value: 26.989
- type: mrr_at_1
value: 25.691999999999997
- type: mrr_at_10
value: 32.495000000000005
- type: mrr_at_100
value: 33.461999999999996
- type: mrr_at_1000
value: 33.534000000000006
- type: mrr_at_3
value: 30.137999999999998
- type: mrr_at_5
value: 31.383
- type: ndcg_at_1
value: 25.691999999999997
- type: ndcg_at_10
value: 33.300000000000004
- type: ndcg_at_100
value: 39.062000000000005
- type: ndcg_at_1000
value: 42.176
- type: ndcg_at_3
value: 28.859
- type: ndcg_at_5
value: 30.805
- type: precision_at_1
value: 25.691999999999997
- type: precision_at_10
value: 6.383
- type: precision_at_100
value: 1.387
- type: precision_at_1000
value: 0.22899999999999998
- type: precision_at_3
value: 13.439
- type: precision_at_5
value: 9.959999999999999
- type: recall_at_1
value: 21.394
- type: recall_at_10
value: 42.853
- type: recall_at_100
value: 69.284
- type: recall_at_1000
value: 89.646
- type: recall_at_3
value: 29.786
- type: recall_at_5
value: 34.797
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackWordpressRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 13.999
- type: map_at_10
value: 19.979
- type: map_at_100
value: 20.682000000000002
- type: map_at_1000
value: 20.775
- type: map_at_3
value: 18.072
- type: map_at_5
value: 19.028
- type: mrr_at_1
value: 15.342
- type: mrr_at_10
value: 21.611
- type: mrr_at_100
value: 22.298000000000002
- type: mrr_at_1000
value: 22.375
- type: mrr_at_3
value: 19.624
- type: mrr_at_5
value: 20.659
- type: ndcg_at_1
value: 15.342
- type: ndcg_at_10
value: 23.809
- type: ndcg_at_100
value: 27.685
- type: ndcg_at_1000
value: 30.542
- type: ndcg_at_3
value: 19.842000000000002
- type: ndcg_at_5
value: 21.490000000000002
- type: precision_at_1
value: 15.342
- type: precision_at_10
value: 3.9190000000000005
- type: precision_at_100
value: 0.627
- type: precision_at_1000
value: 0.093
- type: precision_at_3
value: 8.688
- type: precision_at_5
value: 6.1370000000000005
- type: recall_at_1
value: 13.999
- type: recall_at_10
value: 34.276
- type: recall_at_100
value: 52.825
- type: recall_at_1000
value: 75.154
- type: recall_at_3
value: 23.339
- type: recall_at_5
value: 27.314
- task:
type: Retrieval
dataset:
type: climate-fever
name: MTEB ClimateFEVER
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 8.27
- type: map_at_10
value: 14.161999999999999
- type: map_at_100
value: 15.775
- type: map_at_1000
value: 15.947
- type: map_at_3
value: 11.701
- type: map_at_5
value: 12.952
- type: mrr_at_1
value: 18.632
- type: mrr_at_10
value: 28.871000000000002
- type: mrr_at_100
value: 29.985
- type: mrr_at_1000
value: 30.037999999999997
- type: mrr_at_3
value: 25.451
- type: mrr_at_5
value: 27.366
- type: ndcg_at_1
value: 18.632
- type: ndcg_at_10
value: 21.017
- type: ndcg_at_100
value: 28.022999999999996
- type: ndcg_at_1000
value: 31.518
- type: ndcg_at_3
value: 16.611
- type: ndcg_at_5
value: 18.149
- type: precision_at_1
value: 18.632
- type: precision_at_10
value: 6.736000000000001
- type: precision_at_100
value: 1.414
- type: precision_at_1000
value: 0.20600000000000002
- type: precision_at_3
value: 12.313
- type: precision_at_5
value: 9.759
- type: recall_at_1
value: 8.27
- type: recall_at_10
value: 26.218999999999998
- type: recall_at_100
value: 50.77
- type: recall_at_1000
value: 70.8
- type: recall_at_3
value: 15.526000000000002
- type: recall_at_5
value: 19.724
- task:
type: Retrieval
dataset:
type: C-MTEB/CmedqaRetrieval
name: MTEB CmedqaRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 10.598
- type: map_at_10
value: 15.869
- type: map_at_100
value: 17.081
- type: map_at_1000
value: 17.267
- type: map_at_3
value: 13.877
- type: map_at_5
value: 14.884
- type: mrr_at_1
value: 17.279
- type: mrr_at_10
value: 22.554
- type: mrr_at_100
value: 23.521
- type: mrr_at_1000
value: 23.619
- type: mrr_at_3
value: 20.647
- type: mrr_at_5
value: 21.625
- type: ndcg_at_1
value: 17.279
- type: ndcg_at_10
value: 20.029
- type: ndcg_at_100
value: 25.968000000000004
- type: ndcg_at_1000
value: 30.158
- type: ndcg_at_3
value: 16.947000000000003
- type: ndcg_at_5
value: 18.069
- type: precision_at_1
value: 17.279
- type: precision_at_10
value: 4.704
- type: precision_at_100
value: 0.9690000000000001
- type: precision_at_1000
value: 0.152
- type: precision_at_3
value: 9.777
- type: precision_at_5
value: 7.207
- type: recall_at_1
value: 10.598
- type: recall_at_10
value: 26.034000000000002
- type: recall_at_100
value: 51.385999999999996
- type: recall_at_1000
value: 80.49
- type: recall_at_3
value: 16.834
- type: recall_at_5
value: 20.317
- task:
type: PairClassification
dataset:
type: C-MTEB/CMNLI
name: MTEB Cmnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 70.40288634996993
- type: cos_sim_ap
value: 78.43387766087626
- type: cos_sim_f1
value: 73.09982840415867
- type: cos_sim_precision
value: 64.31616341030195
- type: cos_sim_recall
value: 84.66214636427402
- type: dot_accuracy
value: 65.52014431749849
- type: dot_ap
value: 70.89507344960353
- type: dot_f1
value: 70.7030509759333
- type: dot_precision
value: 59.43922255854708
- type: dot_recall
value: 87.2340425531915
- type: euclidean_accuracy
value: 69.84966927239927
- type: euclidean_ap
value: 78.08825177727368
- type: euclidean_f1
value: 72.68394399761692
- type: euclidean_precision
value: 63.16879530548844
- type: euclidean_recall
value: 85.57400046761748
- type: manhattan_accuracy
value: 69.9579073962718
- type: manhattan_ap
value: 78.38355697667261
- type: manhattan_f1
value: 73.06507508663844
- type: manhattan_precision
value: 62.10112911143839
- type: manhattan_recall
value: 88.73041851765257
- type: max_accuracy
value: 70.40288634996993
- type: max_ap
value: 78.43387766087626
- type: max_f1
value: 73.09982840415867
- task:
type: Retrieval
dataset:
type: C-MTEB/CovidRetrieval
name: MTEB CovidRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 23.973
- type: map_at_10
value: 30.074
- type: map_at_100
value: 31.05
- type: map_at_1000
value: 31.147000000000002
- type: map_at_3
value: 27.977
- type: map_at_5
value: 29.247
- type: mrr_at_1
value: 24.025
- type: mrr_at_10
value: 30.093999999999998
- type: mrr_at_100
value: 31.068
- type: mrr_at_1000
value: 31.165
- type: mrr_at_3
value: 27.994000000000003
- type: mrr_at_5
value: 29.243000000000002
- type: ndcg_at_1
value: 24.025
- type: ndcg_at_10
value: 33.566
- type: ndcg_at_100
value: 38.818999999999996
- type: ndcg_at_1000
value: 41.477000000000004
- type: ndcg_at_3
value: 29.293000000000003
- type: ndcg_at_5
value: 31.564999999999998
- type: precision_at_1
value: 24.025
- type: precision_at_10
value: 4.489
- type: precision_at_100
value: 0.709
- type: precision_at_1000
value: 0.092
- type: precision_at_3
value: 11.064
- type: precision_at_5
value: 7.734000000000001
- type: recall_at_1
value: 23.973
- type: recall_at_10
value: 44.731
- type: recall_at_100
value: 70.52199999999999
- type: recall_at_1000
value: 91.491
- type: recall_at_3
value: 33.087
- type: recall_at_5
value: 38.567
- task:
type: Retrieval
dataset:
type: dbpedia-entity
name: MTEB DBPedia
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 6.950000000000001
- type: map_at_10
value: 13.236999999999998
- type: map_at_100
value: 16.137
- type: map_at_1000
value: 16.785
- type: map_at_3
value: 10.378
- type: map_at_5
value: 11.62
- type: mrr_at_1
value: 54
- type: mrr_at_10
value: 61.861
- type: mrr_at_100
value: 62.436
- type: mrr_at_1000
value: 62.456
- type: mrr_at_3
value: 60.458
- type: mrr_at_5
value: 61.208
- type: ndcg_at_1
value: 43.75
- type: ndcg_at_10
value: 28.224
- type: ndcg_at_100
value: 29.244999999999997
- type: ndcg_at_1000
value: 34.410000000000004
- type: ndcg_at_3
value: 33.955
- type: ndcg_at_5
value: 30.597
- type: precision_at_1
value: 54
- type: precision_at_10
value: 20.825
- type: precision_at_100
value: 5.462
- type: precision_at_1000
value: 1.1320000000000001
- type: precision_at_3
value: 37
- type: precision_at_5
value: 28.849999999999998
- type: recall_at_1
value: 6.950000000000001
- type: recall_at_10
value: 17.159
- type: recall_at_100
value: 31.657999999999998
- type: recall_at_1000
value: 49.155
- type: recall_at_3
value: 11.393
- type: recall_at_5
value: 13.568
- task:
type: Retrieval
dataset:
type: C-MTEB/DuRetrieval
name: MTEB DuRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 16.333000000000002
- type: map_at_10
value: 44.080999999999996
- type: map_at_100
value: 47.958
- type: map_at_1000
value: 48.183
- type: map_at_3
value: 31.468
- type: map_at_5
value: 38.213
- type: mrr_at_1
value: 63
- type: mrr_at_10
value: 72.006
- type: mrr_at_100
value: 72.299
- type: mrr_at_1000
value: 72.313
- type: mrr_at_3
value: 70.375
- type: mrr_at_5
value: 71.33
- type: ndcg_at_1
value: 63
- type: ndcg_at_10
value: 56.044000000000004
- type: ndcg_at_100
value: 63.629999999999995
- type: ndcg_at_1000
value: 66.156
- type: ndcg_at_3
value: 55.85
- type: ndcg_at_5
value: 53.559
- type: precision_at_1
value: 63
- type: precision_at_10
value: 27.279999999999998
- type: precision_at_100
value: 4.005
- type: precision_at_1000
value: 0.462
- type: precision_at_3
value: 49.633
- type: precision_at_5
value: 40.6
- type: recall_at_1
value: 16.333000000000002
- type: recall_at_10
value: 57.152
- type: recall_at_100
value: 80.231
- type: recall_at_1000
value: 92.95400000000001
- type: recall_at_3
value: 34.793
- type: recall_at_5
value: 44.989000000000004
- task:
type: Retrieval
dataset:
type: C-MTEB/EcomRetrieval
name: MTEB EcomRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 33.7
- type: map_at_10
value: 42.327999999999996
- type: map_at_100
value: 43.230000000000004
- type: map_at_1000
value: 43.274
- type: map_at_3
value: 39.883
- type: map_at_5
value: 41.178
- type: mrr_at_1
value: 33.7
- type: mrr_at_10
value: 42.327999999999996
- type: mrr_at_100
value: 43.230000000000004
- type: mrr_at_1000
value: 43.274
- type: mrr_at_3
value: 39.883
- type: mrr_at_5
value: 41.178
- type: ndcg_at_1
value: 33.7
- type: ndcg_at_10
value: 46.996
- type: ndcg_at_100
value: 51.629000000000005
- type: ndcg_at_1000
value: 52.823
- type: ndcg_at_3
value: 41.891
- type: ndcg_at_5
value: 44.232
- type: precision_at_1
value: 33.7
- type: precision_at_10
value: 6.1899999999999995
- type: precision_at_100
value: 0.8410000000000001
- type: precision_at_1000
value: 0.094
- type: precision_at_3
value: 15.9
- type: precision_at_5
value: 10.68
- type: recall_at_1
value: 33.7
- type: recall_at_10
value: 61.9
- type: recall_at_100
value: 84.1
- type: recall_at_1000
value: 93.60000000000001
- type: recall_at_3
value: 47.699999999999996
- type: recall_at_5
value: 53.400000000000006
- task:
type: Classification
dataset:
type: mteb/emotion
name: MTEB EmotionClassification
config: default
split: test
revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37
metrics:
- type: accuracy
value: 44.76500000000001
- type: f1
value: 40.46330006682868
- task:
type: Retrieval
dataset:
type: fever
name: MTEB FEVER
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 45.078
- type: map_at_10
value: 55.443
- type: map_at_100
value: 56.03900000000001
- type: map_at_1000
value: 56.067
- type: map_at_3
value: 53.174
- type: map_at_5
value: 54.510999999999996
- type: mrr_at_1
value: 48.575
- type: mrr_at_10
value: 59.194
- type: mrr_at_100
value: 59.760999999999996
- type: mrr_at_1000
value: 59.784000000000006
- type: mrr_at_3
value: 56.896
- type: mrr_at_5
value: 58.282000000000004
- type: ndcg_at_1
value: 48.575
- type: ndcg_at_10
value: 61.096
- type: ndcg_at_100
value: 63.94800000000001
- type: ndcg_at_1000
value: 64.68199999999999
- type: ndcg_at_3
value: 56.58
- type: ndcg_at_5
value: 58.928000000000004
- type: precision_at_1
value: 48.575
- type: precision_at_10
value: 8.18
- type: precision_at_100
value: 0.968
- type: precision_at_1000
value: 0.104
- type: precision_at_3
value: 22.662
- type: precision_at_5
value: 14.881
- type: recall_at_1
value: 45.078
- type: recall_at_10
value: 75.057
- type: recall_at_100
value: 88.05199999999999
- type: recall_at_1000
value: 93.58999999999999
- type: recall_at_3
value: 62.77700000000001
- type: recall_at_5
value: 68.50699999999999
- task:
type: Retrieval
dataset:
type: fiqa
name: MTEB FiQA2018
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 11.097999999999999
- type: map_at_10
value: 18.288
- type: map_at_100
value: 19.903000000000002
- type: map_at_1000
value: 20.108
- type: map_at_3
value: 15.576
- type: map_at_5
value: 16.997999999999998
- type: mrr_at_1
value: 23.302
- type: mrr_at_10
value: 30.978
- type: mrr_at_100
value: 32.072
- type: mrr_at_1000
value: 32.15
- type: mrr_at_3
value: 28.549000000000003
- type: mrr_at_5
value: 29.931
- type: ndcg_at_1
value: 23.302
- type: ndcg_at_10
value: 24.488
- type: ndcg_at_100
value: 31.052999999999997
- type: ndcg_at_1000
value: 35.124
- type: ndcg_at_3
value: 21.215999999999998
- type: ndcg_at_5
value: 22.314999999999998
- type: precision_at_1
value: 23.302
- type: precision_at_10
value: 7.13
- type: precision_at_100
value: 1.3559999999999999
- type: precision_at_1000
value: 0.20600000000000002
- type: precision_at_3
value: 14.198
- type: precision_at_5
value: 10.895000000000001
- type: recall_at_1
value: 11.097999999999999
- type: recall_at_10
value: 30.352
- type: recall_at_100
value: 54.937999999999995
- type: recall_at_1000
value: 79.586
- type: recall_at_3
value: 19.486
- type: recall_at_5
value: 23.860999999999997
- task:
type: Retrieval
dataset:
type: hotpotqa
name: MTEB HotpotQA
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 28.325
- type: map_at_10
value: 37.305
- type: map_at_100
value: 38
- type: map_at_1000
value: 38.065
- type: map_at_3
value: 35.219
- type: map_at_5
value: 36.466
- type: mrr_at_1
value: 56.650999999999996
- type: mrr_at_10
value: 63.574
- type: mrr_at_100
value: 63.966
- type: mrr_at_1000
value: 63.992000000000004
- type: mrr_at_3
value: 62.107
- type: mrr_at_5
value: 62.976
- type: ndcg_at_1
value: 56.650999999999996
- type: ndcg_at_10
value: 46.046
- type: ndcg_at_100
value: 48.916
- type: ndcg_at_1000
value: 50.410999999999994
- type: ndcg_at_3
value: 42.516999999999996
- type: ndcg_at_5
value: 44.374
- type: precision_at_1
value: 56.650999999999996
- type: precision_at_10
value: 9.392
- type: precision_at_100
value: 1.166
- type: precision_at_1000
value: 0.13699999999999998
- type: precision_at_3
value: 26.068
- type: precision_at_5
value: 17.11
- type: recall_at_1
value: 28.325
- type: recall_at_10
value: 46.961999999999996
- type: recall_at_100
value: 58.318999999999996
- type: recall_at_1000
value: 68.298
- type: recall_at_3
value: 39.102
- type: recall_at_5
value: 42.775
- task:
type: Classification
dataset:
type: C-MTEB/IFlyTek-classification
name: MTEB IFlyTek
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 40.461716044632546
- type: f1
value: 33.890745966734315
- task:
type: Classification
dataset:
type: mteb/imdb
name: MTEB ImdbClassification
config: default
split: test
revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7
metrics:
- type: accuracy
value: 72.21000000000001
- type: ap
value: 66.59963731769069
- type: f1
value: 71.97616824840041
- task:
type: Classification
dataset:
type: C-MTEB/JDReview-classification
name: MTEB JDReview
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 78.25515947467167
- type: ap
value: 38.265118237185064
- type: f1
value: 70.73962826410575
- task:
type: STS
dataset:
type: C-MTEB/LCQMC
name: MTEB LCQMC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 63.98362797180168
- type: cos_sim_spearman
value: 71.97575564053473
- type: euclidean_pearson
value: 70.56052438394708
- type: euclidean_spearman
value: 72.48267176371337
- type: manhattan_pearson
value: 70.7156268448442
- type: manhattan_spearman
value: 72.61065396802094
- task:
type: Retrieval
dataset:
type: C-MTEB/MMarcoRetrieval
name: MTEB MMarcoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 55.775
- type: map_at_10
value: 65.074
- type: map_at_100
value: 65.596
- type: map_at_1000
value: 65.618
- type: map_at_3
value: 62.92
- type: map_at_5
value: 64.277
- type: mrr_at_1
value: 57.708000000000006
- type: mrr_at_10
value: 65.824
- type: mrr_at_100
value: 66.286
- type: mrr_at_1000
value: 66.306
- type: mrr_at_3
value: 63.871
- type: mrr_at_5
value: 65.093
- type: ndcg_at_1
value: 57.708000000000006
- type: ndcg_at_10
value: 69.309
- type: ndcg_at_100
value: 71.723
- type: ndcg_at_1000
value: 72.313
- type: ndcg_at_3
value: 65.134
- type: ndcg_at_5
value: 67.476
- type: precision_at_1
value: 57.708000000000006
- type: precision_at_10
value: 8.668
- type: precision_at_100
value: 0.989
- type: precision_at_1000
value: 0.104
- type: precision_at_3
value: 24.837999999999997
- type: precision_at_5
value: 16.128999999999998
- type: recall_at_1
value: 55.775
- type: recall_at_10
value: 81.702
- type: recall_at_100
value: 92.785
- type: recall_at_1000
value: 97.425
- type: recall_at_3
value: 70.587
- type: recall_at_5
value: 76.199
- task:
type: Retrieval
dataset:
type: msmarco
name: MTEB MSMARCO
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 17.771
- type: map_at_10
value: 28.16
- type: map_at_100
value: 29.363
- type: map_at_1000
value: 29.431
- type: map_at_3
value: 24.767
- type: map_at_5
value: 26.706999999999997
- type: mrr_at_1
value: 18.252
- type: mrr_at_10
value: 28.666000000000004
- type: mrr_at_100
value: 29.837000000000003
- type: mrr_at_1000
value: 29.898999999999997
- type: mrr_at_3
value: 25.308000000000003
- type: mrr_at_5
value: 27.226
- type: ndcg_at_1
value: 18.252
- type: ndcg_at_10
value: 34.176
- type: ndcg_at_100
value: 40.138
- type: ndcg_at_1000
value: 41.923
- type: ndcg_at_3
value: 27.214
- type: ndcg_at_5
value: 30.695
- type: precision_at_1
value: 18.252
- type: precision_at_10
value: 5.503
- type: precision_at_100
value: 0.8500000000000001
- type: precision_at_1000
value: 0.1
- type: precision_at_3
value: 11.667
- type: precision_at_5
value: 8.754000000000001
- type: recall_at_1
value: 17.771
- type: recall_at_10
value: 52.781
- type: recall_at_100
value: 80.638
- type: recall_at_1000
value: 94.46
- type: recall_at_3
value: 33.767
- type: recall_at_5
value: 42.172
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (en)
config: en
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 89.93388052895577
- type: f1
value: 89.55553145791954
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (de)
config: de
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 68.42490842490842
- type: f1
value: 67.01398674117826
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (es)
config: es
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 88.2121414276184
- type: f1
value: 87.61981627763988
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (fr)
config: fr
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 85.49013466958974
- type: f1
value: 85.09758510104221
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (hi)
config: hi
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 84.22732162065257
- type: f1
value: 83.24580378090367
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (th)
config: th
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 53.171790235081374
- type: f1
value: 51.93028909966765
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (en)
config: en
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 66.5640674874601
- type: f1
value: 49.856876973153966
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (de)
config: de
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 49.171597633136095
- type: f1
value: 32.166022205347545
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (es)
config: es
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 65.71714476317545
- type: f1
value: 45.748971341625136
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (fr)
config: fr
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 62.65267773253993
- type: f1
value: 45.904472624086026
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (hi)
config: hi
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 61.8752240946576
- type: f1
value: 40.7359613185448
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (th)
config: th
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 41.67088607594936
- type: f1
value: 28.12210726419673
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (af)
config: af
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 43.29186281102892
- type: f1
value: 41.83461350696014
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (am)
config: am
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 23.214525891055814
- type: f1
value: 22.364131190189962
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ar)
config: ar
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 53.38264963012777
- type: f1
value: 50.74546702709091
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (az)
config: az
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 39.55951580363147
- type: f1
value: 39.07769075741216
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (bn)
config: bn
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 56.73839946200403
- type: f1
value: 54.36728741542025
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (cy)
config: cy
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 39.99663752521857
- type: f1
value: 38.709817953652596
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (da)
config: da
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 46.933422999327504
- type: f1
value: 45.32022679895763
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (de)
config: de
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 45.820443846671154
- type: f1
value: 42.853155158197886
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (el)
config: el
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 37.874915938130464
- type: f1
value: 35.9849010888881
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (en)
config: en
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 66.08944182918628
- type: f1
value: 64.5039080809391
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (es)
config: es
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 61.17350369872226
- type: f1
value: 60.0792530132073
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (fa)
config: fa
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 45.652320107599195
- type: f1
value: 44.28182554287625
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (fi)
config: fi
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 40.282447881640884
- type: f1
value: 38.79927524886836
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (fr)
config: fr
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 62.60591795561533
- type: f1
value: 61.01451309609411
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (he)
config: he
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 32.225958305312716
- type: f1
value: 30.903299940417906
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (hi)
config: hi
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 59.46200403496974
- type: f1
value: 57.34556231956785
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (hu)
config: hu
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 40.907868190988566
- type: f1
value: 39.74702259997524
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (hy)
config: hy
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 29.939475453934094
- type: f1
value: 28.462353413371353
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (id)
config: id
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 59.14256893073302
- type: f1
value: 57.24600767871435
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (is)
config: is
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 39.620040349697376
- type: f1
value: 38.414866180464735
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (it)
config: it
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 51.772024209818426
- type: f1
value: 51.05050942366993
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ja)
config: ja
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 53.749159381304636
- type: f1
value: 52.04563008527909
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (jv)
config: jv
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 46.29455279085406
- type: f1
value: 43.84047527739209
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ka)
config: ka
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 25.107599193006045
- type: f1
value: 24.58731463875415
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (km)
config: km
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 27.21923335574984
- type: f1
value: 25.964338481976796
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (kn)
config: kn
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 47.96906523201077
- type: f1
value: 45.32239408435578
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ko)
config: ko
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 40.53799596503026
- type: f1
value: 39.15655510771227
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (lv)
config: lv
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 43.140551445864155
- type: f1
value: 42.12232733095163
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ml)
config: ml
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 53.69199731002017
- type: f1
value: 50.67085509122796
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (mn)
config: mn
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 33.37256220578346
- type: f1
value: 33.39335560955231
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ms)
config: ms
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 51.94014794889038
- type: f1
value: 50.6207021226521
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (my)
config: my
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 25.322797579018157
- type: f1
value: 23.94164121951907
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (nb)
config: nb
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 44.11903160726294
- type: f1
value: 43.016752983579536
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (nl)
config: nl
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 44.03496973772697
- type: f1
value: 42.322828283176754
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (pl)
config: pl
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 41.63080026899798
- type: f1
value: 39.58824644978166
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (pt)
config: pt
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 61.7350369872226
- type: f1
value: 59.956752206079386
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ro)
config: ro
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 45.72629455279086
- type: f1
value: 44.731249269647826
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ru)
config: ru
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 47.61264290517822
- type: f1
value: 45.5280995218491
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (sl)
config: sl
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 42.82784129119032
- type: f1
value: 41.37165985220223
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (sq)
config: sq
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 43.61466039004707
- type: f1
value: 43.164498227815535
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (sv)
config: sv
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 44.64021519838602
- type: f1
value: 43.04775030948548
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (sw)
config: sw
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 45.54808338937458
- type: f1
value: 44.011677633779975
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ta)
config: ta
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 51.2441156691325
- type: f1
value: 48.73592932403811
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (te)
config: te
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 47.43443174176195
- type: f1
value: 45.08686598891457
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (th)
config: th
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 36.87962340282448
- type: f1
value: 36.50540864756967
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (tl)
config: tl
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 45.9280430396772
- type: f1
value: 44.57216865343283
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (tr)
config: tr
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 38.591123066577
- type: f1
value: 37.886312373767446
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ur)
config: ur
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 51.85272360457296
- type: f1
value: 49.43461566216979
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (vi)
config: vi
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 58.72225958305313
- type: f1
value: 56.95500715299434
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-CN)
config: zh-CN
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 63.74915938130464
- type: f1
value: 62.35543158488615
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-TW)
config: zh-TW
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 59.95292535305985
- type: f1
value: 59.73499569346673
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (af)
config: af
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 47.42098184263618
- type: f1
value: 45.22541854557743
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (am)
config: am
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 24.707464694014796
- type: f1
value: 24.033506081882468
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ar)
config: ar
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 62.09145931405515
- type: f1
value: 62.22048940230962
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (az)
config: az
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 39.25016812373907
- type: f1
value: 38.35431952425269
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (bn)
config: bn
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 63.37256220578345
- type: f1
value: 63.12728180326932
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (cy)
config: cy
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 39.172831203765966
- type: f1
value: 37.078841372640234
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (da)
config: da
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 49.11230665770006
- type: f1
value: 46.489580286547245
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (de)
config: de
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 50.7128446536651
- type: f1
value: 48.27782602378952
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (el)
config: el
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 39.46536650975118
- type: f1
value: 37.4365280056047
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (en)
config: en
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 73.26160053799597
- type: f1
value: 73.4478249967817
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (es)
config: es
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 68.31203765971756
- type: f1
value: 68.70554437788068
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (fa)
config: fa
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 45.652320107599195
- type: f1
value: 44.55357745265521
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (fi)
config: fi
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 38.94754539340955
- type: f1
value: 36.48927336173062
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (fr)
config: fr
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 68.69872225958305
- type: f1
value: 68.81347966311543
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (he)
config: he
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 32.131809011432416
- type: f1
value: 30.212230946937474
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (hi)
config: hi
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 65.57498318762609
- type: f1
value: 65.16084751135229
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (hu)
config: hu
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 42.965702757229316
- type: f1
value: 40.575896627739105
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (hy)
config: hy
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 32.125084061869536
- type: f1
value: 30.708056882129476
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (id)
config: id
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 65.10759919300607
- type: f1
value: 64.5007800119315
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (is)
config: is
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 40.83725622057834
- type: f1
value: 37.855774705520886
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (it)
config: it
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 54.55279085406859
- type: f1
value: 52.73318944173822
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ja)
config: ja
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 57.14525891055817
- type: f1
value: 55.96714177558203
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (jv)
config: jv
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 49.30060524546065
- type: f1
value: 47.82999154670342
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ka)
config: ka
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 25.85743106926698
- type: f1
value: 24.974946990729716
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (km)
config: km
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 31.180228648285137
- type: f1
value: 28.22387838219335
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (kn)
config: kn
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 53.00941492938802
- type: f1
value: 52.39610045092559
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ko)
config: ko
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 40.24546065904505
- type: f1
value: 38.99779773215032
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (lv)
config: lv
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 41.88298587760592
- type: f1
value: 39.53867071594289
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ml)
config: ml
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 59.078681909885674
- type: f1
value: 58.47368723772022
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (mn)
config: mn
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 33.33893745796907
- type: f1
value: 32.113466354321226
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ms)
config: ms
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 57.454606590450574
- type: f1
value: 56.13075383338251
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (my)
config: my
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 27.19569603227976
- type: f1
value: 26.300773160344015
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (nb)
config: nb
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 46.78547410894418
- type: f1
value: 44.233771335183015
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (nl)
config: nl
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 48.4196368527236
- type: f1
value: 45.55838648206857
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (pl)
config: pl
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 41.63080026899798
- type: f1
value: 40.77775839499525
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (pt)
config: pt
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 66.408876933423
- type: f1
value: 66.7358693871042
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ro)
config: ro
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 50.077336919973106
- type: f1
value: 48.572749739090014
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ru)
config: ru
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 49.942837928715534
- type: f1
value: 49.34771836662566
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (sl)
config: sl
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 43.43308675184936
- type: f1
value: 41.818008297000986
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (sq)
config: sq
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 44.082044384667114
- type: f1
value: 43.25002746432129
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (sv)
config: sv
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 46.45258910558171
- type: f1
value: 44.00958237591922
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (sw)
config: sw
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 49.53261600537996
- type: f1
value: 48.01969699634672
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ta)
config: ta
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 56.792199058507066
- type: f1
value: 56.54421925671813
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (te)
config: te
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 54.0114324142569
- type: f1
value: 52.29830350891558
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (th)
config: th
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 38.584398117014125
- type: f1
value: 36.551426239639575
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (tl)
config: tl
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 48.07330195023538
- type: f1
value: 46.463553675519975
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (tr)
config: tr
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 40.645595158036315
- type: f1
value: 40.21280676607986
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ur)
config: ur
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 57.74714189643577
- type: f1
value: 56.8673027258351
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (vi)
config: vi
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 65.83389374579693
- type: f1
value: 66.11273939782248
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-CN)
config: zh-CN
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 72.38735709482181
- type: f1
value: 72.89481650271512
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-TW)
config: zh-TW
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 69.63685272360458
- type: f1
value: 70.72285841806938
- task:
type: Retrieval
dataset:
type: C-MTEB/MedicalRetrieval
name: MTEB MedicalRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 30.8
- type: map_at_10
value: 34.782000000000004
- type: map_at_100
value: 35.333999999999996
- type: map_at_1000
value: 35.405
- type: map_at_3
value: 34
- type: map_at_5
value: 34.345
- type: mrr_at_1
value: 30.8
- type: mrr_at_10
value: 34.782000000000004
- type: mrr_at_100
value: 35.333999999999996
- type: mrr_at_1000
value: 35.405
- type: mrr_at_3
value: 34
- type: mrr_at_5
value: 34.345
- type: ndcg_at_1
value: 30.8
- type: ndcg_at_10
value: 36.675000000000004
- type: ndcg_at_100
value: 39.633
- type: ndcg_at_1000
value: 41.904
- type: ndcg_at_3
value: 35.028
- type: ndcg_at_5
value: 35.648
- type: precision_at_1
value: 30.8
- type: precision_at_10
value: 4.26
- type: precision_at_100
value: 0.571
- type: precision_at_1000
value: 0.076
- type: precision_at_3
value: 12.667
- type: precision_at_5
value: 7.9
- type: recall_at_1
value: 30.8
- type: recall_at_10
value: 42.6
- type: recall_at_100
value: 57.099999999999994
- type: recall_at_1000
value: 75.8
- type: recall_at_3
value: 38
- type: recall_at_5
value: 39.5
- task:
type: Clustering
dataset:
type: mteb/medrxiv-clustering-p2p
name: MTEB MedrxivClusteringP2P
config: default
split: test
revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73
metrics:
- type: v_measure
value: 27.84536559870361
- task:
type: Clustering
dataset:
type: mteb/medrxiv-clustering-s2s
name: MTEB MedrxivClusteringS2S
config: default
split: test
revision: 35191c8c0dca72d8ff3efcd72aa802307d469663
metrics:
- type: v_measure
value: 27.714921841841605
- task:
type: Reranking
dataset:
type: mteb/mind_small
name: MTEB MindSmallReranking
config: default
split: test
revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69
metrics:
- type: map
value: 30.52145905910035
- type: mrr
value: 31.551577344311845
- task:
type: Reranking
dataset:
type: C-MTEB/Mmarco-reranking
name: MTEB MMarcoReranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 23.6853605350459
- type: mrr
value: 22.341269841269842
- task:
type: Classification
dataset:
type: C-MTEB/MultilingualSentiment-classification
name: MTEB MultilingualSentiment
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 63.16666666666666
- type: f1
value: 63.09453591106835
- task:
type: Retrieval
dataset:
type: nfcorpus
name: MTEB NFCorpus
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 3.7060000000000004
- type: map_at_10
value: 9.032
- type: map_at_100
value: 11.395
- type: map_at_1000
value: 12.713
- type: map_at_3
value: 6.502
- type: map_at_5
value: 7.8100000000000005
- type: mrr_at_1
value: 37.461
- type: mrr_at_10
value: 45.839999999999996
- type: mrr_at_100
value: 46.513
- type: mrr_at_1000
value: 46.571
- type: mrr_at_3
value: 43.55
- type: mrr_at_5
value: 44.773
- type: ndcg_at_1
value: 35.913000000000004
- type: ndcg_at_10
value: 27.340999999999998
- type: ndcg_at_100
value: 25.197000000000003
- type: ndcg_at_1000
value: 34.632000000000005
- type: ndcg_at_3
value: 31.952
- type: ndcg_at_5
value: 30.244
- type: precision_at_1
value: 37.461
- type: precision_at_10
value: 20.495
- type: precision_at_100
value: 6.551
- type: precision_at_1000
value: 1.966
- type: precision_at_3
value: 30.753000000000004
- type: precision_at_5
value: 26.935
- type: recall_at_1
value: 3.7060000000000004
- type: recall_at_10
value: 12.958
- type: recall_at_100
value: 26.582
- type: recall_at_1000
value: 59.724
- type: recall_at_3
value: 7.503
- type: recall_at_5
value: 9.808
- task:
type: Retrieval
dataset:
type: nq
name: MTEB NQ
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 22.201999999999998
- type: map_at_10
value: 33.76
- type: map_at_100
value: 34.867
- type: map_at_1000
value: 34.92
- type: map_at_3
value: 30.233999999999998
- type: map_at_5
value: 32.291
- type: mrr_at_1
value: 25.232
- type: mrr_at_10
value: 36.239
- type: mrr_at_100
value: 37.119
- type: mrr_at_1000
value: 37.162
- type: mrr_at_3
value: 33.213
- type: mrr_at_5
value: 35.02
- type: ndcg_at_1
value: 25.232
- type: ndcg_at_10
value: 40.046
- type: ndcg_at_100
value: 45.025
- type: ndcg_at_1000
value: 46.459
- type: ndcg_at_3
value: 33.343
- type: ndcg_at_5
value: 36.801
- type: precision_at_1
value: 25.232
- type: precision_at_10
value: 6.796
- type: precision_at_100
value: 0.959
- type: precision_at_1000
value: 0.109
- type: precision_at_3
value: 15.276
- type: precision_at_5
value: 11.17
- type: recall_at_1
value: 22.201999999999998
- type: recall_at_10
value: 56.733
- type: recall_at_100
value: 79.041
- type: recall_at_1000
value: 90.08500000000001
- type: recall_at_3
value: 39.412000000000006
- type: recall_at_5
value: 47.352
- task:
type: PairClassification
dataset:
type: C-MTEB/OCNLI
name: MTEB Ocnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 62.53383865728208
- type: cos_sim_ap
value: 66.3197921045625
- type: cos_sim_f1
value: 69.3385214007782
- type: cos_sim_precision
value: 54.89833641404805
- type: cos_sim_recall
value: 94.08658922914466
- type: dot_accuracy
value: 59.7184623714131
- type: dot_ap
value: 61.53586693000539
- type: dot_f1
value: 68.26923076923077
- type: dot_precision
value: 52.53272623790552
- type: dot_recall
value: 97.46568109820485
- type: euclidean_accuracy
value: 62.912831618841366
- type: euclidean_ap
value: 67.15479155849464
- type: euclidean_f1
value: 70.64071370640713
- type: euclidean_precision
value: 57.34035549703752
- type: euclidean_recall
value: 91.97465681098205
- type: manhattan_accuracy
value: 63.50839198700595
- type: manhattan_ap
value: 67.55807251483273
- type: manhattan_f1
value: 70.58356490670901
- type: manhattan_precision
value: 56.55216284987278
- type: manhattan_recall
value: 93.8753959873284
- type: max_accuracy
value: 63.50839198700595
- type: max_ap
value: 67.55807251483273
- type: max_f1
value: 70.64071370640713
- task:
type: Classification
dataset:
type: C-MTEB/OnlineShopping-classification
name: MTEB OnlineShopping
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 87.11
- type: ap
value: 84.20351278644551
- type: f1
value: 87.10043002123766
- task:
type: STS
dataset:
type: C-MTEB/PAWSX
name: MTEB PAWSX
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 13.050279647770473
- type: cos_sim_spearman
value: 14.227909232579874
- type: euclidean_pearson
value: 16.372629300358096
- type: euclidean_spearman
value: 14.68140021547196
- type: manhattan_pearson
value: 16.266960163157336
- type: manhattan_spearman
value: 14.627750758965616
- task:
type: STS
dataset:
type: C-MTEB/QBQTC
name: MTEB QBQTC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 30.56036276943463
- type: cos_sim_spearman
value: 32.918859292204
- type: euclidean_pearson
value: 31.679745438037195
- type: euclidean_spearman
value: 33.68461814972644
- type: manhattan_pearson
value: 31.994557954084563
- type: manhattan_spearman
value: 33.97758185204816
- task:
type: Retrieval
dataset:
type: quora
name: MTEB QuoraRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 68.327
- type: map_at_10
value: 81.938
- type: map_at_100
value: 82.581
- type: map_at_1000
value: 82.60300000000001
- type: map_at_3
value: 78.89399999999999
- type: map_at_5
value: 80.816
- type: mrr_at_1
value: 78.75
- type: mrr_at_10
value: 85.302
- type: mrr_at_100
value: 85.432
- type: mrr_at_1000
value: 85.434
- type: mrr_at_3
value: 84.128
- type: mrr_at_5
value: 84.91199999999999
- type: ndcg_at_1
value: 78.74
- type: ndcg_at_10
value: 86.042
- type: ndcg_at_100
value: 87.468
- type: ndcg_at_1000
value: 87.641
- type: ndcg_at_3
value: 82.799
- type: ndcg_at_5
value: 84.603
- type: precision_at_1
value: 78.74
- type: precision_at_10
value: 13.071
- type: precision_at_100
value: 1.508
- type: precision_at_1000
value: 0.156
- type: precision_at_3
value: 36.08
- type: precision_at_5
value: 23.87
- type: recall_at_1
value: 68.327
- type: recall_at_10
value: 93.962
- type: recall_at_100
value: 99.054
- type: recall_at_1000
value: 99.9
- type: recall_at_3
value: 84.788
- type: recall_at_5
value: 89.73
- task:
type: Clustering
dataset:
type: mteb/reddit-clustering
name: MTEB RedditClustering
config: default
split: test
revision: 24640382cdbf8abc73003fb0fa6d111a705499eb
metrics:
- type: v_measure
value: 41.337989152483956
- task:
type: Clustering
dataset:
type: mteb/reddit-clustering-p2p
name: MTEB RedditClusteringP2P
config: default
split: test
revision: 282350215ef01743dc01b456c7f5241fa8937f16
metrics:
- type: v_measure
value: 51.2046136625677
- task:
type: Retrieval
dataset:
type: scidocs
name: MTEB SCIDOCS
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 3.763
- type: map_at_10
value: 8.785
- type: map_at_100
value: 10.266
- type: map_at_1000
value: 10.506
- type: map_at_3
value: 6.551
- type: map_at_5
value: 7.670000000000001
- type: mrr_at_1
value: 18.5
- type: mrr_at_10
value: 27.771
- type: mrr_at_100
value: 28.842000000000002
- type: mrr_at_1000
value: 28.913
- type: mrr_at_3
value: 24.767
- type: mrr_at_5
value: 26.457000000000004
- type: ndcg_at_1
value: 18.5
- type: ndcg_at_10
value: 15.312000000000001
- type: ndcg_at_100
value: 21.599
- type: ndcg_at_1000
value: 26.473999999999997
- type: ndcg_at_3
value: 14.821000000000002
- type: ndcg_at_5
value: 12.836
- type: precision_at_1
value: 18.5
- type: precision_at_10
value: 7.779999999999999
- type: precision_at_100
value: 1.69
- type: precision_at_1000
value: 0.28700000000000003
- type: precision_at_3
value: 13.667000000000002
- type: precision_at_5
value: 11.08
- type: recall_at_1
value: 3.763
- type: recall_at_10
value: 15.798000000000002
- type: recall_at_100
value: 34.313
- type: recall_at_1000
value: 58.318000000000005
- type: recall_at_3
value: 8.312999999999999
- type: recall_at_5
value: 11.238
- task:
type: STS
dataset:
type: mteb/sickr-sts
name: MTEB SICK-R
config: default
split: test
revision: a6ea5a8cab320b040a23452cc28066d9beae2cee
metrics:
- type: cos_sim_pearson
value: 84.33402689861924
- type: cos_sim_spearman
value: 78.52738315932625
- type: euclidean_pearson
value: 80.800678573052
- type: euclidean_spearman
value: 77.86666946799137
- type: manhattan_pearson
value: 81.03106755866989
- type: manhattan_spearman
value: 78.0676393879487
- task:
type: STS
dataset:
type: mteb/sts12-sts
name: MTEB STS12
config: default
split: test
revision: a0d554a64d88156834ff5ae9920b964011b16384
metrics:
- type: cos_sim_pearson
value: 81.86998503723257
- type: cos_sim_spearman
value: 74.07437934108376
- type: euclidean_pearson
value: 80.91626452869946
- type: euclidean_spearman
value: 76.88419802521403
- type: manhattan_pearson
value: 81.50196980117957
- type: manhattan_spearman
value: 77.2456891009073
- task:
type: STS
dataset:
type: mteb/sts13-sts
name: MTEB STS13
config: default
split: test
revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca
metrics:
- type: cos_sim_pearson
value: 81.19616084290932
- type: cos_sim_spearman
value: 81.80834431353927
- type: euclidean_pearson
value: 81.25429737195789
- type: euclidean_spearman
value: 82.00934127307355
- type: manhattan_pearson
value: 81.67403556759655
- type: manhattan_spearman
value: 82.42359818976753
- task:
type: STS
dataset:
type: mteb/sts14-sts
name: MTEB STS14
config: default
split: test
revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375
metrics:
- type: cos_sim_pearson
value: 81.50884725941148
- type: cos_sim_spearman
value: 77.0493522248929
- type: euclidean_pearson
value: 79.15856111178543
- type: euclidean_spearman
value: 77.24292975474096
- type: manhattan_pearson
value: 79.22641788874807
- type: manhattan_spearman
value: 77.37101663798234
- task:
type: STS
dataset:
type: mteb/sts15-sts
name: MTEB STS15
config: default
split: test
revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3
metrics:
- type: cos_sim_pearson
value: 83.75652767224308
- type: cos_sim_spearman
value: 84.61113973428688
- type: euclidean_pearson
value: 83.73646379542737
- type: euclidean_spearman
value: 84.47126779405652
- type: manhattan_pearson
value: 83.89617307570857
- type: manhattan_spearman
value: 84.6073703393468
- task:
type: STS
dataset:
type: mteb/sts16-sts
name: MTEB STS16
config: default
split: test
revision: 4d8694f8f0e0100860b497b999b3dbed754a0513
metrics:
- type: cos_sim_pearson
value: 81.16302763567215
- type: cos_sim_spearman
value: 83.08923353997561
- type: euclidean_pearson
value: 80.08338016232464
- type: euclidean_spearman
value: 80.40181608724076
- type: manhattan_pearson
value: 80.02358856208708
- type: manhattan_spearman
value: 80.30032329982274
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (ko-ko)
config: ko-ko
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 56.45965932801117
- type: cos_sim_spearman
value: 57.28270045199294
- type: euclidean_pearson
value: 57.3615782157595
- type: euclidean_spearman
value: 56.94348399074146
- type: manhattan_pearson
value: 57.9426531718209
- type: manhattan_spearman
value: 57.61844831263504
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (ar-ar)
config: ar-ar
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 80.2973366536596
- type: cos_sim_spearman
value: 80.60259304741632
- type: euclidean_pearson
value: 78.30266089843892
- type: euclidean_spearman
value: 78.06065126709282
- type: manhattan_pearson
value: 78.61370380599344
- type: manhattan_spearman
value: 78.45738598619143
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-ar)
config: en-ar
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 72.35020162217042
- type: cos_sim_spearman
value: 72.59857902847162
- type: euclidean_pearson
value: 65.03547299350457
- type: euclidean_spearman
value: 64.16617373109685
- type: manhattan_pearson
value: 65.68996569454929
- type: manhattan_spearman
value: 64.88542254595046
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-de)
config: en-de
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 39.766484883595425
- type: cos_sim_spearman
value: 40.3429946300341
- type: euclidean_pearson
value: 39.47427150040957
- type: euclidean_spearman
value: 39.072525589079696
- type: manhattan_pearson
value: 40.56345338078474
- type: manhattan_spearman
value: 40.444629078138036
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-en)
config: en-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 88.83798941013089
- type: cos_sim_spearman
value: 89.15159294402415
- type: euclidean_pearson
value: 87.9810618414505
- type: euclidean_spearman
value: 87.90818542026535
- type: manhattan_pearson
value: 88.06116863048229
- type: manhattan_spearman
value: 88.00182442010694
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-tr)
config: en-tr
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 7.416028059666332
- type: cos_sim_spearman
value: 6.792945857606915
- type: euclidean_pearson
value: 11.485332917116061
- type: euclidean_spearman
value: 9.793932873423419
- type: manhattan_pearson
value: 9.148469412558393
- type: manhattan_spearman
value: 7.803450524017845
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (es-en)
config: es-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 80.16381852152489
- type: cos_sim_spearman
value: 81.80324089694928
- type: euclidean_pearson
value: 76.41433274302783
- type: euclidean_spearman
value: 77.15238726996526
- type: manhattan_pearson
value: 77.08610108551368
- type: manhattan_spearman
value: 77.99971298324311
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (es-es)
config: es-es
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 85.11032272383456
- type: cos_sim_spearman
value: 85.64528002839239
- type: euclidean_pearson
value: 85.54301672487198
- type: euclidean_spearman
value: 84.21727806530393
- type: manhattan_pearson
value: 85.57145576255618
- type: manhattan_spearman
value: 84.07127479487694
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (fr-en)
config: fr-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 79.73703272230806
- type: cos_sim_spearman
value: 79.9424510113259
- type: euclidean_pearson
value: 77.64485173960838
- type: euclidean_spearman
value: 77.54693014468836
- type: manhattan_pearson
value: 77.96911553781774
- type: manhattan_spearman
value: 77.87266778206842
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (it-en)
config: it-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 37.260672179617515
- type: cos_sim_spearman
value: 34.80434004457536
- type: euclidean_pearson
value: 38.55806751295782
- type: euclidean_spearman
value: 36.129700913023115
- type: manhattan_pearson
value: 40.74316244582763
- type: manhattan_spearman
value: 38.60667540883322
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (nl-en)
config: nl-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 38.038311386574456
- type: cos_sim_spearman
value: 33.576193063894195
- type: euclidean_pearson
value: 33.712663568034316
- type: euclidean_spearman
value: 32.560617375956916
- type: manhattan_pearson
value: 35.60457167895616
- type: manhattan_spearman
value: 34.63036216555931
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (en)
config: en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 61.01583638162472
- type: cos_sim_spearman
value: 62.92281428893316
- type: euclidean_pearson
value: 62.939630289711815
- type: euclidean_spearman
value: 64.15209661725994
- type: manhattan_pearson
value: 64.24261705090608
- type: manhattan_spearman
value: 64.78283158164017
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de)
config: de
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 21.529440799555704
- type: cos_sim_spearman
value: 26.62727800620091
- type: euclidean_pearson
value: 16.837244578590123
- type: euclidean_spearman
value: 25.012107525591425
- type: manhattan_pearson
value: 18.445531476179454
- type: manhattan_spearman
value: 27.070240480795153
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (es)
config: es
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 49.655500043363624
- type: cos_sim_spearman
value: 56.31248457847469
- type: euclidean_pearson
value: 48.787154598246616
- type: euclidean_spearman
value: 52.90454409579225
- type: manhattan_pearson
value: 55.392327232639836
- type: manhattan_spearman
value: 57.3726886727899
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (pl)
config: pl
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 2.9137753115190304
- type: cos_sim_spearman
value: 15.062114976486532
- type: euclidean_pearson
value: -2.034404984782681
- type: euclidean_spearman
value: 14.683481835467338
- type: manhattan_pearson
value: -0.22204468354050833
- type: manhattan_spearman
value: 15.526420635759743
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (tr)
config: tr
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 4.3616620418459915
- type: cos_sim_spearman
value: 22.11078316878173
- type: euclidean_pearson
value: 15.111514877123403
- type: euclidean_spearman
value: 21.232869644925973
- type: manhattan_pearson
value: 19.71276925909529
- type: manhattan_spearman
value: 25.704469862313466
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (ar)
config: ar
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 44.25888840250496
- type: cos_sim_spearman
value: 54.82352971568842
- type: euclidean_pearson
value: 48.00261414068268
- type: euclidean_spearman
value: 53.3721608428832
- type: manhattan_pearson
value: 50.6442021864215
- type: manhattan_spearman
value: 55.352339945631954
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (ru)
config: ru
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 0.08233514100531068
- type: cos_sim_spearman
value: 28.771721168834276
- type: euclidean_pearson
value: 10.783524938899138
- type: euclidean_spearman
value: 24.67831010432439
- type: manhattan_pearson
value: 16.98415610436092
- type: manhattan_spearman
value: 25.81670115913176
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh)
config: zh
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 36.86678706245425
- type: cos_sim_spearman
value: 40.9736918674032
- type: euclidean_pearson
value: 26.42365971768556
- type: euclidean_spearman
value: 30.479818788692054
- type: manhattan_pearson
value: 41.08694658968258
- type: manhattan_spearman
value: 45.080877435751084
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (fr)
config: fr
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 75.98114217777062
- type: cos_sim_spearman
value: 78.7295845730892
- type: euclidean_pearson
value: 76.99433076522276
- type: euclidean_spearman
value: 79.71421663258973
- type: manhattan_pearson
value: 78.65656344143478
- type: manhattan_spearman
value: 80.60968909615123
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de-en)
config: de-en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 47.33261398683554
- type: cos_sim_spearman
value: 49.547954534754666
- type: euclidean_pearson
value: 48.23362592012922
- type: euclidean_spearman
value: 49.17277986369927
- type: manhattan_pearson
value: 49.06792311033889
- type: manhattan_spearman
value: 51.27529282708198
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (es-en)
config: es-en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 66.10070360470756
- type: cos_sim_spearman
value: 71.03150249855938
- type: euclidean_pearson
value: 67.05372897033872
- type: euclidean_spearman
value: 69.73291838049877
- type: manhattan_pearson
value: 70.34740916239467
- type: manhattan_spearman
value: 72.40053406658815
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (it)
config: it
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 56.581317404418904
- type: cos_sim_spearman
value: 62.61318021096797
- type: euclidean_pearson
value: 57.4403074342031
- type: euclidean_spearman
value: 60.04897783631694
- type: manhattan_pearson
value: 58.441729285803014
- type: manhattan_spearman
value: 60.70510326005463
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (pl-en)
config: pl-en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 47.064414464023905
- type: cos_sim_spearman
value: 43.716659075869465
- type: euclidean_pearson
value: 43.81699490724336
- type: euclidean_spearman
value: 43.784380306563726
- type: manhattan_pearson
value: 53.664583329563264
- type: manhattan_spearman
value: 45.399271192350135
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh-en)
config: zh-en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 63.585903017365055
- type: cos_sim_spearman
value: 63.90147651068459
- type: euclidean_pearson
value: 50.21918146173064
- type: euclidean_spearman
value: 53.02530618040754
- type: manhattan_pearson
value: 62.7472089813117
- type: manhattan_spearman
value: 63.90440606248973
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (es-it)
config: es-it
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 59.06715980430013
- type: cos_sim_spearman
value: 61.2993294424547
- type: euclidean_pearson
value: 53.67335552456426
- type: euclidean_spearman
value: 55.32940583953816
- type: manhattan_pearson
value: 58.08097600675386
- type: manhattan_spearman
value: 57.1966250850173
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de-fr)
config: de-fr
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 18.94271219818519
- type: cos_sim_spearman
value: 22.355519793818935
- type: euclidean_pearson
value: 14.336479135636187
- type: euclidean_spearman
value: 18.862751864788684
- type: manhattan_pearson
value: 14.481730447681057
- type: manhattan_spearman
value: 17.572142526671563
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de-pl)
config: de-pl
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 20.644357537446464
- type: cos_sim_spearman
value: 35.32083671407284
- type: euclidean_pearson
value: 28.24720906134992
- type: euclidean_spearman
value: 46.437508077438395
- type: manhattan_pearson
value: 42.09834718968137
- type: manhattan_spearman
value: 53.02744622635869
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (fr-pl)
config: fr-pl
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 71.84986730523782
- type: cos_sim_spearman
value: 73.24670207647144
- type: euclidean_pearson
value: 62.450055500805604
- type: euclidean_spearman
value: 61.97797868009122
- type: manhattan_pearson
value: 56.32083882980946
- type: manhattan_spearman
value: 39.440531887330785
- task:
type: STS
dataset:
type: C-MTEB/STSB
name: MTEB STSB
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 78.11479317838469
- type: cos_sim_spearman
value: 77.7709743500025
- type: euclidean_pearson
value: 78.83834281752932
- type: euclidean_spearman
value: 78.21978829646487
- type: manhattan_pearson
value: 79.36075578990533
- type: manhattan_spearman
value: 78.72958965446072
- task:
type: STS
dataset:
type: mteb/stsbenchmark-sts
name: MTEB STSBenchmark
config: default
split: test
revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831
metrics:
- type: cos_sim_pearson
value: 82.92539499228975
- type: cos_sim_spearman
value: 83.63025944536395
- type: euclidean_pearson
value: 81.54744230098872
- type: euclidean_spearman
value: 81.08707735758752
- type: manhattan_pearson
value: 81.50252353111375
- type: manhattan_spearman
value: 81.00641210322735
- task:
type: Reranking
dataset:
type: mteb/scidocs-reranking
name: MTEB SciDocsRR
config: default
split: test
revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab
metrics:
- type: map
value: 75.12690809334019
- type: mrr
value: 92.28846951886169
- task:
type: Retrieval
dataset:
type: scifact
name: MTEB SciFact
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 47.15
- type: map_at_10
value: 56.748
- type: map_at_100
value: 57.528999999999996
- type: map_at_1000
value: 57.56400000000001
- type: map_at_3
value: 53.691
- type: map_at_5
value: 55.656000000000006
- type: mrr_at_1
value: 49.667
- type: mrr_at_10
value: 58.24700000000001
- type: mrr_at_100
value: 58.855000000000004
- type: mrr_at_1000
value: 58.888
- type: mrr_at_3
value: 55.72200000000001
- type: mrr_at_5
value: 57.272
- type: ndcg_at_1
value: 49.667
- type: ndcg_at_10
value: 61.739
- type: ndcg_at_100
value: 65.17399999999999
- type: ndcg_at_1000
value: 66.122
- type: ndcg_at_3
value: 56.266000000000005
- type: ndcg_at_5
value: 59.357000000000006
- type: precision_at_1
value: 49.667
- type: precision_at_10
value: 8.5
- type: precision_at_100
value: 1.04
- type: precision_at_1000
value: 0.11199999999999999
- type: precision_at_3
value: 22.111
- type: precision_at_5
value: 15.133
- type: recall_at_1
value: 47.15
- type: recall_at_10
value: 75.52799999999999
- type: recall_at_100
value: 91.167
- type: recall_at_1000
value: 98.667
- type: recall_at_3
value: 60.978
- type: recall_at_5
value: 68.839
- task:
type: PairClassification
dataset:
type: mteb/sprintduplicatequestions-pairclassification
name: MTEB SprintDuplicateQuestions
config: default
split: test
revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46
metrics:
- type: cos_sim_accuracy
value: 99.71188118811881
- type: cos_sim_ap
value: 92.0858173884619
- type: cos_sim_f1
value: 85.48864758144126
- type: cos_sim_precision
value: 84.40545808966861
- type: cos_sim_recall
value: 86.6
- type: dot_accuracy
value: 99.57722772277228
- type: dot_ap
value: 83.92226742515372
- type: dot_f1
value: 78.85091629519565
- type: dot_precision
value: 78.11579980372915
- type: dot_recall
value: 79.60000000000001
- type: euclidean_accuracy
value: 99.6970297029703
- type: euclidean_ap
value: 91.69378964699095
- type: euclidean_f1
value: 85.08771929824562
- type: euclidean_precision
value: 82.98479087452472
- type: euclidean_recall
value: 87.3
- type: manhattan_accuracy
value: 99.7019801980198
- type: manhattan_ap
value: 92.00969741996086
- type: manhattan_f1
value: 84.95752123938031
- type: manhattan_precision
value: 84.91508491508492
- type: manhattan_recall
value: 85
- type: max_accuracy
value: 99.71188118811881
- type: max_ap
value: 92.0858173884619
- type: max_f1
value: 85.48864758144126
- task:
type: Clustering
dataset:
type: mteb/stackexchange-clustering
name: MTEB StackExchangeClustering
config: default
split: test
revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259
metrics:
- type: v_measure
value: 54.50675991473899
- task:
type: Clustering
dataset:
type: mteb/stackexchange-clustering-p2p
name: MTEB StackExchangeClusteringP2P
config: default
split: test
revision: 815ca46b2622cec33ccafc3735d572c266efdb44
metrics:
- type: v_measure
value: 31.12415042272221
- task:
type: Reranking
dataset:
type: mteb/stackoverflowdupquestions-reranking
name: MTEB StackOverflowDupQuestions
config: default
split: test
revision: e185fbe320c72810689fc5848eb6114e1ef5ec69
metrics:
- type: map
value: 47.37961638353922
- type: mrr
value: 48.04425558102029
- task:
type: Summarization
dataset:
type: mteb/summeval
name: MTEB SummEval
config: default
split: test
revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c
metrics:
- type: cos_sim_pearson
value: 31.358583236464177
- type: cos_sim_spearman
value: 32.06044850511017
- type: dot_pearson
value: 30.36343303587471
- type: dot_spearman
value: 30.303932242144704
- task:
type: Reranking
dataset:
type: C-MTEB/T2Reranking
name: MTEB T2Reranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 63.73951666189072
- type: mrr
value: 73.54706021429108
- task:
type: Retrieval
dataset:
type: C-MTEB/T2Retrieval
name: MTEB T2Retrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 16.892
- type: map_at_10
value: 40.215
- type: map_at_100
value: 43.9
- type: map_at_1000
value: 44.185
- type: map_at_3
value: 30.008000000000003
- type: map_at_5
value: 35.465
- type: mrr_at_1
value: 63.931000000000004
- type: mrr_at_10
value: 70.35
- type: mrr_at_100
value: 70.762
- type: mrr_at_1000
value: 70.784
- type: mrr_at_3
value: 68.863
- type: mrr_at_5
value: 69.758
- type: ndcg_at_1
value: 63.931000000000004
- type: ndcg_at_10
value: 51.573
- type: ndcg_at_100
value: 59.067
- type: ndcg_at_1000
value: 62.388
- type: ndcg_at_3
value: 55.422000000000004
- type: ndcg_at_5
value: 52.322
- type: precision_at_1
value: 63.931000000000004
- type: precision_at_10
value: 25.373
- type: precision_at_100
value: 3.894
- type: precision_at_1000
value: 0.47400000000000003
- type: precision_at_3
value: 48.083
- type: precision_at_5
value: 38.513
- type: recall_at_1
value: 16.892
- type: recall_at_10
value: 49.945
- type: recall_at_100
value: 73.41499999999999
- type: recall_at_1000
value: 89.776
- type: recall_at_3
value: 32.544000000000004
- type: recall_at_5
value: 40.501
- task:
type: Classification
dataset:
type: C-MTEB/TNews-classification
name: MTEB TNews
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 44.153999999999996
- type: f1
value: 42.69123774230511
- task:
type: Retrieval
dataset:
type: trec-covid
name: MTEB TRECCOVID
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 0.22300000000000003
- type: map_at_10
value: 1.7999999999999998
- type: map_at_100
value: 9.098
- type: map_at_1000
value: 20.59
- type: map_at_3
value: 0.6459999999999999
- type: map_at_5
value: 1.006
- type: mrr_at_1
value: 84
- type: mrr_at_10
value: 91.5
- type: mrr_at_100
value: 91.5
- type: mrr_at_1000
value: 91.5
- type: mrr_at_3
value: 91
- type: mrr_at_5
value: 91.5
- type: ndcg_at_1
value: 80
- type: ndcg_at_10
value: 72.992
- type: ndcg_at_100
value: 51.778999999999996
- type: ndcg_at_1000
value: 44.473
- type: ndcg_at_3
value: 77.531
- type: ndcg_at_5
value: 74.685
- type: precision_at_1
value: 84
- type: precision_at_10
value: 78.60000000000001
- type: precision_at_100
value: 52.800000000000004
- type: precision_at_1000
value: 19.736
- type: precision_at_3
value: 83.333
- type: precision_at_5
value: 80
- type: recall_at_1
value: 0.22300000000000003
- type: recall_at_10
value: 2.016
- type: recall_at_100
value: 12.21
- type: recall_at_1000
value: 41.427
- type: recall_at_3
value: 0.6839999999999999
- type: recall_at_5
value: 1.083
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (sqi-eng)
config: sqi-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 11
- type: f1
value: 8.487309997179562
- type: precision
value: 7.935185890268856
- type: recall
value: 11
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fry-eng)
config: fry-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 23.699421965317917
- type: f1
value: 18.09982567208001
- type: precision
value: 16.582017825552963
- type: recall
value: 23.699421965317917
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kur-eng)
config: kur-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.780487804878048
- type: f1
value: 6.484836753129436
- type: precision
value: 5.916220801747723
- type: recall
value: 8.780487804878048
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tur-eng)
config: tur-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5
- type: f1
value: 3.493223480735001
- type: precision
value: 3.1492116349139385
- type: recall
value: 5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (deu-eng)
config: deu-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 33.6
- type: f1
value: 29.339340352229065
- type: precision
value: 27.997920626374693
- type: recall
value: 33.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nld-eng)
config: nld-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 20.200000000000003
- type: f1
value: 16.330981736231458
- type: precision
value: 15.250949969794044
- type: recall
value: 20.200000000000003
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ron-eng)
config: ron-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 19.6
- type: f1
value: 14.951120083366323
- type: precision
value: 13.617335362707001
- type: recall
value: 19.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ang-eng)
config: ang-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 20.149253731343283
- type: f1
value: 13.312899786780385
- type: precision
value: 11.979388770433545
- type: recall
value: 20.149253731343283
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ido-eng)
config: ido-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 31.4
- type: f1
value: 26.21323201417634
- type: precision
value: 24.607830064672168
- type: recall
value: 31.4
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (jav-eng)
config: jav-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 18.048780487804876
- type: f1
value: 14.347798542920492
- type: precision
value: 13.301672920575362
- type: recall
value: 18.048780487804876
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (isl-eng)
config: isl-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5.2
- type: f1
value: 3.2713297295122503
- type: precision
value: 2.978548911585725
- type: recall
value: 5.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (slv-eng)
config: slv-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.411907654921021
- type: f1
value: 5.412915976323278
- type: precision
value: 4.975402373122839
- type: recall
value: 7.411907654921021
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cym-eng)
config: cym-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.521739130434783
- type: f1
value: 5.871393789897329
- type: precision
value: 5.350472658912557
- type: recall
value: 8.521739130434783
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kaz-eng)
config: kaz-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 1.565217391304348
- type: f1
value: 0.7422394530145001
- type: precision
value: 0.7201734373569025
- type: recall
value: 1.565217391304348
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (est-eng)
config: est-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5.3
- type: f1
value: 3.0838354401589694
- type: precision
value: 2.709942839090994
- type: recall
value: 5.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (heb-eng)
config: heb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 0.8
- type: f1
value: 0.24583802742178057
- type: precision
value: 0.18710578268453032
- type: recall
value: 0.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (gla-eng)
config: gla-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.945717732207479
- type: f1
value: 2.7266734043909437
- type: precision
value: 2.3247505400014186
- type: recall
value: 4.945717732207479
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mar-eng)
config: mar-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 54.2
- type: f1
value: 47.22780366692132
- type: precision
value: 44.740178571428565
- type: recall
value: 54.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lat-eng)
config: lat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 25.8
- type: f1
value: 19.547406382656526
- type: precision
value: 17.80766233766234
- type: recall
value: 25.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bel-eng)
config: bel-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.9
- type: f1
value: 3.283031457969928
- type: precision
value: 3.0361515007649467
- type: recall
value: 4.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pms-eng)
config: pms-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 22.476190476190478
- type: f1
value: 17.494204011570957
- type: precision
value: 16.16236240785113
- type: recall
value: 22.476190476190478
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (gle-eng)
config: gle-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.3
- type: f1
value: 3.461898170471662
- type: precision
value: 2.975546957350575
- type: recall
value: 6.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pes-eng)
config: pes-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.6
- type: f1
value: 5.874235156578609
- type: precision
value: 5.201352547725499
- type: recall
value: 8.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nob-eng)
config: nob-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 15.2
- type: f1
value: 11.908986787697534
- type: precision
value: 11.090628985937808
- type: recall
value: 15.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bul-eng)
config: bul-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.9
- type: f1
value: 4.58348360335125
- type: precision
value: 4.183620994869927
- type: recall
value: 6.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cbk-eng)
config: cbk-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 62.1
- type: f1
value: 55.70845598845599
- type: precision
value: 53.22281746031747
- type: recall
value: 62.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hun-eng)
config: hun-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.8
- type: f1
value: 3.246932234432234
- type: precision
value: 2.9738765839703265
- type: recall
value: 4.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (uig-eng)
config: uig-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 0.8999999999999999
- type: f1
value: 0.5331481481481481
- type: precision
value: 0.4918990604783396
- type: recall
value: 0.8999999999999999
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (rus-eng)
config: rus-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 31.7
- type: f1
value: 25.22406237037816
- type: precision
value: 23.27273155929038
- type: recall
value: 31.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (spa-eng)
config: spa-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 96.5
- type: f1
value: 95.48333333333333
- type: precision
value: 95
- type: recall
value: 96.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hye-eng)
config: hye-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 0.40431266846361186
- type: f1
value: 0.22521185350542844
- type: precision
value: 0.20245384171411912
- type: recall
value: 0.40431266846361186
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tel-eng)
config: tel-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 43.162393162393165
- type: f1
value: 35.83662064431295
- type: precision
value: 33.66590199923534
- type: recall
value: 43.162393162393165
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (afr-eng)
config: afr-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 12.2
- type: f1
value: 9.007009351120605
- type: precision
value: 8.26509907921979
- type: recall
value: 12.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mon-eng)
config: mon-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 2.0454545454545454
- type: f1
value: 0.846869670733307
- type: precision
value: 0.719285857023819
- type: recall
value: 2.0454545454545454
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (arz-eng)
config: arz-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 56.18448637316562
- type: f1
value: 49.41850369523325
- type: precision
value: 46.84486373165618
- type: recall
value: 56.18448637316562
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hrv-eng)
config: hrv-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.4
- type: f1
value: 6.274306734742452
- type: precision
value: 5.854786915151029
- type: recall
value: 8.4
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nov-eng)
config: nov-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 45.13618677042802
- type: f1
value: 38.784818726452976
- type: precision
value: 36.65848310789945
- type: recall
value: 45.13618677042802
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (gsw-eng)
config: gsw-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 23.076923076923077
- type: f1
value: 17.501757501757503
- type: precision
value: 16.06289721674337
- type: recall
value: 23.076923076923077
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nds-eng)
config: nds-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 15.8
- type: f1
value: 11.834682187321722
- type: precision
value: 10.871016304088595
- type: recall
value: 15.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ukr-eng)
config: ukr-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.3
- type: f1
value: 4.929314970921539
- type: precision
value: 4.427714750128542
- type: recall
value: 7.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (uzb-eng)
config: uzb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5.14018691588785
- type: f1
value: 2.543797914741945
- type: precision
value: 2.1476927403586066
- type: recall
value: 5.14018691588785
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lit-eng)
config: lit-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5
- type: f1
value: 3.173243817101591
- type: precision
value: 2.8643206769285485
- type: recall
value: 5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ina-eng)
config: ina-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 69.5
- type: f1
value: 63.89614902641219
- type: precision
value: 61.628650793650785
- type: recall
value: 69.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lfn-eng)
config: lfn-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 41.8
- type: f1
value: 37.523909714712914
- type: precision
value: 36.054581750900766
- type: recall
value: 41.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (zsm-eng)
config: zsm-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 79.2
- type: f1
value: 74.88805555555554
- type: precision
value: 73.05083333333333
- type: recall
value: 79.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ita-eng)
config: ita-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 43.5
- type: f1
value: 37.28660019590605
- type: precision
value: 35.18067447433519
- type: recall
value: 43.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cmn-eng)
config: cmn-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 94.5
- type: f1
value: 92.95
- type: precision
value: 92.2
- type: recall
value: 94.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lvs-eng)
config: lvs-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5.2
- type: f1
value: 3.5297755651484026
- type: precision
value: 3.190013722690584
- type: recall
value: 5.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (glg-eng)
config: glg-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 74.7
- type: f1
value: 69.2602380952381
- type: precision
value: 67.03261904761905
- type: recall
value: 74.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ceb-eng)
config: ceb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8
- type: f1
value: 5.639611303143687
- type: precision
value: 5.209856824277429
- type: recall
value: 8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bre-eng)
config: bre-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.1
- type: f1
value: 3.847611167634209
- type: precision
value: 3.3324923687423693
- type: recall
value: 6.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ben-eng)
config: ben-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 75.5
- type: f1
value: 70.14214285714286
- type: precision
value: 67.88761904761904
- type: recall
value: 75.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (swg-eng)
config: swg-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 20.535714285714285
- type: f1
value: 16.437074829931973
- type: precision
value: 15.459837781266353
- type: recall
value: 20.535714285714285
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (arq-eng)
config: arq-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 21.405049396267835
- type: f1
value: 16.162968480476714
- type: precision
value: 14.506603642481391
- type: recall
value: 21.405049396267835
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kab-eng)
config: kab-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 1.4000000000000001
- type: f1
value: 0.8861559696342305
- type: precision
value: 0.7898232323232323
- type: recall
value: 1.4000000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fra-eng)
config: fra-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 93.5
- type: f1
value: 91.65333333333334
- type: precision
value: 90.80833333333332
- type: recall
value: 93.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (por-eng)
config: por-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 93.8
- type: f1
value: 92.08333333333333
- type: precision
value: 91.23333333333333
- type: recall
value: 93.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tat-eng)
config: tat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 1.3
- type: f1
value: 0.9654912597950575
- type: precision
value: 0.911237853823405
- type: recall
value: 1.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (oci-eng)
config: oci-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 35.5
- type: f1
value: 29.385868020868024
- type: precision
value: 27.38218614718615
- type: recall
value: 35.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pol-eng)
config: pol-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.3
- type: f1
value: 5.625495291471218
- type: precision
value: 5.006352187769519
- type: recall
value: 8.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (war-eng)
config: war-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 9.3
- type: f1
value: 7.188871139201601
- type: precision
value: 6.68110313042221
- type: recall
value: 9.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (aze-eng)
config: aze-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.9
- type: f1
value: 3.4368196711816386
- type: precision
value: 3.1516575755476186
- type: recall
value: 4.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (vie-eng)
config: vie-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 94.5
- type: f1
value: 92.85666666666667
- type: precision
value: 92.07499999999999
- type: recall
value: 94.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nno-eng)
config: nno-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 10.9
- type: f1
value: 8.052880589619718
- type: precision
value: 7.2833020438680816
- type: recall
value: 10.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cha-eng)
config: cha-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 21.897810218978105
- type: f1
value: 16.459096459096457
- type: precision
value: 14.99391727493917
- type: recall
value: 21.897810218978105
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mhr-eng)
config: mhr-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 0.8
- type: f1
value: 0.43900258600589265
- type: precision
value: 0.42151473277789064
- type: recall
value: 0.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (dan-eng)
config: dan-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 14.899999999999999
- type: f1
value: 11.403181682754628
- type: precision
value: 10.506373051667312
- type: recall
value: 14.899999999999999
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ell-eng)
config: ell-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 1.9
- type: f1
value: 0.8872641689515834
- type: precision
value: 0.7857231069685399
- type: recall
value: 1.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (amh-eng)
config: amh-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 1.1904761904761905
- type: f1
value: 0.20847048496818082
- type: precision
value: 0.11904761904761904
- type: recall
value: 1.1904761904761905
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pam-eng)
config: pam-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5.3
- type: f1
value: 3.784571880595977
- type: precision
value: 3.4556477020719782
- type: recall
value: 5.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hsb-eng)
config: hsb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 9.316770186335404
- type: f1
value: 6.80343720685027
- type: precision
value: 6.316650292717499
- type: recall
value: 9.316770186335404
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (srp-eng)
config: srp-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5.8999999999999995
- type: f1
value: 4.5486926228313695
- type: precision
value: 4.311121913612427
- type: recall
value: 5.8999999999999995
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (epo-eng)
config: epo-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 18.099999999999998
- type: f1
value: 13.4170874831821
- type: precision
value: 12.178193046524806
- type: recall
value: 18.099999999999998
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kzj-eng)
config: kzj-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.3999999999999995
- type: f1
value: 3.3905735425765524
- type: precision
value: 3.2588935800436625
- type: recall
value: 4.3999999999999995
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (awa-eng)
config: awa-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 37.66233766233766
- type: f1
value: 30.539579468150897
- type: precision
value: 28.60288100547841
- type: recall
value: 37.66233766233766
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fao-eng)
config: fao-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 12.213740458015266
- type: f1
value: 8.297822182308039
- type: precision
value: 7.463649581970193
- type: recall
value: 12.213740458015266
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mal-eng)
config: mal-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 78.31149927219796
- type: f1
value: 73.35759340126152
- type: precision
value: 71.26394953905871
- type: recall
value: 78.31149927219796
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ile-eng)
config: ile-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 51.800000000000004
- type: f1
value: 44.24010323010323
- type: precision
value: 41.450707972582975
- type: recall
value: 51.800000000000004
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bos-eng)
config: bos-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 13.27683615819209
- type: f1
value: 9.167320569156727
- type: precision
value: 8.200402665583079
- type: recall
value: 13.27683615819209
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cor-eng)
config: cor-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.8
- type: f1
value: 3.1268763352790283
- type: precision
value: 2.84393718699601
- type: recall
value: 4.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cat-eng)
config: cat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 85.1
- type: f1
value: 81.55
- type: precision
value: 79.98166666666665
- type: recall
value: 85.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (eus-eng)
config: eus-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 48.3
- type: f1
value: 42.347894491129786
- type: precision
value: 40.36040404040404
- type: recall
value: 48.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (yue-eng)
config: yue-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 78.8
- type: f1
value: 74.35484848484847
- type: precision
value: 72.43277777777777
- type: recall
value: 78.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (swe-eng)
config: swe-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 13.900000000000002
- type: f1
value: 10.718252991153888
- type: precision
value: 9.835761434404196
- type: recall
value: 13.900000000000002
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (dtp-eng)
config: dtp-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.9
- type: f1
value: 3.371714825002496
- type: precision
value: 3.085928254003479
- type: recall
value: 4.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kat-eng)
config: kat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 0.5361930294906166
- type: f1
value: 0.40389703692021933
- type: precision
value: 0.40302666854804575
- type: recall
value: 0.5361930294906166
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (jpn-eng)
config: jpn-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 55.300000000000004
- type: f1
value: 48.83353113553113
- type: precision
value: 46.48630659536542
- type: recall
value: 55.300000000000004
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (csb-eng)
config: csb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.300395256916996
- type: f1
value: 5.261552988548536
- type: precision
value: 4.724388115499655
- type: recall
value: 8.300395256916996
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (xho-eng)
config: xho-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.450704225352112
- type: f1
value: 4.829974470478787
- type: precision
value: 4.337585798478816
- type: recall
value: 8.450704225352112
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (orv-eng)
config: orv-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 1.0778443113772456
- type: f1
value: 0.5373251562068135
- type: precision
value: 0.5107640721914694
- type: recall
value: 1.0778443113772456
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ind-eng)
config: ind-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 88.5
- type: f1
value: 85.46333333333334
- type: precision
value: 84.1
- type: recall
value: 88.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tuk-eng)
config: tuk-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5.41871921182266
- type: f1
value: 2.8063639248802965
- type: precision
value: 2.2699550039451513
- type: recall
value: 5.41871921182266
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (max-eng)
config: max-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 40.49295774647887
- type: f1
value: 33.455454951933824
- type: precision
value: 31.4339393461183
- type: recall
value: 40.49295774647887
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (swh-eng)
config: swh-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 18.974358974358974
- type: f1
value: 14.517578026097205
- type: precision
value: 13.3510327465177
- type: recall
value: 18.974358974358974
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hin-eng)
config: hin-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 88.5
- type: f1
value: 85.34666666666666
- type: precision
value: 83.89999999999999
- type: recall
value: 88.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (dsb-eng)
config: dsb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.1419624217119
- type: f1
value: 5.830783012763732
- type: precision
value: 5.4408714223116545
- type: recall
value: 8.1419624217119
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ber-eng)
config: ber-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5.800000000000001
- type: f1
value: 3.9245687335866406
- type: precision
value: 3.5535667824951584
- type: recall
value: 5.800000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tam-eng)
config: tam-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 68.40390879478826
- type: f1
value: 62.25738069386277
- type: precision
value: 60.10935318752908
- type: recall
value: 68.40390879478826
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (slk-eng)
config: slk-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.1
- type: f1
value: 5.4876787833762135
- type: precision
value: 5.126663482701374
- type: recall
value: 7.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tgl-eng)
config: tgl-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.9
- type: f1
value: 6.519531004112515
- type: precision
value: 5.987707404636394
- type: recall
value: 8.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ast-eng)
config: ast-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 66.92913385826772
- type: f1
value: 59.96062992125984
- type: precision
value: 57.13348331458567
- type: recall
value: 66.92913385826772
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mkd-eng)
config: mkd-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.3
- type: f1
value: 2.765805343607201
- type: precision
value: 2.5247851243177144
- type: recall
value: 4.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (khm-eng)
config: khm-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 0.41551246537396125
- type: f1
value: 0.1497838495760933
- type: precision
value: 0.14429034844729552
- type: recall
value: 0.41551246537396125
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ces-eng)
config: ces-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5.800000000000001
- type: f1
value: 3.761224995516873
- type: precision
value: 3.2689210175496086
- type: recall
value: 5.800000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tzl-eng)
config: tzl-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 16.346153846153847
- type: f1
value: 14.524291497975709
- type: precision
value: 13.995726495726496
- type: recall
value: 16.346153846153847
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (urd-eng)
config: urd-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 67.80000000000001
- type: f1
value: 61.615800865800864
- type: precision
value: 59.12333333333334
- type: recall
value: 67.80000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ara-eng)
config: ara-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 83.8
- type: f1
value: 80.08857142857143
- type: precision
value: 78.46666666666667
- type: recall
value: 83.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kor-eng)
config: kor-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.2
- type: f1
value: 2.6507751588440254
- type: precision
value: 2.335273168189835
- type: recall
value: 4.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (yid-eng)
config: yid-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 0.4716981132075472
- type: f1
value: 0.19293763102725367
- type: precision
value: 0.1622040325564188
- type: recall
value: 0.4716981132075472
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fin-eng)
config: fin-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.9
- type: f1
value: 3.5001791555125235
- type: precision
value: 3.277940522301425
- type: recall
value: 4.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tha-eng)
config: tha-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 0.9124087591240875
- type: f1
value: 0.5083420229405631
- type: precision
value: 0.4674562188049969
- type: recall
value: 0.9124087591240875
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (wuu-eng)
config: wuu-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 79.4
- type: f1
value: 74.62333333333333
- type: precision
value: 72.52333333333334
- type: recall
value: 79.4
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringP2P
name: MTEB ThuNewsClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 51.02719281751054
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringS2S
name: MTEB ThuNewsClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 48.31885339280247
- task:
type: Retrieval
dataset:
type: webis-touche2020
name: MTEB Touche2020
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 2.426
- type: map_at_10
value: 9.029
- type: map_at_100
value: 14.299999999999999
- type: map_at_1000
value: 15.798000000000002
- type: map_at_3
value: 4.626
- type: map_at_5
value: 6.221
- type: mrr_at_1
value: 32.653
- type: mrr_at_10
value: 46.608
- type: mrr_at_100
value: 47.195
- type: mrr_at_1000
value: 47.208
- type: mrr_at_3
value: 41.837
- type: mrr_at_5
value: 43.673
- type: ndcg_at_1
value: 29.592000000000002
- type: ndcg_at_10
value: 23.354
- type: ndcg_at_100
value: 33.875
- type: ndcg_at_1000
value: 45.369
- type: ndcg_at_3
value: 25.734
- type: ndcg_at_5
value: 23.873
- type: precision_at_1
value: 32.653
- type: precision_at_10
value: 21.224
- type: precision_at_100
value: 7.122000000000001
- type: precision_at_1000
value: 1.459
- type: precision_at_3
value: 26.531
- type: precision_at_5
value: 24.082
- type: recall_at_1
value: 2.426
- type: recall_at_10
value: 15.622
- type: recall_at_100
value: 44.318999999999996
- type: recall_at_1000
value: 78.632
- type: recall_at_3
value: 5.798
- type: recall_at_5
value: 8.927
- task:
type: Classification
dataset:
type: mteb/toxic_conversations_50k
name: MTEB ToxicConversationsClassification
config: default
split: test
revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c
metrics:
- type: accuracy
value: 67.9606
- type: ap
value: 12.665547829558923
- type: f1
value: 52.10043478110198
- task:
type: Classification
dataset:
type: mteb/tweet_sentiment_extraction
name: MTEB TweetSentimentExtractionClassification
config: default
split: test
revision: d604517c81ca91fe16a244d1248fc021f9ecee7a
metrics:
- type: accuracy
value: 59.601018675721576
- type: f1
value: 59.91486569196274
- task:
type: Clustering
dataset:
type: mteb/twentynewsgroups-clustering
name: MTEB TwentyNewsgroupsClustering
config: default
split: test
revision: 6125ec4e24fa026cec8a478383ee943acfbd5449
metrics:
- type: v_measure
value: 37.881729581540135
- task:
type: PairClassification
dataset:
type: mteb/twittersemeval2015-pairclassification
name: MTEB TwitterSemEval2015
config: default
split: test
revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1
metrics:
- type: cos_sim_accuracy
value: 83.68003814746379
- type: cos_sim_ap
value: 65.95659315362258
- type: cos_sim_f1
value: 61.94669484560291
- type: cos_sim_precision
value: 55.80706579225725
- type: cos_sim_recall
value: 69.6042216358839
- type: dot_accuracy
value: 81.97532335936103
- type: dot_ap
value: 58.99091918849294
- type: dot_f1
value: 57.098765432098766
- type: dot_precision
value: 51.8990073370738
- type: dot_recall
value: 63.45646437994723
- type: euclidean_accuracy
value: 83.18531322644095
- type: euclidean_ap
value: 64.5631762106556
- type: euclidean_f1
value: 61.150808574652125
- type: euclidean_precision
value: 58.25173155003582
- type: euclidean_recall
value: 64.35356200527704
- type: manhattan_accuracy
value: 83.14358943792097
- type: manhattan_ap
value: 64.73090464118813
- type: manhattan_f1
value: 61.228384019081695
- type: manhattan_precision
value: 55.86507072905332
- type: manhattan_recall
value: 67.73087071240106
- type: max_accuracy
value: 83.68003814746379
- type: max_ap
value: 65.95659315362258
- type: max_f1
value: 61.94669484560291
- task:
type: PairClassification
dataset:
type: mteb/twitterurlcorpus-pairclassification
name: MTEB TwitterURLCorpus
config: default
split: test
revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf
metrics:
- type: cos_sim_accuracy
value: 88.7161873714441
- type: cos_sim_ap
value: 85.10870963707444
- type: cos_sim_f1
value: 77.88396923766146
- type: cos_sim_precision
value: 75.59791274097695
- type: cos_sim_recall
value: 80.31259624268556
- type: dot_accuracy
value: 87.74595412737222
- type: dot_ap
value: 81.22910623983562
- type: dot_f1
value: 76.08511889448344
- type: dot_precision
value: 72.78672385908163
- type: dot_recall
value: 79.69664305512781
- type: euclidean_accuracy
value: 88.13404742500097
- type: euclidean_ap
value: 84.03032098854915
- type: euclidean_f1
value: 76.3909440662918
- type: euclidean_precision
value: 73.51894047279977
- type: euclidean_recall
value: 79.49645826917154
- type: manhattan_accuracy
value: 88.13598789148911
- type: manhattan_ap
value: 84.13258714083858
- type: manhattan_f1
value: 76.44922164566346
- type: manhattan_precision
value: 73.70640365923384
- type: manhattan_recall
value: 79.40406529103788
- type: max_accuracy
value: 88.7161873714441
- type: max_ap
value: 85.10870963707444
- type: max_f1
value: 77.88396923766146
- task:
type: Retrieval
dataset:
type: C-MTEB/VideoRetrieval
name: MTEB VideoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 41.8
- type: map_at_10
value: 50.57000000000001
- type: map_at_100
value: 51.271
- type: map_at_1000
value: 51.31099999999999
- type: map_at_3
value: 48.283
- type: map_at_5
value: 49.633
- type: mrr_at_1
value: 41.8
- type: mrr_at_10
value: 50.57000000000001
- type: mrr_at_100
value: 51.271
- type: mrr_at_1000
value: 51.31099999999999
- type: mrr_at_3
value: 48.283
- type: mrr_at_5
value: 49.633
- type: ndcg_at_1
value: 41.8
- type: ndcg_at_10
value: 55.071999999999996
- type: ndcg_at_100
value: 58.604
- type: ndcg_at_1000
value: 59.679
- type: ndcg_at_3
value: 50.394000000000005
- type: ndcg_at_5
value: 52.825
- type: precision_at_1
value: 41.8
- type: precision_at_10
value: 6.93
- type: precision_at_100
value: 0.861
- type: precision_at_1000
value: 0.095
- type: precision_at_3
value: 18.833
- type: precision_at_5
value: 12.479999999999999
- type: recall_at_1
value: 41.8
- type: recall_at_10
value: 69.3
- type: recall_at_100
value: 86.1
- type: recall_at_1000
value: 94.6
- type: recall_at_3
value: 56.49999999999999
- type: recall_at_5
value: 62.4
- task:
type: Classification
dataset:
type: C-MTEB/waimai-classification
name: MTEB Waimai
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 80.65
- type: ap
value: 59.927241826012924
- type: f1
value: 78.72456184299979
Model Card for udever-bloom
udever-bloom-560m is finetuned from bigscience/bloom-560m via BitFit on MS MARCO Passage Ranking, SNLI and MultiNLI data.
It is a universal embedding model across tasks, natural and programming languages.
(From the technical view, udever is merely with some minor improvements to sgpt-bloom)
Model Details
Model Description
- Developed by: Alibaba Group
- Model type: Transformer-based Language Model (decoder-only)
- Language(s) (NLP): Multiple; see bloom training data
- Finetuned from model : bigscience/bloom-560m
Model Sources
- Repository: github.com/izhx/uni-rep
- Paper : Language Models are Universal Embedders
- Training Date : 2023-06
How to Get Started with the Model
Use the code below to get started with the model.
import torch
from transformers import AutoTokenizer, BloomModel
tokenizer = AutoTokenizer.from_pretrained('izhx/udever-bloom-560m')
model = BloomModel.from_pretrained('izhx/udever-bloom-560m')
boq, eoq, bod, eod = '[BOQ]', '[EOQ]', '[BOD]', '[EOD]'
eoq_id, eod_id = tokenizer.convert_tokens_to_ids([eoq, eod])
if tokenizer.padding_side != 'left':
print('!!!', tokenizer.padding_side)
tokenizer.padding_side = 'left'
def encode(texts: list, is_query: bool = True, max_length=300):
bos = boq if is_query else bod
eos_id = eoq_id if is_query else eod_id
texts = [bos + t for t in texts]
encoding = tokenizer(
texts, truncation=True, max_length=max_length - 1, padding=True
)
for ids, mask in zip(encoding['input_ids'], encoding['attention_mask']):
ids.append(eos_id)
mask.append(1)
inputs = tokenizer.pad(encoding, return_tensors='pt')
with torch.inference_mode():
outputs = model(**inputs)
embeds = outputs.last_hidden_state[:, -1]
return embeds
encode(['I am Bert', 'You are Elmo'])
Training Details
Training Data
- MS MARCO Passage Ranking, retrieved by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86)
- SNLI and MultiNLI (https://sbert.net/datasets/AllNLI.tsv.gz)
Training Procedure
Preprocessing
MS MARCO hard negatives provided by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86). Negatives for SNLI and MultiNLI are randomly sampled.
Training Hyperparameters
- Training regime: tf32, BitFit
- Batch size: 1024
- Epochs: 3
- Optimizer: AdamW
- Learning rate: 1e-4
- Scheduler: constant with warmup.
- Warmup: 0.25 epoch
Evaluation
Table 1: Massive Text Embedding Benchmark MTEB
| MTEB | Avg. | Class. | Clust. | PairClass. | Rerank. | Retr. | STS | Summ. |
|---|---|---|---|---|---|---|---|---|
| #Datasets ➡️ | 56 | 12 | 11 | 3 | 4 | 15 | 10 | 1 |
| bge-large-en-v1.5 | 64.23 | 75.97 | 46.08 | 87.12 | 60.03 | 54.29 | 83.11 | 31.61 |
| bge-base-en-v1.5 | 63.55 | 75.53 | 45.77 | 86.55 | 58.86 | 53.25 | 82.4 | 31.07 |
| gte-large | 63.13 | 73.33 | 46.84 | 85 | 59.13 | 52.22 | 83.35 | 31.66 |
| gte-base | 62.39 | 73.01 | 46.2 | 84.57 | 58.61 | 51.14 | 82.3 | 31.17 |
| e5-large-v2 | 62.25 | 75.24 | 44.49 | 86.03 | 56.61 | 50.56 | 82.05 | 30.19 |
| instructor-xl | 61.79 | 73.12 | 44.74 | 86.62 | 57.29 | 49.26 | 83.06 | 32.32 |
| instructor-large | 61.59 | 73.86 | 45.29 | 85.89 | 57.54 | 47.57 | 83.15 | 31.84 |
| e5-base-v2 | 61.5 | 73.84 | 43.8 | 85.73 | 55.91 | 50.29 | 81.05 | 30.28 |
| e5-large | 61.42 | 73.14 | 43.33 | 85.94 | 56.53 | 49.99 | 82.06 | 30.97 |
| text-embedding-ada-002 (OpenAI API) | 60.99 | 70.93 | 45.9 | 84.89 | 56.32 | 49.25 | 80.97 | 30.8 |
| e5-base | 60.44 | 72.63 | 42.11 | 85.09 | 55.7 | 48.75 | 80.96 | 31.01 |
| SGPT-5.8B-msmarco | 58.93 | 68.13 | 40.34 | 82 | 56.56 | 50.25 | 78.1 | 31.46 |
| sgpt-bloom-7b1-msmarco | 57.59 | 66.19 | 38.93 | 81.9 | 55.65 | 48.22 | 77.74 | 33.6 |
| Udever-bloom-560m | 55.80 | 68.04 | 36.89 | 81.05 | 52.60 | 41.19 | 79.93 | 32.06 |
| Udever-bloom-1b1 | 58.28 | 70.18 | 39.11 | 83.11 | 54.28 | 45.27 | 81.52 | 31.10 |
| Udever-bloom-3b | 59.86 | 71.91 | 40.74 | 84.06 | 54.90 | 47.67 | 82.37 | 30.62 |
| Udever-bloom-7b1 | 60.63 | 72.13 | 40.81 | 85.40 | 55.91 | 49.34 | 83.01 | 30.97 |
Table 2: CodeSearchNet
| CodeSearchNet | Go | Ruby | Python | Java | JS | PHP | Avg. |
|---|---|---|---|---|---|---|---|
| CodeBERT | 69.3 | 70.6 | 84.0 | 86.8 | 74.8 | 70.6 | 76.0 |
| GraphCodeBERT | 84.1 | 73.2 | 87.9 | 75.7 | 71.1 | 72.5 | 77.4 |
| cpt-code S | 97.7 | 86.3 | 99.8 | 94.0 | 86.0 | 96.7 | 93.4 |
| cpt-code M | 97.5 | 85.5 | 99.9 | 94.4 | 86.5 | 97.2 | 93.5 |
| sgpt-bloom-7b1-msmarco | 76.79 | 69.25 | 95.68 | 77.93 | 70.35 | 73.45 | 77.24 |
| Udever-bloom-560m | 75.38 | 66.67 | 96.23 | 78.99 | 69.39 | 73.69 | 76.73 |
| Udever-bloom-1b1 | 78.76 | 72.85 | 97.67 | 82.77 | 74.38 | 78.97 | 80.90 |
| Udever-bloom-3b | 80.63 | 75.40 | 98.02 | 83.88 | 76.18 | 79.67 | 82.29 |
| Udever-bloom-7b1 | 79.37 | 76.59 | 98.38 | 84.68 | 77.49 | 80.03 | 82.76 |
Table 3: Chinese multi-domain retrieval Multi-cpr
| E-commerce | Entertainment video | Medical | ||||||
|---|---|---|---|---|---|---|---|---|
| Model | Train | Backbone | MRR@10 | Recall@1k | MRR@10 | Recall@1k | MRR@10 | Recall@1k |
| BM25 | - | - | 0.225 | 0.815 | 0.225 | 0.780 | 0.187 | 0.482 |
| Doc2Query | - | - | 0.239 | 0.826 | 0.238 | 0.794 | 0.210 | 0.505 |
| DPR-1 | In-Domain | BERT | 0.270 | 0.921 | 0.254 | 0.934 | 0.327 | 0.747 |
| DPR-2 | In-Domain | BERT-CT | 0.289 | 0.926 | 0.263 | 0.935 | 0.339 | 0.769 |
| text-embedding-ada-002 | General | GPT | 0.183 | 0.825 | 0.159 | 0.786 | 0.245 | 0.593 |
| sgpt-bloom-7b1-msmarco | General | BLOOM | 0.242 | 0.840 | 0.227 | 0.829 | 0.311 | 0.675 |
| Udever-bloom-560m | General | BLOOM | 0.156 | 0.802 | 0.149 | 0.749 | 0.245 | 0.571 |
| Udever-bloom-1b1 | General | BLOOM | 0.244 | 0.863 | 0.208 | 0.815 | 0.241 | 0.557 |
| Udever-bloom-3b | General | BLOOM | 0.267 | 0.871 | 0.228 | 0.836 | 0.288 | 0.619 |
| Udever-bloom-7b1 | General | BLOOM | 0.296 | 0.889 | 0.267 | 0.907 | 0.343 | 0.705 |
More results refer to paper section 3.
Technical Specifications
Model Architecture and Objective
- Model: bigscience/bloom-560m.
- Objective: Constrastive loss with hard negatives (refer to paper section 2.2).
Compute Infrastructure
- Nvidia A100 SXM4 80GB.
- torch 2.0.0, transformers 4.29.2.
Citation
BibTeX:
@article{zhang2023language,
title={Language Models are Universal Embedders},
author={Zhang, Xin and Li, Zehan and Zhang, Yanzhao and Long, Dingkun and Xie, Pengjun and Zhang, Meishan and Zhang, Min},
journal={arXiv preprint arXiv:2310.08232},
year={2023}
}