metadata
license: bigscience-bloom-rail-1.0
language:
- ak
- ar
- as
- bm
- bn
- ca
- code
- en
- es
- eu
- fon
- fr
- gu
- hi
- id
- ig
- ki
- kn
- lg
- ln
- ml
- mr
- ne
- nso
- ny
- or
- pa
- pt
- rn
- rw
- sn
- st
- sw
- ta
- te
- tn
- ts
- tum
- tw
- ur
- vi
- wo
- xh
- yo
- zh
- zhs
- zht
- zu
tags:
- mteb
model-index:
- name: udever-bloom-1b1
results:
- task:
type: STS
dataset:
type: C-MTEB/AFQMC
name: MTEB AFQMC
config: default
split: validation
revision: None
metrics:
- type: cos_sim_pearson
value: 27.90020553155914
- type: cos_sim_spearman
value: 27.980812877007445
- type: euclidean_pearson
value: 27.412021502878105
- type: euclidean_spearman
value: 27.608320539898134
- type: manhattan_pearson
value: 27.493591460276278
- type: manhattan_spearman
value: 27.715134644174423
- task:
type: STS
dataset:
type: C-MTEB/ATEC
name: MTEB ATEC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 35.15277604796132
- type: cos_sim_spearman
value: 35.863846005221575
- type: euclidean_pearson
value: 37.65681598655078
- type: euclidean_spearman
value: 35.50116107334066
- type: manhattan_pearson
value: 37.736463166370854
- type: manhattan_spearman
value: 35.53412987209704
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (en)
config: en
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
value: 69.9402985074627
- type: ap
value: 33.4661141650045
- type: f1
value: 64.31759903129324
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (de)
config: de
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
value: 66.02783725910065
- type: ap
value: 78.25152113775748
- type: f1
value: 64.00236113368896
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (en-ext)
config: en-ext
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
value: 72.01649175412295
- type: ap
value: 21.28416661100625
- type: f1
value: 59.481902269256096
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (ja)
config: ja
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
value: 58.76873661670234
- type: ap
value: 12.828869547428084
- type: f1
value: 47.5200475889544
- task:
type: Classification
dataset:
type: mteb/amazon_polarity
name: MTEB AmazonPolarityClassification
config: default
split: test
revision: e2d317d38cd51312af73b3d32a06d1a08b442046
metrics:
- type: accuracy
value: 87.191175
- type: ap
value: 82.4408783026622
- type: f1
value: 87.16605834054603
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (en)
config: en
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 41.082
- type: f1
value: 40.54924237159631
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (de)
config: de
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 30.447999999999997
- type: f1
value: 30.0643283775686
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (es)
config: es
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 40.800000000000004
- type: f1
value: 39.64954112879312
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (fr)
config: fr
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 40.686
- type: f1
value: 39.917643425172
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (ja)
config: ja
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 32.074
- type: f1
value: 31.878305643409334
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (zh)
config: zh
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 38.122
- type: f1
value: 37.296210966123446
- task:
type: Retrieval
dataset:
type: arguana
name: MTEB ArguAna
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 22.262
- type: map_at_10
value: 37.667
- type: map_at_100
value: 38.812999999999995
- type: map_at_1000
value: 38.829
- type: map_at_3
value: 32.421
- type: map_at_5
value: 35.202
- type: mrr_at_1
value: 22.759999999999998
- type: mrr_at_10
value: 37.817
- type: mrr_at_100
value: 38.983000000000004
- type: mrr_at_1000
value: 38.999
- type: mrr_at_3
value: 32.61
- type: mrr_at_5
value: 35.333999999999996
- type: ndcg_at_1
value: 22.262
- type: ndcg_at_10
value: 46.671
- type: ndcg_at_100
value: 51.519999999999996
- type: ndcg_at_1000
value: 51.876999999999995
- type: ndcg_at_3
value: 35.696
- type: ndcg_at_5
value: 40.722
- type: precision_at_1
value: 22.262
- type: precision_at_10
value: 7.575
- type: precision_at_100
value: 0.9690000000000001
- type: precision_at_1000
value: 0.1
- type: precision_at_3
value: 15.055
- type: precision_at_5
value: 11.479000000000001
- type: recall_at_1
value: 22.262
- type: recall_at_10
value: 75.747
- type: recall_at_100
value: 96.871
- type: recall_at_1000
value: 99.57300000000001
- type: recall_at_3
value: 45.164
- type: recall_at_5
value: 57.397
- task:
type: Clustering
dataset:
type: mteb/arxiv-clustering-p2p
name: MTEB ArxivClusteringP2P
config: default
split: test
revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d
metrics:
- type: v_measure
value: 44.51799756336072
- task:
type: Clustering
dataset:
type: mteb/arxiv-clustering-s2s
name: MTEB ArxivClusteringS2S
config: default
split: test
revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53
metrics:
- type: v_measure
value: 34.44923356952161
- task:
type: Reranking
dataset:
type: mteb/askubuntudupquestions-reranking
name: MTEB AskUbuntuDupQuestions
config: default
split: test
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54
metrics:
- type: map
value: 59.49540399419566
- type: mrr
value: 73.43028624192061
- task:
type: STS
dataset:
type: mteb/biosses-sts
name: MTEB BIOSSES
config: default
split: test
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a
metrics:
- type: cos_sim_pearson
value: 87.67018580352695
- type: cos_sim_spearman
value: 84.64530219460785
- type: euclidean_pearson
value: 87.10187265189109
- type: euclidean_spearman
value: 86.19051812629264
- type: manhattan_pearson
value: 86.78890467534343
- type: manhattan_spearman
value: 85.60134807514734
- task:
type: STS
dataset:
type: C-MTEB/BQ
name: MTEB BQ
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 46.308790362891266
- type: cos_sim_spearman
value: 46.22674926863126
- type: euclidean_pearson
value: 47.36625172551589
- type: euclidean_spearman
value: 47.55854392572494
- type: manhattan_pearson
value: 47.3342490976193
- type: manhattan_spearman
value: 47.52249648456463
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (de-en)
config: de-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 42.67223382045929
- type: f1
value: 42.02704262244064
- type: precision
value: 41.76166726545405
- type: recall
value: 42.67223382045929
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (fr-en)
config: fr-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 97.95289456306405
- type: f1
value: 97.70709516472228
- type: precision
value: 97.58602978941964
- type: recall
value: 97.95289456306405
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (ru-en)
config: ru-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 25.375822653273296
- type: f1
value: 24.105776263207947
- type: precision
value: 23.644628498465117
- type: recall
value: 25.375822653273296
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (zh-en)
config: zh-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 98.31490258030541
- type: f1
value: 98.24469018781815
- type: precision
value: 98.2095839915745
- type: recall
value: 98.31490258030541
- task:
type: Classification
dataset:
type: mteb/banking77
name: MTEB Banking77Classification
config: default
split: test
revision: 0fd18e25b25c072e09e0d92ab615fda904d66300
metrics:
- type: accuracy
value: 82.89285714285714
- type: f1
value: 82.84943089389121
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-p2p
name: MTEB BiorxivClusteringP2P
config: default
split: test
revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40
metrics:
- type: v_measure
value: 35.25261508107809
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-s2s
name: MTEB BiorxivClusteringS2S
config: default
split: test
revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908
metrics:
- type: v_measure
value: 30.708512338509653
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringP2P
name: MTEB CLSClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 35.361295166692464
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringS2S
name: MTEB CLSClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 37.06879287045825
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv1-reranking
name: MTEB CMedQAv1
config: default
split: test
revision: None
metrics:
- type: map
value: 66.06033605600476
- type: mrr
value: 70.82825396825396
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv2-reranking
name: MTEB CMedQAv2
config: default
split: test
revision: None
metrics:
- type: map
value: 66.9600733219955
- type: mrr
value: 72.19742063492063
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackAndroidRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 29.526999999999997
- type: map_at_10
value: 38.747
- type: map_at_100
value: 40.172999999999995
- type: map_at_1000
value: 40.311
- type: map_at_3
value: 35.969
- type: map_at_5
value: 37.344
- type: mrr_at_1
value: 36.767
- type: mrr_at_10
value: 45.082
- type: mrr_at_100
value: 45.898
- type: mrr_at_1000
value: 45.958
- type: mrr_at_3
value: 43.085
- type: mrr_at_5
value: 44.044
- type: ndcg_at_1
value: 36.767
- type: ndcg_at_10
value: 44.372
- type: ndcg_at_100
value: 49.908
- type: ndcg_at_1000
value: 52.358000000000004
- type: ndcg_at_3
value: 40.711000000000006
- type: ndcg_at_5
value: 41.914
- type: precision_at_1
value: 36.767
- type: precision_at_10
value: 8.283
- type: precision_at_100
value: 1.3679999999999999
- type: precision_at_1000
value: 0.189
- type: precision_at_3
value: 19.599
- type: precision_at_5
value: 13.505
- type: recall_at_1
value: 29.526999999999997
- type: recall_at_10
value: 54.198
- type: recall_at_100
value: 77.818
- type: recall_at_1000
value: 93.703
- type: recall_at_3
value: 42.122
- type: recall_at_5
value: 46.503
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackEnglishRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 22.646
- type: map_at_10
value: 30.447999999999997
- type: map_at_100
value: 31.417
- type: map_at_1000
value: 31.528
- type: map_at_3
value: 28.168
- type: map_at_5
value: 29.346
- type: mrr_at_1
value: 28.854000000000003
- type: mrr_at_10
value: 35.611
- type: mrr_at_100
value: 36.321
- type: mrr_at_1000
value: 36.378
- type: mrr_at_3
value: 33.726
- type: mrr_at_5
value: 34.745
- type: ndcg_at_1
value: 28.854000000000003
- type: ndcg_at_10
value: 35.052
- type: ndcg_at_100
value: 39.190999999999995
- type: ndcg_at_1000
value: 41.655
- type: ndcg_at_3
value: 31.684
- type: ndcg_at_5
value: 32.998
- type: precision_at_1
value: 28.854000000000003
- type: precision_at_10
value: 6.49
- type: precision_at_100
value: 1.057
- type: precision_at_1000
value: 0.153
- type: precision_at_3
value: 15.244
- type: precision_at_5
value: 10.599
- type: recall_at_1
value: 22.646
- type: recall_at_10
value: 43.482
- type: recall_at_100
value: 61.324
- type: recall_at_1000
value: 77.866
- type: recall_at_3
value: 33.106
- type: recall_at_5
value: 37.124
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGamingRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 35.061
- type: map_at_10
value: 46.216
- type: map_at_100
value: 47.318
- type: map_at_1000
value: 47.384
- type: map_at_3
value: 43.008
- type: map_at_5
value: 44.79
- type: mrr_at_1
value: 40.251
- type: mrr_at_10
value: 49.677
- type: mrr_at_100
value: 50.39
- type: mrr_at_1000
value: 50.429
- type: mrr_at_3
value: 46.792
- type: mrr_at_5
value: 48.449999999999996
- type: ndcg_at_1
value: 40.251
- type: ndcg_at_10
value: 51.99399999999999
- type: ndcg_at_100
value: 56.418
- type: ndcg_at_1000
value: 57.798
- type: ndcg_at_3
value: 46.192
- type: ndcg_at_5
value: 48.998000000000005
- type: precision_at_1
value: 40.251
- type: precision_at_10
value: 8.469999999999999
- type: precision_at_100
value: 1.159
- type: precision_at_1000
value: 0.133
- type: precision_at_3
value: 20.46
- type: precision_at_5
value: 14.332
- type: recall_at_1
value: 35.061
- type: recall_at_10
value: 65.818
- type: recall_at_100
value: 84.935
- type: recall_at_1000
value: 94.69300000000001
- type: recall_at_3
value: 50.300999999999995
- type: recall_at_5
value: 57.052
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGisRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 20.776
- type: map_at_10
value: 27.945999999999998
- type: map_at_100
value: 28.976000000000003
- type: map_at_1000
value: 29.073999999999998
- type: map_at_3
value: 25.673000000000002
- type: map_at_5
value: 26.96
- type: mrr_at_1
value: 22.486
- type: mrr_at_10
value: 29.756
- type: mrr_at_100
value: 30.735
- type: mrr_at_1000
value: 30.81
- type: mrr_at_3
value: 27.571
- type: mrr_at_5
value: 28.808
- type: ndcg_at_1
value: 22.486
- type: ndcg_at_10
value: 32.190000000000005
- type: ndcg_at_100
value: 37.61
- type: ndcg_at_1000
value: 40.116
- type: ndcg_at_3
value: 27.688000000000002
- type: ndcg_at_5
value: 29.87
- type: precision_at_1
value: 22.486
- type: precision_at_10
value: 5.028
- type: precision_at_100
value: 0.818
- type: precision_at_1000
value: 0.107
- type: precision_at_3
value: 11.827
- type: precision_at_5
value: 8.362
- type: recall_at_1
value: 20.776
- type: recall_at_10
value: 43.588
- type: recall_at_100
value: 69.139
- type: recall_at_1000
value: 88.144
- type: recall_at_3
value: 31.411
- type: recall_at_5
value: 36.655
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackMathematicaRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 12.994
- type: map_at_10
value: 19.747999999999998
- type: map_at_100
value: 20.877000000000002
- type: map_at_1000
value: 21.021
- type: map_at_3
value: 17.473
- type: map_at_5
value: 18.683
- type: mrr_at_1
value: 16.542
- type: mrr_at_10
value: 23.830000000000002
- type: mrr_at_100
value: 24.789
- type: mrr_at_1000
value: 24.877
- type: mrr_at_3
value: 21.476
- type: mrr_at_5
value: 22.838
- type: ndcg_at_1
value: 16.542
- type: ndcg_at_10
value: 24.422
- type: ndcg_at_100
value: 30.011
- type: ndcg_at_1000
value: 33.436
- type: ndcg_at_3
value: 20.061999999999998
- type: ndcg_at_5
value: 22.009999999999998
- type: precision_at_1
value: 16.542
- type: precision_at_10
value: 4.664
- type: precision_at_100
value: 0.876
- type: precision_at_1000
value: 0.132
- type: precision_at_3
value: 9.826
- type: precision_at_5
value: 7.2139999999999995
- type: recall_at_1
value: 12.994
- type: recall_at_10
value: 34.917
- type: recall_at_100
value: 59.455000000000005
- type: recall_at_1000
value: 83.87299999999999
- type: recall_at_3
value: 22.807
- type: recall_at_5
value: 27.773999999999997
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackPhysicsRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 24.85
- type: map_at_10
value: 35.285
- type: map_at_100
value: 36.592999999999996
- type: map_at_1000
value: 36.720000000000006
- type: map_at_3
value: 32.183
- type: map_at_5
value: 33.852
- type: mrr_at_1
value: 30.703000000000003
- type: mrr_at_10
value: 40.699000000000005
- type: mrr_at_100
value: 41.598
- type: mrr_at_1000
value: 41.654
- type: mrr_at_3
value: 38.080999999999996
- type: mrr_at_5
value: 39.655
- type: ndcg_at_1
value: 30.703000000000003
- type: ndcg_at_10
value: 41.422
- type: ndcg_at_100
value: 46.998
- type: ndcg_at_1000
value: 49.395
- type: ndcg_at_3
value: 36.353
- type: ndcg_at_5
value: 38.7
- type: precision_at_1
value: 30.703000000000003
- type: precision_at_10
value: 7.757
- type: precision_at_100
value: 1.2349999999999999
- type: precision_at_1000
value: 0.164
- type: precision_at_3
value: 17.613
- type: precision_at_5
value: 12.589
- type: recall_at_1
value: 24.85
- type: recall_at_10
value: 54.19500000000001
- type: recall_at_100
value: 77.697
- type: recall_at_1000
value: 93.35900000000001
- type: recall_at_3
value: 39.739999999999995
- type: recall_at_5
value: 46.03
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackProgrammersRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 19.844
- type: map_at_10
value: 28.663
- type: map_at_100
value: 30.013
- type: map_at_1000
value: 30.139
- type: map_at_3
value: 25.953
- type: map_at_5
value: 27.425
- type: mrr_at_1
value: 25.457
- type: mrr_at_10
value: 34.266000000000005
- type: mrr_at_100
value: 35.204
- type: mrr_at_1000
value: 35.27
- type: mrr_at_3
value: 31.791999999999998
- type: mrr_at_5
value: 33.213
- type: ndcg_at_1
value: 25.457
- type: ndcg_at_10
value: 34.266000000000005
- type: ndcg_at_100
value: 40.239999999999995
- type: ndcg_at_1000
value: 42.917
- type: ndcg_at_3
value: 29.593999999999998
- type: ndcg_at_5
value: 31.71
- type: precision_at_1
value: 25.457
- type: precision_at_10
value: 6.438000000000001
- type: precision_at_100
value: 1.1159999999999999
- type: precision_at_1000
value: 0.153
- type: precision_at_3
value: 14.46
- type: precision_at_5
value: 10.388
- type: recall_at_1
value: 19.844
- type: recall_at_10
value: 45.787
- type: recall_at_100
value: 71.523
- type: recall_at_1000
value: 89.689
- type: recall_at_3
value: 32.665
- type: recall_at_5
value: 38.292
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 21.601166666666668
- type: map_at_10
value: 29.434166666666666
- type: map_at_100
value: 30.5905
- type: map_at_1000
value: 30.716583333333343
- type: map_at_3
value: 26.962333333333333
- type: map_at_5
value: 28.287250000000004
- type: mrr_at_1
value: 25.84825
- type: mrr_at_10
value: 33.49966666666667
- type: mrr_at_100
value: 34.39425000000001
- type: mrr_at_1000
value: 34.46366666666667
- type: mrr_at_3
value: 31.256
- type: mrr_at_5
value: 32.52016666666667
- type: ndcg_at_1
value: 25.84825
- type: ndcg_at_10
value: 34.2975
- type: ndcg_at_100
value: 39.50983333333333
- type: ndcg_at_1000
value: 42.17958333333333
- type: ndcg_at_3
value: 30.00558333333333
- type: ndcg_at_5
value: 31.931416666666664
- type: precision_at_1
value: 25.84825
- type: precision_at_10
value: 6.075083333333334
- type: precision_at_100
value: 1.0205833333333334
- type: precision_at_1000
value: 0.14425
- type: precision_at_3
value: 13.903249999999998
- type: precision_at_5
value: 9.874999999999998
- type: recall_at_1
value: 21.601166666666668
- type: recall_at_10
value: 44.787333333333336
- type: recall_at_100
value: 67.89450000000001
- type: recall_at_1000
value: 86.62424999999999
- type: recall_at_3
value: 32.66375
- type: recall_at_5
value: 37.71825
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackStatsRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 19.804
- type: map_at_10
value: 25.983
- type: map_at_100
value: 26.956999999999997
- type: map_at_1000
value: 27.067999999999998
- type: map_at_3
value: 23.804
- type: map_at_5
value: 24.978
- type: mrr_at_1
value: 22.853
- type: mrr_at_10
value: 28.974
- type: mrr_at_100
value: 29.855999999999998
- type: mrr_at_1000
value: 29.936
- type: mrr_at_3
value: 26.866
- type: mrr_at_5
value: 28.032
- type: ndcg_at_1
value: 22.853
- type: ndcg_at_10
value: 29.993
- type: ndcg_at_100
value: 34.735
- type: ndcg_at_1000
value: 37.637
- type: ndcg_at_3
value: 25.863000000000003
- type: ndcg_at_5
value: 27.769
- type: precision_at_1
value: 22.853
- type: precision_at_10
value: 4.8469999999999995
- type: precision_at_100
value: 0.779
- type: precision_at_1000
value: 0.11
- type: precision_at_3
value: 11.35
- type: precision_at_5
value: 7.9750000000000005
- type: recall_at_1
value: 19.804
- type: recall_at_10
value: 39.616
- type: recall_at_100
value: 61.06399999999999
- type: recall_at_1000
value: 82.69800000000001
- type: recall_at_3
value: 28.012999999999998
- type: recall_at_5
value: 32.96
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackTexRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 13.156
- type: map_at_10
value: 18.734
- type: map_at_100
value: 19.721
- type: map_at_1000
value: 19.851
- type: map_at_3
value: 17.057
- type: map_at_5
value: 17.941
- type: mrr_at_1
value: 16.07
- type: mrr_at_10
value: 22.113
- type: mrr_at_100
value: 23.021
- type: mrr_at_1000
value: 23.108
- type: mrr_at_3
value: 20.429
- type: mrr_at_5
value: 21.332
- type: ndcg_at_1
value: 16.07
- type: ndcg_at_10
value: 22.427
- type: ndcg_at_100
value: 27.277
- type: ndcg_at_1000
value: 30.525000000000002
- type: ndcg_at_3
value: 19.374
- type: ndcg_at_5
value: 20.695
- type: precision_at_1
value: 16.07
- type: precision_at_10
value: 4.1259999999999994
- type: precision_at_100
value: 0.769
- type: precision_at_1000
value: 0.122
- type: precision_at_3
value: 9.325999999999999
- type: precision_at_5
value: 6.683
- type: recall_at_1
value: 13.156
- type: recall_at_10
value: 30.223
- type: recall_at_100
value: 52.012
- type: recall_at_1000
value: 75.581
- type: recall_at_3
value: 21.508
- type: recall_at_5
value: 24.975
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackUnixRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 22.14
- type: map_at_10
value: 28.961
- type: map_at_100
value: 29.996000000000002
- type: map_at_1000
value: 30.112
- type: map_at_3
value: 26.540000000000003
- type: map_at_5
value: 27.916999999999998
- type: mrr_at_1
value: 25.746000000000002
- type: mrr_at_10
value: 32.936
- type: mrr_at_100
value: 33.811
- type: mrr_at_1000
value: 33.887
- type: mrr_at_3
value: 30.55
- type: mrr_at_5
value: 32.08
- type: ndcg_at_1
value: 25.746000000000002
- type: ndcg_at_10
value: 33.536
- type: ndcg_at_100
value: 38.830999999999996
- type: ndcg_at_1000
value: 41.644999999999996
- type: ndcg_at_3
value: 29.004
- type: ndcg_at_5
value: 31.284
- type: precision_at_1
value: 25.746000000000002
- type: precision_at_10
value: 5.569
- type: precision_at_100
value: 0.9259999999999999
- type: precision_at_1000
value: 0.128
- type: precision_at_3
value: 12.748999999999999
- type: precision_at_5
value: 9.216000000000001
- type: recall_at_1
value: 22.14
- type: recall_at_10
value: 43.628
- type: recall_at_100
value: 67.581
- type: recall_at_1000
value: 87.737
- type: recall_at_3
value: 31.579
- type: recall_at_5
value: 37.12
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackWebmastersRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 22.384
- type: map_at_10
value: 30.156
- type: map_at_100
value: 31.728
- type: map_at_1000
value: 31.971
- type: map_at_3
value: 27.655
- type: map_at_5
value: 28.965000000000003
- type: mrr_at_1
value: 27.075
- type: mrr_at_10
value: 34.894
- type: mrr_at_100
value: 36
- type: mrr_at_1000
value: 36.059000000000005
- type: mrr_at_3
value: 32.708
- type: mrr_at_5
value: 33.893
- type: ndcg_at_1
value: 27.075
- type: ndcg_at_10
value: 35.58
- type: ndcg_at_100
value: 41.597
- type: ndcg_at_1000
value: 44.529999999999994
- type: ndcg_at_3
value: 31.628
- type: ndcg_at_5
value: 33.333
- type: precision_at_1
value: 27.075
- type: precision_at_10
value: 6.9959999999999996
- type: precision_at_100
value: 1.431
- type: precision_at_1000
value: 0.23800000000000002
- type: precision_at_3
value: 15.02
- type: precision_at_5
value: 10.909
- type: recall_at_1
value: 22.384
- type: recall_at_10
value: 45.052
- type: recall_at_100
value: 72.441
- type: recall_at_1000
value: 91.047
- type: recall_at_3
value: 33.617000000000004
- type: recall_at_5
value: 38.171
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackWordpressRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 16.032
- type: map_at_10
value: 22.323
- type: map_at_100
value: 23.317
- type: map_at_1000
value: 23.419999999999998
- type: map_at_3
value: 20.064999999999998
- type: map_at_5
value: 21.246000000000002
- type: mrr_at_1
value: 17.375
- type: mrr_at_10
value: 24.157999999999998
- type: mrr_at_100
value: 25.108000000000004
- type: mrr_at_1000
value: 25.197999999999997
- type: mrr_at_3
value: 21.996
- type: mrr_at_5
value: 23.152
- type: ndcg_at_1
value: 17.375
- type: ndcg_at_10
value: 26.316
- type: ndcg_at_100
value: 31.302000000000003
- type: ndcg_at_1000
value: 34.143
- type: ndcg_at_3
value: 21.914
- type: ndcg_at_5
value: 23.896
- type: precision_at_1
value: 17.375
- type: precision_at_10
value: 4.233
- type: precision_at_100
value: 0.713
- type: precision_at_1000
value: 0.10200000000000001
- type: precision_at_3
value: 9.365
- type: precision_at_5
value: 6.728000000000001
- type: recall_at_1
value: 16.032
- type: recall_at_10
value: 36.944
- type: recall_at_100
value: 59.745000000000005
- type: recall_at_1000
value: 81.101
- type: recall_at_3
value: 25.096
- type: recall_at_5
value: 29.963
- task:
type: Retrieval
dataset:
type: climate-fever
name: MTEB ClimateFEVER
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 9.656
- type: map_at_10
value: 17.578
- type: map_at_100
value: 19.38
- type: map_at_1000
value: 19.552
- type: map_at_3
value: 14.544
- type: map_at_5
value: 15.914
- type: mrr_at_1
value: 21.041999999999998
- type: mrr_at_10
value: 33.579
- type: mrr_at_100
value: 34.483000000000004
- type: mrr_at_1000
value: 34.526
- type: mrr_at_3
value: 30
- type: mrr_at_5
value: 31.813999999999997
- type: ndcg_at_1
value: 21.041999999999998
- type: ndcg_at_10
value: 25.563999999999997
- type: ndcg_at_100
value: 32.714
- type: ndcg_at_1000
value: 35.943000000000005
- type: ndcg_at_3
value: 20.357
- type: ndcg_at_5
value: 21.839
- type: precision_at_1
value: 21.041999999999998
- type: precision_at_10
value: 8.319
- type: precision_at_100
value: 1.593
- type: precision_at_1000
value: 0.219
- type: precision_at_3
value: 15.440000000000001
- type: precision_at_5
value: 11.792
- type: recall_at_1
value: 9.656
- type: recall_at_10
value: 32.023
- type: recall_at_100
value: 56.812
- type: recall_at_1000
value: 75.098
- type: recall_at_3
value: 19.455
- type: recall_at_5
value: 23.68
- task:
type: Retrieval
dataset:
type: C-MTEB/CmedqaRetrieval
name: MTEB CmedqaRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 13.084999999999999
- type: map_at_10
value: 19.389
- type: map_at_100
value: 20.761
- type: map_at_1000
value: 20.944
- type: map_at_3
value: 17.273
- type: map_at_5
value: 18.37
- type: mrr_at_1
value: 20.955
- type: mrr_at_10
value: 26.741999999999997
- type: mrr_at_100
value: 27.724
- type: mrr_at_1000
value: 27.819
- type: mrr_at_3
value: 24.881
- type: mrr_at_5
value: 25.833000000000002
- type: ndcg_at_1
value: 20.955
- type: ndcg_at_10
value: 23.905
- type: ndcg_at_100
value: 30.166999999999998
- type: ndcg_at_1000
value: 34.202
- type: ndcg_at_3
value: 20.854
- type: ndcg_at_5
value: 21.918000000000003
- type: precision_at_1
value: 20.955
- type: precision_at_10
value: 5.479
- type: precision_at_100
value: 1.065
- type: precision_at_1000
value: 0.159
- type: precision_at_3
value: 11.960999999999999
- type: precision_at_5
value: 8.647
- type: recall_at_1
value: 13.084999999999999
- type: recall_at_10
value: 30.202
- type: recall_at_100
value: 56.579
- type: recall_at_1000
value: 84.641
- type: recall_at_3
value: 20.751
- type: recall_at_5
value: 24.317
- task:
type: PairClassification
dataset:
type: C-MTEB/CMNLI
name: MTEB Cmnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 72.8322309079976
- type: cos_sim_ap
value: 81.34356949111096
- type: cos_sim_f1
value: 74.88546438983758
- type: cos_sim_precision
value: 67.50516238032664
- type: cos_sim_recall
value: 84.07762450315643
- type: dot_accuracy
value: 69.28442573662056
- type: dot_ap
value: 74.87961278837321
- type: dot_f1
value: 72.20502901353966
- type: dot_precision
value: 61.5701797789873
- type: dot_recall
value: 87.2808043020809
- type: euclidean_accuracy
value: 71.99037883343355
- type: euclidean_ap
value: 80.70039825164011
- type: euclidean_f1
value: 74.23149154887813
- type: euclidean_precision
value: 64.29794520547945
- type: euclidean_recall
value: 87.79518353986438
- type: manhattan_accuracy
value: 72.0625375826819
- type: manhattan_ap
value: 80.78886354854423
- type: manhattan_f1
value: 74.20842299415924
- type: manhattan_precision
value: 66.0525355709595
- type: manhattan_recall
value: 84.66214636427402
- type: max_accuracy
value: 72.8322309079976
- type: max_ap
value: 81.34356949111096
- type: max_f1
value: 74.88546438983758
- task:
type: Retrieval
dataset:
type: C-MTEB/CovidRetrieval
name: MTEB CovidRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 54.847
- type: map_at_10
value: 63.736000000000004
- type: map_at_100
value: 64.302
- type: map_at_1000
value: 64.319
- type: map_at_3
value: 61.565000000000005
- type: map_at_5
value: 62.671
- type: mrr_at_1
value: 54.900000000000006
- type: mrr_at_10
value: 63.744
- type: mrr_at_100
value: 64.287
- type: mrr_at_1000
value: 64.30399999999999
- type: mrr_at_3
value: 61.590999999999994
- type: mrr_at_5
value: 62.724000000000004
- type: ndcg_at_1
value: 55.005
- type: ndcg_at_10
value: 68.142
- type: ndcg_at_100
value: 70.95
- type: ndcg_at_1000
value: 71.40100000000001
- type: ndcg_at_3
value: 63.641999999999996
- type: ndcg_at_5
value: 65.62599999999999
- type: precision_at_1
value: 55.005
- type: precision_at_10
value: 8.272
- type: precision_at_100
value: 0.963
- type: precision_at_1000
value: 0.1
- type: precision_at_3
value: 23.288
- type: precision_at_5
value: 14.963000000000001
- type: recall_at_1
value: 54.847
- type: recall_at_10
value: 81.955
- type: recall_at_100
value: 95.258
- type: recall_at_1000
value: 98.84100000000001
- type: recall_at_3
value: 69.547
- type: recall_at_5
value: 74.315
- task:
type: Retrieval
dataset:
type: dbpedia-entity
name: MTEB DBPedia
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 7.2620000000000005
- type: map_at_10
value: 15.196000000000002
- type: map_at_100
value: 19.454
- type: map_at_1000
value: 20.445
- type: map_at_3
value: 11.532
- type: map_at_5
value: 13.053999999999998
- type: mrr_at_1
value: 57.49999999999999
- type: mrr_at_10
value: 66.661
- type: mrr_at_100
value: 67.086
- type: mrr_at_1000
value: 67.105
- type: mrr_at_3
value: 64.625
- type: mrr_at_5
value: 65.962
- type: ndcg_at_1
value: 46.125
- type: ndcg_at_10
value: 32.609
- type: ndcg_at_100
value: 34.611999999999995
- type: ndcg_at_1000
value: 40.836
- type: ndcg_at_3
value: 37.513000000000005
- type: ndcg_at_5
value: 34.699999999999996
- type: precision_at_1
value: 57.49999999999999
- type: precision_at_10
value: 24.975
- type: precision_at_100
value: 6.9830000000000005
- type: precision_at_1000
value: 1.505
- type: precision_at_3
value: 40.75
- type: precision_at_5
value: 33.2
- type: recall_at_1
value: 7.2620000000000005
- type: recall_at_10
value: 20.341
- type: recall_at_100
value: 38.690999999999995
- type: recall_at_1000
value: 58.879000000000005
- type: recall_at_3
value: 12.997
- type: recall_at_5
value: 15.628
- task:
type: Retrieval
dataset:
type: C-MTEB/DuRetrieval
name: MTEB DuRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 20.86
- type: map_at_10
value: 62.28
- type: map_at_100
value: 65.794
- type: map_at_1000
value: 65.903
- type: map_at_3
value: 42.616
- type: map_at_5
value: 53.225
- type: mrr_at_1
value: 76.75
- type: mrr_at_10
value: 83.387
- type: mrr_at_100
value: 83.524
- type: mrr_at_1000
value: 83.531
- type: mrr_at_3
value: 82.592
- type: mrr_at_5
value: 83.07900000000001
- type: ndcg_at_1
value: 76.75
- type: ndcg_at_10
value: 72.83500000000001
- type: ndcg_at_100
value: 77.839
- type: ndcg_at_1000
value: 78.976
- type: ndcg_at_3
value: 70.977
- type: ndcg_at_5
value: 69.419
- type: precision_at_1
value: 76.75
- type: precision_at_10
value: 35.825
- type: precision_at_100
value: 4.507
- type: precision_at_1000
value: 0.47800000000000004
- type: precision_at_3
value: 63.733
- type: precision_at_5
value: 53.44
- type: recall_at_1
value: 20.86
- type: recall_at_10
value: 75.115
- type: recall_at_100
value: 90.47699999999999
- type: recall_at_1000
value: 96.304
- type: recall_at_3
value: 45.976
- type: recall_at_5
value: 59.971
- task:
type: Retrieval
dataset:
type: C-MTEB/EcomRetrieval
name: MTEB EcomRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 37.8
- type: map_at_10
value: 47.154
- type: map_at_100
value: 48.012
- type: map_at_1000
value: 48.044
- type: map_at_3
value: 44.667
- type: map_at_5
value: 45.992
- type: mrr_at_1
value: 37.8
- type: mrr_at_10
value: 47.154
- type: mrr_at_100
value: 48.012
- type: mrr_at_1000
value: 48.044
- type: mrr_at_3
value: 44.667
- type: mrr_at_5
value: 45.992
- type: ndcg_at_1
value: 37.8
- type: ndcg_at_10
value: 52.025
- type: ndcg_at_100
value: 56.275
- type: ndcg_at_1000
value: 57.174
- type: ndcg_at_3
value: 46.861999999999995
- type: ndcg_at_5
value: 49.229
- type: precision_at_1
value: 37.8
- type: precision_at_10
value: 6.75
- type: precision_at_100
value: 0.8750000000000001
- type: precision_at_1000
value: 0.095
- type: precision_at_3
value: 17.732999999999997
- type: precision_at_5
value: 11.78
- type: recall_at_1
value: 37.8
- type: recall_at_10
value: 67.5
- type: recall_at_100
value: 87.5
- type: recall_at_1000
value: 94.69999999999999
- type: recall_at_3
value: 53.2
- type: recall_at_5
value: 58.9
- task:
type: Classification
dataset:
type: mteb/emotion
name: MTEB EmotionClassification
config: default
split: test
revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37
metrics:
- type: accuracy
value: 46.845
- type: f1
value: 42.70952656074019
- task:
type: Retrieval
dataset:
type: fever
name: MTEB FEVER
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 50.058
- type: map_at_10
value: 61.295
- type: map_at_100
value: 61.82
- type: map_at_1000
value: 61.843
- type: map_at_3
value: 58.957
- type: map_at_5
value: 60.467999999999996
- type: mrr_at_1
value: 54.05
- type: mrr_at_10
value: 65.52900000000001
- type: mrr_at_100
value: 65.984
- type: mrr_at_1000
value: 65.999
- type: mrr_at_3
value: 63.286
- type: mrr_at_5
value: 64.777
- type: ndcg_at_1
value: 54.05
- type: ndcg_at_10
value: 67.216
- type: ndcg_at_100
value: 69.594
- type: ndcg_at_1000
value: 70.13000000000001
- type: ndcg_at_3
value: 62.778999999999996
- type: ndcg_at_5
value: 65.36
- type: precision_at_1
value: 54.05
- type: precision_at_10
value: 8.924
- type: precision_at_100
value: 1.019
- type: precision_at_1000
value: 0.108
- type: precision_at_3
value: 25.218
- type: precision_at_5
value: 16.547
- type: recall_at_1
value: 50.058
- type: recall_at_10
value: 81.39699999999999
- type: recall_at_100
value: 92.022
- type: recall_at_1000
value: 95.877
- type: recall_at_3
value: 69.485
- type: recall_at_5
value: 75.833
- task:
type: Retrieval
dataset:
type: fiqa
name: MTEB FiQA2018
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 15.078
- type: map_at_10
value: 24.162
- type: map_at_100
value: 25.818
- type: map_at_1000
value: 26.009
- type: map_at_3
value: 20.706
- type: map_at_5
value: 22.542
- type: mrr_at_1
value: 30.709999999999997
- type: mrr_at_10
value: 38.828
- type: mrr_at_100
value: 39.794000000000004
- type: mrr_at_1000
value: 39.843
- type: mrr_at_3
value: 36.163000000000004
- type: mrr_at_5
value: 37.783
- type: ndcg_at_1
value: 30.709999999999997
- type: ndcg_at_10
value: 31.290000000000003
- type: ndcg_at_100
value: 38.051
- type: ndcg_at_1000
value: 41.487
- type: ndcg_at_3
value: 27.578999999999997
- type: ndcg_at_5
value: 28.799000000000003
- type: precision_at_1
value: 30.709999999999997
- type: precision_at_10
value: 8.92
- type: precision_at_100
value: 1.5599999999999998
- type: precision_at_1000
value: 0.219
- type: precision_at_3
value: 18.416
- type: precision_at_5
value: 13.827
- type: recall_at_1
value: 15.078
- type: recall_at_10
value: 37.631
- type: recall_at_100
value: 63.603
- type: recall_at_1000
value: 84.121
- type: recall_at_3
value: 24.438
- type: recall_at_5
value: 29.929
- task:
type: Retrieval
dataset:
type: hotpotqa
name: MTEB HotpotQA
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 31.202
- type: map_at_10
value: 42.653
- type: map_at_100
value: 43.411
- type: map_at_1000
value: 43.479
- type: map_at_3
value: 40.244
- type: map_at_5
value: 41.736000000000004
- type: mrr_at_1
value: 62.404
- type: mrr_at_10
value: 69.43599999999999
- type: mrr_at_100
value: 69.788
- type: mrr_at_1000
value: 69.809
- type: mrr_at_3
value: 68.12700000000001
- type: mrr_at_5
value: 68.961
- type: ndcg_at_1
value: 62.404
- type: ndcg_at_10
value: 51.665000000000006
- type: ndcg_at_100
value: 54.623
- type: ndcg_at_1000
value: 56.154
- type: ndcg_at_3
value: 47.861
- type: ndcg_at_5
value: 49.968
- type: precision_at_1
value: 62.404
- type: precision_at_10
value: 10.57
- type: precision_at_100
value: 1.2890000000000001
- type: precision_at_1000
value: 0.149
- type: precision_at_3
value: 29.624
- type: precision_at_5
value: 19.441
- type: recall_at_1
value: 31.202
- type: recall_at_10
value: 52.849000000000004
- type: recall_at_100
value: 64.47
- type: recall_at_1000
value: 74.74
- type: recall_at_3
value: 44.436
- type: recall_at_5
value: 48.602000000000004
- task:
type: Classification
dataset:
type: C-MTEB/IFlyTek-classification
name: MTEB IFlyTek
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 43.51673720661793
- type: f1
value: 35.81126468608715
- task:
type: Classification
dataset:
type: mteb/imdb
name: MTEB ImdbClassification
config: default
split: test
revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7
metrics:
- type: accuracy
value: 74.446
- type: ap
value: 68.71359666500074
- type: f1
value: 74.32080431056023
- task:
type: Classification
dataset:
type: C-MTEB/JDReview-classification
name: MTEB JDReview
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 81.08818011257036
- type: ap
value: 43.68599141287235
- type: f1
value: 74.37787266346157
- task:
type: STS
dataset:
type: C-MTEB/LCQMC
name: MTEB LCQMC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 65.9116523539515
- type: cos_sim_spearman
value: 72.79966865646485
- type: euclidean_pearson
value: 71.4995885009818
- type: euclidean_spearman
value: 72.91799793240196
- type: manhattan_pearson
value: 71.83065174544116
- type: manhattan_spearman
value: 73.22568775268935
- task:
type: Retrieval
dataset:
type: C-MTEB/MMarcoRetrieval
name: MTEB MMarcoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 61.79900000000001
- type: map_at_10
value: 70.814
- type: map_at_100
value: 71.22500000000001
- type: map_at_1000
value: 71.243
- type: map_at_3
value: 68.795
- type: map_at_5
value: 70.12
- type: mrr_at_1
value: 63.910999999999994
- type: mrr_at_10
value: 71.437
- type: mrr_at_100
value: 71.807
- type: mrr_at_1000
value: 71.82300000000001
- type: mrr_at_3
value: 69.65599999999999
- type: mrr_at_5
value: 70.821
- type: ndcg_at_1
value: 63.910999999999994
- type: ndcg_at_10
value: 74.664
- type: ndcg_at_100
value: 76.545
- type: ndcg_at_1000
value: 77.00099999999999
- type: ndcg_at_3
value: 70.838
- type: ndcg_at_5
value: 73.076
- type: precision_at_1
value: 63.910999999999994
- type: precision_at_10
value: 9.139999999999999
- type: precision_at_100
value: 1.008
- type: precision_at_1000
value: 0.105
- type: precision_at_3
value: 26.729000000000003
- type: precision_at_5
value: 17.232
- type: recall_at_1
value: 61.79900000000001
- type: recall_at_10
value: 85.941
- type: recall_at_100
value: 94.514
- type: recall_at_1000
value: 98.04899999999999
- type: recall_at_3
value: 75.85499999999999
- type: recall_at_5
value: 81.15599999999999
- task:
type: Retrieval
dataset:
type: msmarco
name: MTEB MSMARCO
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 20.079
- type: map_at_10
value: 31.735000000000003
- type: map_at_100
value: 32.932
- type: map_at_1000
value: 32.987
- type: map_at_3
value: 28.216
- type: map_at_5
value: 30.127
- type: mrr_at_1
value: 20.688000000000002
- type: mrr_at_10
value: 32.357
- type: mrr_at_100
value: 33.487
- type: mrr_at_1000
value: 33.536
- type: mrr_at_3
value: 28.887
- type: mrr_at_5
value: 30.764000000000003
- type: ndcg_at_1
value: 20.688000000000002
- type: ndcg_at_10
value: 38.266
- type: ndcg_at_100
value: 44.105
- type: ndcg_at_1000
value: 45.554
- type: ndcg_at_3
value: 31.046000000000003
- type: ndcg_at_5
value: 34.44
- type: precision_at_1
value: 20.688000000000002
- type: precision_at_10
value: 6.0920000000000005
- type: precision_at_100
value: 0.903
- type: precision_at_1000
value: 0.10300000000000001
- type: precision_at_3
value: 13.338
- type: precision_at_5
value: 9.725
- type: recall_at_1
value: 20.079
- type: recall_at_10
value: 58.315
- type: recall_at_100
value: 85.50999999999999
- type: recall_at_1000
value: 96.72800000000001
- type: recall_at_3
value: 38.582
- type: recall_at_5
value: 46.705999999999996
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (en)
config: en
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 92.18422252621978
- type: f1
value: 91.82800582693794
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (de)
config: de
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 74.63792617638771
- type: f1
value: 73.13966942566492
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (es)
config: es
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 92.07138092061375
- type: f1
value: 91.58983799467875
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (fr)
config: fr
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 89.19824616348262
- type: f1
value: 89.06796384273765
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (hi)
config: hi
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 88.54069558981713
- type: f1
value: 87.83448658971352
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (th)
config: th
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 55.63471971066908
- type: f1
value: 53.84017845089774
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (en)
config: en
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 70.29867761057912
- type: f1
value: 52.76509068762125
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (de)
config: de
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 53.39814032121725
- type: f1
value: 34.27161745913036
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (es)
config: es
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 71.33422281521014
- type: f1
value: 52.171603212251384
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (fr)
config: fr
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 66.6019417475728
- type: f1
value: 49.212091278323975
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (hi)
config: hi
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 66.73001075654356
- type: f1
value: 45.97084834271623
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (th)
config: th
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 42.13381555153707
- type: f1
value: 27.222558885215964
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (af)
config: af
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 44.97982515131137
- type: f1
value: 43.08686679862984
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (am)
config: am
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 25.353059852051107
- type: f1
value: 24.56465252790922
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ar)
config: ar
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 57.078009414929376
- type: f1
value: 54.933541125458795
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (az)
config: az
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 39.10558170813719
- type: f1
value: 39.15270496151374
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (bn)
config: bn
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 61.368527236045736
- type: f1
value: 58.65381984021665
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (cy)
config: cy
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 42.96906523201076
- type: f1
value: 41.88085083446726
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (da)
config: da
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 49.54270342972428
- type: f1
value: 48.44206747172913
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (de)
config: de
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 50.93140551445864
- type: f1
value: 47.40396853548677
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (el)
config: el
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 40.09414929388029
- type: f1
value: 38.27158057191927
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (en)
config: en
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 67.93207800941494
- type: f1
value: 66.50282035579518
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (es)
config: es
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 63.81304640215198
- type: f1
value: 62.51979490279083
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (fa)
config: fa
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 49.05850706119704
- type: f1
value: 47.49872899848797
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (fi)
config: fi
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 42.57901815736382
- type: f1
value: 40.386069905109956
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (fr)
config: fr
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 65.33960995292534
- type: f1
value: 63.96475759829612
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (he)
config: he
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 37.14862138533962
- type: f1
value: 35.954583318470384
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (hi)
config: hi
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 62.88836583725621
- type: f1
value: 61.139092331276856
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (hu)
config: hu
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 41.62071284465366
- type: f1
value: 40.23779890980788
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (hy)
config: hy
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 32.982515131136516
- type: f1
value: 31.82828709111086
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (id)
config: id
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 62.11499663752521
- type: f1
value: 60.307651330689716
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (is)
config: is
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 41.039004707464684
- type: f1
value: 39.531615524370686
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (it)
config: it
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 55.8338937457969
- type: f1
value: 54.86425916837068
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ja)
config: ja
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 58.83322125084061
- type: f1
value: 56.52595986400214
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (jv)
config: jv
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 49.31069266980497
- type: f1
value: 47.241381065322265
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ka)
config: ka
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 26.432414256893072
- type: f1
value: 25.787833437725848
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (km)
config: km
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 28.76933422999327
- type: f1
value: 27.34778980866226
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (kn)
config: kn
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 52.33019502353733
- type: f1
value: 49.49897965390079
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ko)
config: ko
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 46.930060524546064
- type: f1
value: 44.71215467580226
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (lv)
config: lv
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 44.25689307330195
- type: f1
value: 43.61087006714549
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ml)
config: ml
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 57.74714189643577
- type: f1
value: 54.571431590522735
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (mn)
config: mn
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 33.30531271015468
- type: f1
value: 33.4982889160085
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ms)
config: ms
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 55.699394754539334
- type: f1
value: 54.00478534026828
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (my)
config: my
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 27.38735709482179
- type: f1
value: 26.139112212692474
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (nb)
config: nb
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 46.18359112306658
- type: f1
value: 45.298479798547106
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (nl)
config: nl
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 48.33557498318763
- type: f1
value: 46.102865846786294
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (pl)
config: pl
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 44.46872898453261
- type: f1
value: 42.43443803309795
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (pt)
config: pt
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 64.74445191661063
- type: f1
value: 63.453679590322174
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ro)
config: ro
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 48.41291190316072
- type: f1
value: 47.14401920664497
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ru)
config: ru
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 52.989240080699396
- type: f1
value: 50.91931775407477
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (sl)
config: sl
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 44.771351714862135
- type: f1
value: 42.90054169209577
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (sq)
config: sq
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 45.45393409549428
- type: f1
value: 45.027761715583146
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (sv)
config: sv
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 45.67585743106927
- type: f1
value: 44.45608727957947
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (sw)
config: sw
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 46.45595158036314
- type: f1
value: 44.70548836690419
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ta)
config: ta
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 55.4640215198386
- type: f1
value: 52.28532276735651
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (te)
config: te
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 51.408876933422995
- type: f1
value: 48.86454236156204
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (th)
config: th
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 39.19636852723604
- type: f1
value: 38.88247037601754
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (tl)
config: tl
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 48.53396099529254
- type: f1
value: 46.961492802320656
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (tr)
config: tr
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 39.509078681909884
- type: f1
value: 39.30973355583357
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (ur)
config: ur
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 54.717552118359116
- type: f1
value: 52.08348704897728
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (vi)
config: vi
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 62.007397444519164
- type: f1
value: 60.57772322803523
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-CN)
config: zh-CN
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 66.906523201076
- type: f1
value: 65.2730417732602
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-TW)
config: zh-TW
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 62.562205783456626
- type: f1
value: 62.3944953225828
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (af)
config: af
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 50.46738399462004
- type: f1
value: 48.277337351043066
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (am)
config: am
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 27.222595830531272
- type: f1
value: 26.15959037949326
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ar)
config: ar
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 65.4303967720242
- type: f1
value: 65.58227814316872
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (az)
config: az
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 40.736381977135174
- type: f1
value: 39.85702036251076
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (bn)
config: bn
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 67.64626765299259
- type: f1
value: 67.12298813657769
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (cy)
config: cy
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 43.940820443846675
- type: f1
value: 41.63412499587839
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (da)
config: da
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 52.5252185608608
- type: f1
value: 50.25821961669483
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (de)
config: de
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 56.67114996637525
- type: f1
value: 54.204117831814244
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (el)
config: el
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 41.8123739071957
- type: f1
value: 40.25676895490678
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (en)
config: en
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 75.71956960322798
- type: f1
value: 75.95126212201126
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (es)
config: es
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 71.7787491593813
- type: f1
value: 71.90678548502461
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (fa)
config: fa
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 49.95965030262274
- type: f1
value: 48.625859921623515
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (fi)
config: fi
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 41.005379959650305
- type: f1
value: 38.25957953711836
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (fr)
config: fr
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 71.99058507061198
- type: f1
value: 72.30034867942928
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (he)
config: he
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 36.691324815063886
- type: f1
value: 35.09762112518494
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (hi)
config: hi
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 69.27706792199058
- type: f1
value: 68.96935505580095
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (hu)
config: hu
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 44.31405514458642
- type: f1
value: 41.75837557089336
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (hy)
config: hy
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 33.63819771351715
- type: f1
value: 32.00999199645466
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (id)
config: id
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 68.98117014122394
- type: f1
value: 68.48993356947226
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (is)
config: is
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 42.10154673839946
- type: f1
value: 39.537580201439035
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (it)
config: it
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 60.27236045729657
- type: f1
value: 58.8041857941664
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ja)
config: ja
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 62.47814391392063
- type: f1
value: 61.4800551358116
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (jv)
config: jv
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 54.68392737054473
- type: f1
value: 53.28619831432411
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ka)
config: ka
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 27.215870880968396
- type: f1
value: 26.137784395348483
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (km)
config: km
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 32.1385339609953
- type: f1
value: 29.886918185071977
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (kn)
config: kn
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 57.94889038332213
- type: f1
value: 57.19252000109654
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ko)
config: ko
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 47.94552790854068
- type: f1
value: 46.21337507975437
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (lv)
config: lv
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 42.75722932078009
- type: f1
value: 40.62195245815035
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ml)
config: ml
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 62.84129119031607
- type: f1
value: 62.56205475932971
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (mn)
config: mn
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 33.21116341627438
- type: f1
value: 32.231827617771046
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ms)
config: ms
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 62.56893073301949
- type: f1
value: 60.94616552257348
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (my)
config: my
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 28.8399462004035
- type: f1
value: 27.8503615081592
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (nb)
config: nb
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 50.31607262945528
- type: f1
value: 47.993368005418205
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (nl)
config: nl
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 52.851378614660405
- type: f1
value: 50.444332639513824
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (pl)
config: pl
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 45.595158036314736
- type: f1
value: 44.241686886064755
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (pt)
config: pt
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 70.24209818426363
- type: f1
value: 70.48109122752663
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ro)
config: ro
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 52.73369199731002
- type: f1
value: 51.14034087602817
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ru)
config: ru
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 54.263618022864826
- type: f1
value: 53.3188846615122
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (sl)
config: sl
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 46.88634835238735
- type: f1
value: 45.257261686960796
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (sq)
config: sq
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 47.15534633490249
- type: f1
value: 45.218807618409215
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (sv)
config: sv
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 47.9119031607263
- type: f1
value: 45.96730030717468
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (sw)
config: sw
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 51.20040349697377
- type: f1
value: 49.113423730259214
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ta)
config: ta
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 61.8392737054472
- type: f1
value: 61.65834459536364
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (te)
config: te
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 59.791526563550775
- type: f1
value: 58.2891677685128
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (th)
config: th
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 41.62071284465366
- type: f1
value: 39.591525429243575
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (tl)
config: tl
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 50.46738399462004
- type: f1
value: 49.50612154409957
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (tr)
config: tr
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 43.41291190316072
- type: f1
value: 43.85070302174815
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (ur)
config: ur
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 60.15131136516476
- type: f1
value: 59.260012738676316
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (vi)
config: vi
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 68.98789509078682
- type: f1
value: 69.86968024553558
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-CN)
config: zh-CN
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 74.72091459314055
- type: f1
value: 74.69866015852224
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-TW)
config: zh-TW
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 71.7014122394082
- type: f1
value: 72.66856729607628
- task:
type: Retrieval
dataset:
type: C-MTEB/MedicalRetrieval
name: MTEB MedicalRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 35.8
- type: map_at_10
value: 40.949999999999996
- type: map_at_100
value: 41.455999999999996
- type: map_at_1000
value: 41.52
- type: map_at_3
value: 40.033
- type: map_at_5
value: 40.493
- type: mrr_at_1
value: 35.9
- type: mrr_at_10
value: 41
- type: mrr_at_100
value: 41.506
- type: mrr_at_1000
value: 41.57
- type: mrr_at_3
value: 40.083
- type: mrr_at_5
value: 40.543
- type: ndcg_at_1
value: 35.8
- type: ndcg_at_10
value: 43.269000000000005
- type: ndcg_at_100
value: 45.974
- type: ndcg_at_1000
value: 47.969
- type: ndcg_at_3
value: 41.339999999999996
- type: ndcg_at_5
value: 42.167
- type: precision_at_1
value: 35.8
- type: precision_at_10
value: 5.050000000000001
- type: precision_at_100
value: 0.637
- type: precision_at_1000
value: 0.08
- type: precision_at_3
value: 15.033
- type: precision_at_5
value: 9.42
- type: recall_at_1
value: 35.8
- type: recall_at_10
value: 50.5
- type: recall_at_100
value: 63.7
- type: recall_at_1000
value: 80
- type: recall_at_3
value: 45.1
- type: recall_at_5
value: 47.099999999999994
- task:
type: Clustering
dataset:
type: mteb/medrxiv-clustering-p2p
name: MTEB MedrxivClusteringP2P
config: default
split: test
revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73
metrics:
- type: v_measure
value: 29.43291218491871
- task:
type: Clustering
dataset:
type: mteb/medrxiv-clustering-s2s
name: MTEB MedrxivClusteringS2S
config: default
split: test
revision: 35191c8c0dca72d8ff3efcd72aa802307d469663
metrics:
- type: v_measure
value: 28.87018200800912
- task:
type: Reranking
dataset:
type: mteb/mind_small
name: MTEB MindSmallReranking
config: default
split: test
revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69
metrics:
- type: map
value: 30.51003589330728
- type: mrr
value: 31.57412386045135
- task:
type: Reranking
dataset:
type: C-MTEB/Mmarco-reranking
name: MTEB MMarcoReranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 26.136250989818222
- type: mrr
value: 25.00753968253968
- task:
type: Classification
dataset:
type: C-MTEB/MultilingualSentiment-classification
name: MTEB MultilingualSentiment
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 66.32999999999998
- type: f1
value: 66.2828795526323
- task:
type: Retrieval
dataset:
type: nfcorpus
name: MTEB NFCorpus
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 4.369
- type: map_at_10
value: 11.04
- type: map_at_100
value: 13.850000000000001
- type: map_at_1000
value: 15.290000000000001
- type: map_at_3
value: 8.014000000000001
- type: map_at_5
value: 9.4
- type: mrr_at_1
value: 39.938
- type: mrr_at_10
value: 49.043
- type: mrr_at_100
value: 49.775000000000006
- type: mrr_at_1000
value: 49.803999999999995
- type: mrr_at_3
value: 47.007
- type: mrr_at_5
value: 48.137
- type: ndcg_at_1
value: 37.461
- type: ndcg_at_10
value: 30.703000000000003
- type: ndcg_at_100
value: 28.686
- type: ndcg_at_1000
value: 37.809
- type: ndcg_at_3
value: 35.697
- type: ndcg_at_5
value: 33.428000000000004
- type: precision_at_1
value: 39.628
- type: precision_at_10
value: 23.250999999999998
- type: precision_at_100
value: 7.553999999999999
- type: precision_at_1000
value: 2.077
- type: precision_at_3
value: 34.159
- type: precision_at_5
value: 29.164
- type: recall_at_1
value: 4.369
- type: recall_at_10
value: 15.024000000000001
- type: recall_at_100
value: 30.642999999999997
- type: recall_at_1000
value: 62.537
- type: recall_at_3
value: 9.504999999999999
- type: recall_at_5
value: 11.89
- task:
type: Retrieval
dataset:
type: nq
name: MTEB NQ
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 26.161
- type: map_at_10
value: 39.126
- type: map_at_100
value: 40.201
- type: map_at_1000
value: 40.247
- type: map_at_3
value: 35.169
- type: map_at_5
value: 37.403
- type: mrr_at_1
value: 29.403000000000002
- type: mrr_at_10
value: 41.644999999999996
- type: mrr_at_100
value: 42.503
- type: mrr_at_1000
value: 42.535000000000004
- type: mrr_at_3
value: 38.321
- type: mrr_at_5
value: 40.265
- type: ndcg_at_1
value: 29.403000000000002
- type: ndcg_at_10
value: 46.155
- type: ndcg_at_100
value: 50.869
- type: ndcg_at_1000
value: 52.004
- type: ndcg_at_3
value: 38.65
- type: ndcg_at_5
value: 42.400999999999996
- type: precision_at_1
value: 29.403000000000002
- type: precision_at_10
value: 7.743
- type: precision_at_100
value: 1.0410000000000001
- type: precision_at_1000
value: 0.11499999999999999
- type: precision_at_3
value: 17.623
- type: precision_at_5
value: 12.764000000000001
- type: recall_at_1
value: 26.161
- type: recall_at_10
value: 65.155
- type: recall_at_100
value: 85.885
- type: recall_at_1000
value: 94.443
- type: recall_at_3
value: 45.592
- type: recall_at_5
value: 54.234
- task:
type: PairClassification
dataset:
type: C-MTEB/OCNLI
name: MTEB Ocnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 65.34921494315105
- type: cos_sim_ap
value: 68.58191894316523
- type: cos_sim_f1
value: 70.47294418406477
- type: cos_sim_precision
value: 59.07142857142858
- type: cos_sim_recall
value: 87.32840549102428
- type: dot_accuracy
value: 61.93827828911749
- type: dot_ap
value: 64.19230712895958
- type: dot_f1
value: 68.30769230769232
- type: dot_precision
value: 53.72050816696915
- type: dot_recall
value: 93.76979936642027
- type: euclidean_accuracy
value: 67.0817541959935
- type: euclidean_ap
value: 69.17499163875786
- type: euclidean_f1
value: 71.67630057803468
- type: euclidean_precision
value: 61.904761904761905
- type: euclidean_recall
value: 85.11087645195353
- type: manhattan_accuracy
value: 67.19003789929616
- type: manhattan_ap
value: 69.72684682556992
- type: manhattan_f1
value: 71.25396106835673
- type: manhattan_precision
value: 62.361331220285265
- type: manhattan_recall
value: 83.10454065469905
- type: max_accuracy
value: 67.19003789929616
- type: max_ap
value: 69.72684682556992
- type: max_f1
value: 71.67630057803468
- task:
type: Classification
dataset:
type: C-MTEB/OnlineShopping-classification
name: MTEB OnlineShopping
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 88.35000000000001
- type: ap
value: 85.45377991151882
- type: f1
value: 88.33274122313945
- task:
type: STS
dataset:
type: C-MTEB/PAWSX
name: MTEB PAWSX
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 13.700131726042631
- type: cos_sim_spearman
value: 15.663851577320184
- type: euclidean_pearson
value: 17.869909454798112
- type: euclidean_spearman
value: 16.09518673735175
- type: manhattan_pearson
value: 18.030818366917593
- type: manhattan_spearman
value: 16.34096397687474
- task:
type: STS
dataset:
type: C-MTEB/QBQTC
name: MTEB QBQTC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 30.200343733562946
- type: cos_sim_spearman
value: 32.645434631834966
- type: euclidean_pearson
value: 32.612030669583234
- type: euclidean_spearman
value: 34.67603837485763
- type: manhattan_pearson
value: 32.6673080122766
- type: manhattan_spearman
value: 34.8163622783733
- task:
type: Retrieval
dataset:
type: quora
name: MTEB QuoraRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 69.321
- type: map_at_10
value: 83.07
- type: map_at_100
value: 83.737
- type: map_at_1000
value: 83.758
- type: map_at_3
value: 80.12700000000001
- type: map_at_5
value: 81.97
- type: mrr_at_1
value: 79.74
- type: mrr_at_10
value: 86.22
- type: mrr_at_100
value: 86.345
- type: mrr_at_1000
value: 86.347
- type: mrr_at_3
value: 85.172
- type: mrr_at_5
value: 85.89099999999999
- type: ndcg_at_1
value: 79.77
- type: ndcg_at_10
value: 87.01299999999999
- type: ndcg_at_100
value: 88.382
- type: ndcg_at_1000
value: 88.53
- type: ndcg_at_3
value: 84.04
- type: ndcg_at_5
value: 85.68
- type: precision_at_1
value: 79.77
- type: precision_at_10
value: 13.211999999999998
- type: precision_at_100
value: 1.52
- type: precision_at_1000
value: 0.157
- type: precision_at_3
value: 36.730000000000004
- type: precision_at_5
value: 24.21
- type: recall_at_1
value: 69.321
- type: recall_at_10
value: 94.521
- type: recall_at_100
value: 99.258
- type: recall_at_1000
value: 99.97200000000001
- type: recall_at_3
value: 85.97200000000001
- type: recall_at_5
value: 90.589
- task:
type: Clustering
dataset:
type: mteb/reddit-clustering
name: MTEB RedditClustering
config: default
split: test
revision: 24640382cdbf8abc73003fb0fa6d111a705499eb
metrics:
- type: v_measure
value: 44.51751457277441
- task:
type: Clustering
dataset:
type: mteb/reddit-clustering-p2p
name: MTEB RedditClusteringP2P
config: default
split: test
revision: 282350215ef01743dc01b456c7f5241fa8937f16
metrics:
- type: v_measure
value: 53.60727449352775
- task:
type: Retrieval
dataset:
type: scidocs
name: MTEB SCIDOCS
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 4.058
- type: map_at_10
value: 9.995999999999999
- type: map_at_100
value: 11.738
- type: map_at_1000
value: 11.999
- type: map_at_3
value: 7.353999999999999
- type: map_at_5
value: 8.68
- type: mrr_at_1
value: 20
- type: mrr_at_10
value: 30.244
- type: mrr_at_100
value: 31.378
- type: mrr_at_1000
value: 31.445
- type: mrr_at_3
value: 26.933
- type: mrr_at_5
value: 28.748
- type: ndcg_at_1
value: 20
- type: ndcg_at_10
value: 17.235
- type: ndcg_at_100
value: 24.241
- type: ndcg_at_1000
value: 29.253
- type: ndcg_at_3
value: 16.542
- type: ndcg_at_5
value: 14.386
- type: precision_at_1
value: 20
- type: precision_at_10
value: 8.9
- type: precision_at_100
value: 1.8929999999999998
- type: precision_at_1000
value: 0.31
- type: precision_at_3
value: 15.567
- type: precision_at_5
value: 12.620000000000001
- type: recall_at_1
value: 4.058
- type: recall_at_10
value: 18.062
- type: recall_at_100
value: 38.440000000000005
- type: recall_at_1000
value: 63.044999999999995
- type: recall_at_3
value: 9.493
- type: recall_at_5
value: 12.842
- task:
type: STS
dataset:
type: mteb/sickr-sts
name: MTEB SICK-R
config: default
split: test
revision: a6ea5a8cab320b040a23452cc28066d9beae2cee
metrics:
- type: cos_sim_pearson
value: 85.36702895231333
- type: cos_sim_spearman
value: 79.91790376084445
- type: euclidean_pearson
value: 81.58989754571684
- type: euclidean_spearman
value: 79.43876559435684
- type: manhattan_pearson
value: 81.5041355053572
- type: manhattan_spearman
value: 79.35411927652234
- task:
type: STS
dataset:
type: mteb/sts12-sts
name: MTEB STS12
config: default
split: test
revision: a0d554a64d88156834ff5ae9920b964011b16384
metrics:
- type: cos_sim_pearson
value: 83.77166067512005
- type: cos_sim_spearman
value: 75.7961015562481
- type: euclidean_pearson
value: 82.03845114943047
- type: euclidean_spearman
value: 78.75422268992615
- type: manhattan_pearson
value: 82.11841609875198
- type: manhattan_spearman
value: 78.79349601386468
- task:
type: STS
dataset:
type: mteb/sts13-sts
name: MTEB STS13
config: default
split: test
revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca
metrics:
- type: cos_sim_pearson
value: 83.28403658061106
- type: cos_sim_spearman
value: 83.61682237930194
- type: euclidean_pearson
value: 84.50220149144553
- type: euclidean_spearman
value: 85.01944483089126
- type: manhattan_pearson
value: 84.5526583345216
- type: manhattan_spearman
value: 85.06290695547032
- task:
type: STS
dataset:
type: mteb/sts14-sts
name: MTEB STS14
config: default
split: test
revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375
metrics:
- type: cos_sim_pearson
value: 82.66893263127082
- type: cos_sim_spearman
value: 78.73277873007592
- type: euclidean_pearson
value: 80.78325001462842
- type: euclidean_spearman
value: 79.1692321029638
- type: manhattan_pearson
value: 80.82812137898084
- type: manhattan_spearman
value: 79.23433932409523
- task:
type: STS
dataset:
type: mteb/sts15-sts
name: MTEB STS15
config: default
split: test
revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3
metrics:
- type: cos_sim_pearson
value: 85.6046231732945
- type: cos_sim_spearman
value: 86.41326579037185
- type: euclidean_pearson
value: 85.85739124012164
- type: euclidean_spearman
value: 86.54285701350923
- type: manhattan_pearson
value: 85.78835254765399
- type: manhattan_spearman
value: 86.45431641050791
- task:
type: STS
dataset:
type: mteb/sts16-sts
name: MTEB STS16
config: default
split: test
revision: 4d8694f8f0e0100860b497b999b3dbed754a0513
metrics:
- type: cos_sim_pearson
value: 82.97881854103466
- type: cos_sim_spearman
value: 84.50343997301495
- type: euclidean_pearson
value: 82.83306004280789
- type: euclidean_spearman
value: 83.2801802732528
- type: manhattan_pearson
value: 82.73250604776496
- type: manhattan_spearman
value: 83.12452727964241
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (ko-ko)
config: ko-ko
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 61.59564206989664
- type: cos_sim_spearman
value: 61.88740058576333
- type: euclidean_pearson
value: 60.23297902405152
- type: euclidean_spearman
value: 60.21120786234968
- type: manhattan_pearson
value: 60.48897723321176
- type: manhattan_spearman
value: 60.44230460138873
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (ar-ar)
config: ar-ar
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 80.44912821552151
- type: cos_sim_spearman
value: 81.13348443154915
- type: euclidean_pearson
value: 81.09038308120358
- type: euclidean_spearman
value: 80.5609874348409
- type: manhattan_pearson
value: 81.13776188970186
- type: manhattan_spearman
value: 80.5900946438308
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-ar)
config: en-ar
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 78.72913217243624
- type: cos_sim_spearman
value: 79.63696165091363
- type: euclidean_pearson
value: 73.19989464436063
- type: euclidean_spearman
value: 73.54600704085456
- type: manhattan_pearson
value: 72.86702738433412
- type: manhattan_spearman
value: 72.90617504239171
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-de)
config: en-de
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 50.732677791011525
- type: cos_sim_spearman
value: 52.523598781843916
- type: euclidean_pearson
value: 49.35416337421446
- type: euclidean_spearman
value: 51.33696662867874
- type: manhattan_pearson
value: 50.506295752592145
- type: manhattan_spearman
value: 52.62915407476881
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-en)
config: en-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 89.36491555020613
- type: cos_sim_spearman
value: 89.9454102616469
- type: euclidean_pearson
value: 88.86298725696331
- type: euclidean_spearman
value: 88.65552919486326
- type: manhattan_pearson
value: 88.92114540797368
- type: manhattan_spearman
value: 88.70527010857221
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-tr)
config: en-tr
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 8.714024392790805
- type: cos_sim_spearman
value: 4.749252746175972
- type: euclidean_pearson
value: 10.22053449467633
- type: euclidean_spearman
value: 9.037870998258068
- type: manhattan_pearson
value: 12.0555115545086
- type: manhattan_spearman
value: 10.63527037732596
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (es-en)
config: es-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 84.02829923391249
- type: cos_sim_spearman
value: 85.4083636563418
- type: euclidean_pearson
value: 80.36151292795275
- type: euclidean_spearman
value: 80.77292573694929
- type: manhattan_pearson
value: 80.6693169692864
- type: manhattan_spearman
value: 81.14159565166888
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (es-es)
config: es-es
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 86.99900583005198
- type: cos_sim_spearman
value: 87.3279898301188
- type: euclidean_pearson
value: 86.87787294488236
- type: euclidean_spearman
value: 85.53646010337043
- type: manhattan_pearson
value: 86.9509718845318
- type: manhattan_spearman
value: 85.71691660800931
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (fr-en)
config: fr-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 83.46126526473
- type: cos_sim_spearman
value: 83.95970248728918
- type: euclidean_pearson
value: 81.73140443111127
- type: euclidean_spearman
value: 81.74150374966206
- type: manhattan_pearson
value: 81.86557893665228
- type: manhattan_spearman
value: 82.09645552492371
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (it-en)
config: it-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 46.49174934231959
- type: cos_sim_spearman
value: 45.61787630214591
- type: euclidean_pearson
value: 49.99290765454166
- type: euclidean_spearman
value: 49.69936044179364
- type: manhattan_pearson
value: 51.3375093082487
- type: manhattan_spearman
value: 51.28438118049182
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (nl-en)
config: nl-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 48.29554395534795
- type: cos_sim_spearman
value: 46.68726750723354
- type: euclidean_pearson
value: 47.17222230888035
- type: euclidean_spearman
value: 45.92754616369105
- type: manhattan_pearson
value: 47.75493126673596
- type: manhattan_spearman
value: 46.20677181839115
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (en)
config: en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 66.3630120343016
- type: cos_sim_spearman
value: 65.81094140725656
- type: euclidean_pearson
value: 67.90672012385122
- type: euclidean_spearman
value: 67.81659181369037
- type: manhattan_pearson
value: 68.0253831292356
- type: manhattan_spearman
value: 67.6187327404364
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de)
config: de
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 29.18452426712489
- type: cos_sim_spearman
value: 37.51420703956064
- type: euclidean_pearson
value: 28.026224447990934
- type: euclidean_spearman
value: 38.80123640343127
- type: manhattan_pearson
value: 28.71522521219943
- type: manhattan_spearman
value: 39.336233734574066
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (es)
config: es
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 56.859180417788316
- type: cos_sim_spearman
value: 59.78915219131012
- type: euclidean_pearson
value: 62.96361204638708
- type: euclidean_spearman
value: 61.17669127090527
- type: manhattan_pearson
value: 63.76244034298364
- type: manhattan_spearman
value: 61.86264089685531
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (pl)
config: pl
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 16.606738041913964
- type: cos_sim_spearman
value: 27.979167349378507
- type: euclidean_pearson
value: 9.681469291321502
- type: euclidean_spearman
value: 28.088375191612652
- type: manhattan_pearson
value: 10.511180494241913
- type: manhattan_spearman
value: 28.551302212661085
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (tr)
config: tr
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 25.299512638088835
- type: cos_sim_spearman
value: 42.32704160389304
- type: euclidean_pearson
value: 38.695432241220615
- type: euclidean_spearman
value: 42.64456376476522
- type: manhattan_pearson
value: 39.85979335053606
- type: manhattan_spearman
value: 42.769358737309716
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (ar)
config: ar
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 47.92303842321097
- type: cos_sim_spearman
value: 55.000760154318996
- type: euclidean_pearson
value: 54.09534510237817
- type: euclidean_spearman
value: 56.174584414116055
- type: manhattan_pearson
value: 56.361913198454616
- type: manhattan_spearman
value: 58.34526441198397
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (ru)
config: ru
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 31.742856551594826
- type: cos_sim_spearman
value: 43.13787302806463
- type: euclidean_pearson
value: 31.905579993088136
- type: euclidean_spearman
value: 39.885035201343186
- type: manhattan_pearson
value: 32.43242118943698
- type: manhattan_spearman
value: 40.11107248799126
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh)
config: zh
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 47.44633750616152
- type: cos_sim_spearman
value: 54.083033284097816
- type: euclidean_pearson
value: 51.444658791680155
- type: euclidean_spearman
value: 53.1381741726486
- type: manhattan_pearson
value: 56.75523385117588
- type: manhattan_spearman
value: 58.34517911003165
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (fr)
config: fr
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 79.36983311049038
- type: cos_sim_spearman
value: 81.25208121596035
- type: euclidean_pearson
value: 79.0841246591628
- type: euclidean_spearman
value: 79.63170247237287
- type: manhattan_pearson
value: 79.76857988012227
- type: manhattan_spearman
value: 80.19933344030764
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de-en)
config: de-en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 50.08537255290631
- type: cos_sim_spearman
value: 51.6560951182032
- type: euclidean_pearson
value: 56.245817211229856
- type: euclidean_spearman
value: 57.84579505485162
- type: manhattan_pearson
value: 57.178628792860394
- type: manhattan_spearman
value: 58.868316567418965
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (es-en)
config: es-en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 69.32518691946098
- type: cos_sim_spearman
value: 73.58536905137812
- type: euclidean_pearson
value: 73.3593301595928
- type: euclidean_spearman
value: 74.72443890443692
- type: manhattan_pearson
value: 73.89491090838783
- type: manhattan_spearman
value: 75.01810348241496
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (it)
config: it
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 65.63185657261381
- type: cos_sim_spearman
value: 68.8680524426534
- type: euclidean_pearson
value: 65.8069214967351
- type: euclidean_spearman
value: 67.58006300921988
- type: manhattan_pearson
value: 66.42691541820066
- type: manhattan_spearman
value: 68.20501753012334
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (pl-en)
config: pl-en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 63.5746658293195
- type: cos_sim_spearman
value: 60.766781234511114
- type: euclidean_pearson
value: 63.87934914483433
- type: euclidean_spearman
value: 57.609930019070575
- type: manhattan_pearson
value: 66.02268099209732
- type: manhattan_spearman
value: 60.27189531789914
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh-en)
config: zh-en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 66.00715694009531
- type: cos_sim_spearman
value: 65.00759157082473
- type: euclidean_pearson
value: 46.532834841771916
- type: euclidean_spearman
value: 45.726258106671516
- type: manhattan_pearson
value: 67.32238041001737
- type: manhattan_spearman
value: 66.143420656417
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (es-it)
config: es-it
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 62.65123838155666
- type: cos_sim_spearman
value: 67.8261281384735
- type: euclidean_pearson
value: 63.477912220562025
- type: euclidean_spearman
value: 65.51430407718927
- type: manhattan_pearson
value: 61.935191484002964
- type: manhattan_spearman
value: 63.836661905551374
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de-fr)
config: de-fr
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 38.397676312074786
- type: cos_sim_spearman
value: 39.66141773675305
- type: euclidean_pearson
value: 32.78160515193193
- type: euclidean_spearman
value: 33.754398073832384
- type: manhattan_pearson
value: 31.542566989070103
- type: manhattan_spearman
value: 31.84555978703678
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de-pl)
config: de-pl
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 16.134054972017115
- type: cos_sim_spearman
value: 26.113399767684193
- type: euclidean_pearson
value: 24.956029896964587
- type: euclidean_spearman
value: 26.513723113179346
- type: manhattan_pearson
value: 27.504346443344712
- type: manhattan_spearman
value: 35.382424921072094
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (fr-pl)
config: fr-pl
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 74.63601297425362
- type: cos_sim_spearman
value: 84.51542547285167
- type: euclidean_pearson
value: 72.60877043745072
- type: euclidean_spearman
value: 73.24670207647144
- type: manhattan_pearson
value: 69.30655335948613
- type: manhattan_spearman
value: 73.24670207647144
- task:
type: STS
dataset:
type: C-MTEB/STSB
name: MTEB STSB
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 79.4028184159866
- type: cos_sim_spearman
value: 79.53464687577328
- type: euclidean_pearson
value: 79.25913610578554
- type: euclidean_spearman
value: 79.55288323830753
- type: manhattan_pearson
value: 79.44759977916512
- type: manhattan_spearman
value: 79.71927216173198
- task:
type: STS
dataset:
type: mteb/stsbenchmark-sts
name: MTEB STSBenchmark
config: default
split: test
revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831
metrics:
- type: cos_sim_pearson
value: 85.07398235741444
- type: cos_sim_spearman
value: 85.78865814488006
- type: euclidean_pearson
value: 83.2824378418878
- type: euclidean_spearman
value: 83.36258201307002
- type: manhattan_pearson
value: 83.22221949643878
- type: manhattan_spearman
value: 83.27892691688584
- task:
type: Reranking
dataset:
type: mteb/scidocs-reranking
name: MTEB SciDocsRR
config: default
split: test
revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab
metrics:
- type: map
value: 78.1122816381465
- type: mrr
value: 93.44523849425809
- task:
type: Retrieval
dataset:
type: scifact
name: MTEB SciFact
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 51.132999999999996
- type: map_at_10
value: 60.672000000000004
- type: map_at_100
value: 61.504000000000005
- type: map_at_1000
value: 61.526
- type: map_at_3
value: 57.536
- type: map_at_5
value: 59.362
- type: mrr_at_1
value: 53.667
- type: mrr_at_10
value: 61.980000000000004
- type: mrr_at_100
value: 62.633
- type: mrr_at_1000
value: 62.653000000000006
- type: mrr_at_3
value: 59.721999999999994
- type: mrr_at_5
value: 60.789
- type: ndcg_at_1
value: 53.667
- type: ndcg_at_10
value: 65.42099999999999
- type: ndcg_at_100
value: 68.884
- type: ndcg_at_1000
value: 69.494
- type: ndcg_at_3
value: 60.007
- type: ndcg_at_5
value: 62.487
- type: precision_at_1
value: 53.667
- type: precision_at_10
value: 8.833
- type: precision_at_100
value: 1.0699999999999998
- type: precision_at_1000
value: 0.11199999999999999
- type: precision_at_3
value: 23.222
- type: precision_at_5
value: 15.667
- type: recall_at_1
value: 51.132999999999996
- type: recall_at_10
value: 78.989
- type: recall_at_100
value: 94.167
- type: recall_at_1000
value: 99
- type: recall_at_3
value: 64.328
- type: recall_at_5
value: 70.35
- task:
type: PairClassification
dataset:
type: mteb/sprintduplicatequestions-pairclassification
name: MTEB SprintDuplicateQuestions
config: default
split: test
revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46
metrics:
- type: cos_sim_accuracy
value: 99.78910891089109
- type: cos_sim_ap
value: 94.58344155979994
- type: cos_sim_f1
value: 89.2354124748491
- type: cos_sim_precision
value: 89.77732793522267
- type: cos_sim_recall
value: 88.7
- type: dot_accuracy
value: 99.74158415841585
- type: dot_ap
value: 92.08599680108772
- type: dot_f1
value: 87.00846192135391
- type: dot_precision
value: 86.62041625371654
- type: dot_recall
value: 87.4
- type: euclidean_accuracy
value: 99.78316831683168
- type: euclidean_ap
value: 94.57715670055748
- type: euclidean_f1
value: 88.98765432098766
- type: euclidean_precision
value: 87.90243902439025
- type: euclidean_recall
value: 90.10000000000001
- type: manhattan_accuracy
value: 99.78811881188119
- type: manhattan_ap
value: 94.73016642953513
- type: manhattan_f1
value: 89.3326838772528
- type: manhattan_precision
value: 87.08452041785375
- type: manhattan_recall
value: 91.7
- type: max_accuracy
value: 99.78910891089109
- type: max_ap
value: 94.73016642953513
- type: max_f1
value: 89.3326838772528
- task:
type: Clustering
dataset:
type: mteb/stackexchange-clustering
name: MTEB StackExchangeClustering
config: default
split: test
revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259
metrics:
- type: v_measure
value: 57.11358892084413
- task:
type: Clustering
dataset:
type: mteb/stackexchange-clustering-p2p
name: MTEB StackExchangeClusteringP2P
config: default
split: test
revision: 815ca46b2622cec33ccafc3735d572c266efdb44
metrics:
- type: v_measure
value: 31.914375833951354
- task:
type: Reranking
dataset:
type: mteb/stackoverflowdupquestions-reranking
name: MTEB StackOverflowDupQuestions
config: default
split: test
revision: e185fbe320c72810689fc5848eb6114e1ef5ec69
metrics:
- type: map
value: 48.9994487557691
- type: mrr
value: 49.78547290128173
- task:
type: Summarization
dataset:
type: mteb/summeval
name: MTEB SummEval
config: default
split: test
revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c
metrics:
- type: cos_sim_pearson
value: 30.19567881069216
- type: cos_sim_spearman
value: 31.098791519646298
- type: dot_pearson
value: 30.61141391110544
- type: dot_spearman
value: 30.995416064312153
- task:
type: Reranking
dataset:
type: C-MTEB/T2Reranking
name: MTEB T2Reranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 65.9449793956858
- type: mrr
value: 75.83074738584217
- task:
type: Retrieval
dataset:
type: C-MTEB/T2Retrieval
name: MTEB T2Retrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 23.186999999999998
- type: map_at_10
value: 63.007000000000005
- type: map_at_100
value: 66.956
- type: map_at_1000
value: 67.087
- type: map_at_3
value: 44.769999999999996
- type: map_at_5
value: 54.629000000000005
- type: mrr_at_1
value: 81.22500000000001
- type: mrr_at_10
value: 85.383
- type: mrr_at_100
value: 85.555
- type: mrr_at_1000
value: 85.564
- type: mrr_at_3
value: 84.587
- type: mrr_at_5
value: 85.105
- type: ndcg_at_1
value: 81.22500000000001
- type: ndcg_at_10
value: 72.81
- type: ndcg_at_100
value: 78.108
- type: ndcg_at_1000
value: 79.477
- type: ndcg_at_3
value: 75.36
- type: ndcg_at_5
value: 73.19099999999999
- type: precision_at_1
value: 81.22500000000001
- type: precision_at_10
value: 36.419000000000004
- type: precision_at_100
value: 4.6850000000000005
- type: precision_at_1000
value: 0.502
- type: precision_at_3
value: 66.125
- type: precision_at_5
value: 54.824
- type: recall_at_1
value: 23.186999999999998
- type: recall_at_10
value: 71.568
- type: recall_at_100
value: 88.32799999999999
- type: recall_at_1000
value: 95.256
- type: recall_at_3
value: 47.04
- type: recall_at_5
value: 59.16400000000001
- task:
type: Classification
dataset:
type: C-MTEB/TNews-classification
name: MTEB TNews
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 46.08
- type: f1
value: 44.576714769815986
- task:
type: Retrieval
dataset:
type: trec-covid
name: MTEB TRECCOVID
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 0.23600000000000002
- type: map_at_10
value: 2.01
- type: map_at_100
value: 11.237
- type: map_at_1000
value: 26.241999999999997
- type: map_at_3
value: 0.705
- type: map_at_5
value: 1.134
- type: mrr_at_1
value: 92
- type: mrr_at_10
value: 95.667
- type: mrr_at_100
value: 95.667
- type: mrr_at_1000
value: 95.667
- type: mrr_at_3
value: 95.667
- type: mrr_at_5
value: 95.667
- type: ndcg_at_1
value: 88
- type: ndcg_at_10
value: 80.028
- type: ndcg_at_100
value: 58.557
- type: ndcg_at_1000
value: 51.108
- type: ndcg_at_3
value: 86.235
- type: ndcg_at_5
value: 83.776
- type: precision_at_1
value: 92
- type: precision_at_10
value: 83.6
- type: precision_at_100
value: 59.9
- type: precision_at_1000
value: 22.556
- type: precision_at_3
value: 92.667
- type: precision_at_5
value: 89.60000000000001
- type: recall_at_1
value: 0.23600000000000002
- type: recall_at_10
value: 2.164
- type: recall_at_100
value: 14.268
- type: recall_at_1000
value: 47.993
- type: recall_at_3
value: 0.728
- type: recall_at_5
value: 1.18
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (sqi-eng)
config: sqi-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 16
- type: f1
value: 12.072197229668266
- type: precision
value: 11.07125213426268
- type: recall
value: 16
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fry-eng)
config: fry-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 31.79190751445087
- type: f1
value: 25.33993944398569
- type: precision
value: 23.462449892587426
- type: recall
value: 31.79190751445087
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kur-eng)
config: kur-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 14.390243902439023
- type: f1
value: 10.647146321087272
- type: precision
value: 9.753700307679768
- type: recall
value: 14.390243902439023
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tur-eng)
config: tur-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.8
- type: f1
value: 5.087296515623526
- type: precision
value: 4.543963123070674
- type: recall
value: 7.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (deu-eng)
config: deu-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 58.5
- type: f1
value: 53.26571428571428
- type: precision
value: 51.32397398353281
- type: recall
value: 58.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nld-eng)
config: nld-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 29.5
- type: f1
value: 25.14837668933257
- type: precision
value: 23.949224030449837
- type: recall
value: 29.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ron-eng)
config: ron-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 28.7
- type: f1
value: 23.196045369663018
- type: precision
value: 21.502155293536873
- type: recall
value: 28.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ang-eng)
config: ang-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 27.611940298507463
- type: f1
value: 19.431414356787492
- type: precision
value: 17.160948504232085
- type: recall
value: 27.611940298507463
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ido-eng)
config: ido-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 46
- type: f1
value: 39.146820760938404
- type: precision
value: 36.89055652165172
- type: recall
value: 46
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (jav-eng)
config: jav-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 23.414634146341466
- type: f1
value: 18.60234074868221
- type: precision
value: 17.310239781020474
- type: recall
value: 23.414634146341466
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (isl-eng)
config: isl-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.3
- type: f1
value: 5.456411432480631
- type: precision
value: 5.073425278627456
- type: recall
value: 7.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (slv-eng)
config: slv-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 10.814094775212636
- type: f1
value: 8.096556306772158
- type: precision
value: 7.501928709802902
- type: recall
value: 10.814094775212636
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cym-eng)
config: cym-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 11.304347826086957
- type: f1
value: 7.766717493033283
- type: precision
value: 6.980930791147511
- type: recall
value: 11.304347826086957
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kaz-eng)
config: kaz-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.260869565217392
- type: f1
value: 4.695624631925284
- type: precision
value: 4.520242639508398
- type: recall
value: 6.260869565217392
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (est-eng)
config: est-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.9
- type: f1
value: 4.467212205066257
- type: precision
value: 4.004142723685108
- type: recall
value: 6.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (heb-eng)
config: heb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 1.0999999999999999
- type: f1
value: 0.6945869191049914
- type: precision
value: 0.6078431372549019
- type: recall
value: 1.0999999999999999
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (gla-eng)
config: gla-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.583835946924005
- type: f1
value: 2.9858475730729075
- type: precision
value: 2.665996515212438
- type: recall
value: 4.583835946924005
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mar-eng)
config: mar-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 59.199999999999996
- type: f1
value: 52.67345238095238
- type: precision
value: 50.13575757575758
- type: recall
value: 59.199999999999996
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lat-eng)
config: lat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 35
- type: f1
value: 27.648653013653007
- type: precision
value: 25.534839833369244
- type: recall
value: 35
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bel-eng)
config: bel-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 13.100000000000001
- type: f1
value: 9.62336638477808
- type: precision
value: 8.875194920058407
- type: recall
value: 13.100000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pms-eng)
config: pms-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 32.95238095238095
- type: f1
value: 27.600581429152854
- type: precision
value: 26.078624096473064
- type: recall
value: 32.95238095238095
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (gle-eng)
config: gle-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.5
- type: f1
value: 3.9595645184317045
- type: precision
value: 3.5893378968989453
- type: recall
value: 6.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pes-eng)
config: pes-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 17.8
- type: f1
value: 13.508124743694003
- type: precision
value: 12.24545634920635
- type: recall
value: 17.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nob-eng)
config: nob-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 21.7
- type: f1
value: 17.67074499610417
- type: precision
value: 16.47070885787265
- type: recall
value: 21.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bul-eng)
config: bul-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 19.3
- type: f1
value: 14.249803276788573
- type: precision
value: 12.916981621996223
- type: recall
value: 19.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cbk-eng)
config: cbk-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 67.2
- type: f1
value: 61.03507936507936
- type: precision
value: 58.69699346405229
- type: recall
value: 67.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hun-eng)
config: hun-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.5
- type: f1
value: 4.295097572176196
- type: precision
value: 3.809609027256814
- type: recall
value: 6.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (uig-eng)
config: uig-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 2.8000000000000003
- type: f1
value: 1.678577135635959
- type: precision
value: 1.455966810966811
- type: recall
value: 2.8000000000000003
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (rus-eng)
config: rus-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 47.9
- type: f1
value: 40.26661017143776
- type: precision
value: 37.680778943278945
- type: recall
value: 47.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (spa-eng)
config: spa-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 97
- type: f1
value: 96.05
- type: precision
value: 95.58333333333334
- type: recall
value: 97
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hye-eng)
config: hye-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 0.9433962264150944
- type: f1
value: 0.6457074216068709
- type: precision
value: 0.6068362258275373
- type: recall
value: 0.9433962264150944
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tel-eng)
config: tel-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 74.78632478632478
- type: f1
value: 69.05372405372405
- type: precision
value: 66.82336182336182
- type: recall
value: 74.78632478632478
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (afr-eng)
config: afr-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 19.2
- type: f1
value: 14.54460169057995
- type: precision
value: 13.265236397589335
- type: recall
value: 19.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mon-eng)
config: mon-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.8181818181818175
- type: f1
value: 4.78808236251355
- type: precision
value: 4.4579691142191145
- type: recall
value: 6.8181818181818175
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (arz-eng)
config: arz-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 72.53668763102725
- type: f1
value: 66.00978336827393
- type: precision
value: 63.21104122990915
- type: recall
value: 72.53668763102725
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hrv-eng)
config: hrv-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 12.7
- type: f1
value: 9.731576351893512
- type: precision
value: 8.986658245110663
- type: recall
value: 12.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nov-eng)
config: nov-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 57.19844357976653
- type: f1
value: 49.138410227904394
- type: precision
value: 45.88197146562906
- type: recall
value: 57.19844357976653
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (gsw-eng)
config: gsw-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 28.205128205128204
- type: f1
value: 21.863766936230704
- type: precision
value: 20.212164378831048
- type: recall
value: 28.205128205128204
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nds-eng)
config: nds-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 23.3
- type: f1
value: 17.75959261382939
- type: precision
value: 16.18907864830205
- type: recall
value: 23.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ukr-eng)
config: ukr-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 19.1
- type: f1
value: 14.320618913993744
- type: precision
value: 12.980748202777615
- type: recall
value: 19.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (uzb-eng)
config: uzb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.411214953271028
- type: f1
value: 5.152309182683014
- type: precision
value: 4.456214003721122
- type: recall
value: 8.411214953271028
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lit-eng)
config: lit-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.7
- type: f1
value: 4.833930504764646
- type: precision
value: 4.475394510103751
- type: recall
value: 6.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ina-eng)
config: ina-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 79.4
- type: f1
value: 74.59166666666667
- type: precision
value: 72.59928571428571
- type: recall
value: 79.4
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lfn-eng)
config: lfn-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 47.8
- type: f1
value: 41.944877899877895
- type: precision
value: 39.87211701696996
- type: recall
value: 47.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (zsm-eng)
config: zsm-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 85
- type: f1
value: 81.47666666666666
- type: precision
value: 79.95909090909092
- type: recall
value: 85
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ita-eng)
config: ita-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 62.6
- type: f1
value: 55.96755336167101
- type: precision
value: 53.49577131202131
- type: recall
value: 62.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cmn-eng)
config: cmn-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 95.3
- type: f1
value: 93.96666666666668
- type: precision
value: 93.33333333333333
- type: recall
value: 95.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lvs-eng)
config: lvs-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.7
- type: f1
value: 5.534253062728994
- type: precision
value: 4.985756669800788
- type: recall
value: 7.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (glg-eng)
config: glg-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 80.5
- type: f1
value: 75.91705128205129
- type: precision
value: 73.96261904761904
- type: recall
value: 80.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ceb-eng)
config: ceb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 10.333333333333334
- type: f1
value: 7.753678057001793
- type: precision
value: 7.207614225986279
- type: recall
value: 10.333333333333334
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bre-eng)
config: bre-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.6
- type: f1
value: 5.345683110450071
- type: precision
value: 4.569931461907268
- type: recall
value: 8.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ben-eng)
config: ben-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 82.8
- type: f1
value: 78.75999999999999
- type: precision
value: 76.97666666666666
- type: recall
value: 82.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (swg-eng)
config: swg-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 26.785714285714285
- type: f1
value: 21.62627551020408
- type: precision
value: 20.17219387755102
- type: recall
value: 26.785714285714285
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (arq-eng)
config: arq-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 32.93084522502745
- type: f1
value: 26.281513627941628
- type: precision
value: 24.05050619189897
- type: recall
value: 32.93084522502745
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kab-eng)
config: kab-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 2.1
- type: f1
value: 1.144678201129814
- type: precision
value: 1.0228433014856975
- type: recall
value: 2.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fra-eng)
config: fra-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 94.3
- type: f1
value: 92.77000000000001
- type: precision
value: 92.09166666666667
- type: recall
value: 94.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (por-eng)
config: por-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 94.1
- type: f1
value: 92.51666666666667
- type: precision
value: 91.75
- type: recall
value: 94.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tat-eng)
config: tat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.1000000000000005
- type: f1
value: 2.856566814643248
- type: precision
value: 2.6200368188362506
- type: recall
value: 4.1000000000000005
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (oci-eng)
config: oci-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 45.9
- type: f1
value: 39.02207792207792
- type: precision
value: 36.524158064158065
- type: recall
value: 45.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pol-eng)
config: pol-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 13.4
- type: f1
value: 9.61091517529598
- type: precision
value: 8.755127233877234
- type: recall
value: 13.4
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (war-eng)
config: war-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 11.1
- type: f1
value: 8.068379205189386
- type: precision
value: 7.400827352459544
- type: recall
value: 11.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (aze-eng)
config: aze-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.9
- type: f1
value: 6.632376174517077
- type: precision
value: 6.07114926880766
- type: recall
value: 8.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (vie-eng)
config: vie-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 95.8
- type: f1
value: 94.57333333333334
- type: precision
value: 93.99166666666667
- type: recall
value: 95.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nno-eng)
config: nno-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 16.6
- type: f1
value: 13.328940031174618
- type: precision
value: 12.47204179664362
- type: recall
value: 16.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cha-eng)
config: cha-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 29.927007299270077
- type: f1
value: 22.899432278994322
- type: precision
value: 20.917701519891303
- type: recall
value: 29.927007299270077
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mhr-eng)
config: mhr-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 3.5000000000000004
- type: f1
value: 2.3809722674927083
- type: precision
value: 2.1368238705738705
- type: recall
value: 3.5000000000000004
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (dan-eng)
config: dan-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 21.6
- type: f1
value: 17.54705304666238
- type: precision
value: 16.40586970344022
- type: recall
value: 21.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ell-eng)
config: ell-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 3.5999999999999996
- type: f1
value: 2.3374438522182763
- type: precision
value: 2.099034070054354
- type: recall
value: 3.5999999999999996
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (amh-eng)
config: amh-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 1.7857142857142856
- type: f1
value: 0.12056962540054328
- type: precision
value: 0.0628414244485673
- type: recall
value: 1.7857142857142856
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pam-eng)
config: pam-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.3999999999999995
- type: f1
value: 5.677284679983816
- type: precision
value: 5.314304945764335
- type: recall
value: 7.3999999999999995
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hsb-eng)
config: hsb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 13.043478260869565
- type: f1
value: 9.776306477806768
- type: precision
value: 9.09389484497104
- type: recall
value: 13.043478260869565
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (srp-eng)
config: srp-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 12.3
- type: f1
value: 8.757454269574472
- type: precision
value: 7.882868657107786
- type: recall
value: 12.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (epo-eng)
config: epo-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 28.9
- type: f1
value: 23.108557220070377
- type: precision
value: 21.35433328562513
- type: recall
value: 28.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kzj-eng)
config: kzj-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.4
- type: f1
value: 4.781499273475174
- type: precision
value: 4.4496040053464565
- type: recall
value: 6.4
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (awa-eng)
config: awa-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 51.94805194805194
- type: f1
value: 45.658020784071205
- type: precision
value: 43.54163933709388
- type: recall
value: 51.94805194805194
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fao-eng)
config: fao-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 14.50381679389313
- type: f1
value: 9.416337348733041
- type: precision
value: 8.17070085031468
- type: recall
value: 14.50381679389313
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mal-eng)
config: mal-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 88.79184861717613
- type: f1
value: 85.56040756914118
- type: precision
value: 84.08539543910723
- type: recall
value: 88.79184861717613
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ile-eng)
config: ile-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 62.5
- type: f1
value: 56.0802331002331
- type: precision
value: 53.613788230739445
- type: recall
value: 62.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bos-eng)
config: bos-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 16.101694915254235
- type: f1
value: 11.927172795816864
- type: precision
value: 10.939011968423735
- type: recall
value: 16.101694915254235
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cor-eng)
config: cor-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5.5
- type: f1
value: 3.1258727724517197
- type: precision
value: 2.679506580565404
- type: recall
value: 5.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cat-eng)
config: cat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 87.6
- type: f1
value: 84.53666666666666
- type: precision
value: 83.125
- type: recall
value: 87.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (eus-eng)
config: eus-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 65.7
- type: f1
value: 59.64428571428571
- type: precision
value: 57.30171568627451
- type: recall
value: 65.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (yue-eng)
config: yue-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 84.7
- type: f1
value: 81.34523809523809
- type: precision
value: 79.82777777777778
- type: recall
value: 84.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (swe-eng)
config: swe-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 18.6
- type: f1
value: 14.93884103295868
- type: precision
value: 14.059478087803882
- type: recall
value: 18.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (dtp-eng)
config: dtp-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5.5
- type: f1
value: 3.815842342611909
- type: precision
value: 3.565130046415928
- type: recall
value: 5.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kat-eng)
config: kat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 1.2064343163538873
- type: f1
value: 0.9147778048582338
- type: precision
value: 0.8441848589301671
- type: recall
value: 1.2064343163538873
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (jpn-eng)
config: jpn-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 71.3
- type: f1
value: 65.97350649350648
- type: precision
value: 63.85277777777777
- type: recall
value: 71.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (csb-eng)
config: csb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 13.043478260869565
- type: f1
value: 9.043759194508343
- type: precision
value: 8.097993164155737
- type: recall
value: 13.043478260869565
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (xho-eng)
config: xho-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 11.267605633802818
- type: f1
value: 8.30172606520348
- type: precision
value: 7.737059013603729
- type: recall
value: 11.267605633802818
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (orv-eng)
config: orv-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5.029940119760479
- type: f1
value: 3.07264903262435
- type: precision
value: 2.7633481831401783
- type: recall
value: 5.029940119760479
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ind-eng)
config: ind-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 90.60000000000001
- type: f1
value: 88.29666666666667
- type: precision
value: 87.21666666666667
- type: recall
value: 90.60000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tuk-eng)
config: tuk-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.389162561576355
- type: f1
value: 5.142049156827481
- type: precision
value: 4.756506859714838
- type: recall
value: 7.389162561576355
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (max-eng)
config: max-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 44.36619718309859
- type: f1
value: 39.378676538811256
- type: precision
value: 37.71007182068377
- type: recall
value: 44.36619718309859
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (swh-eng)
config: swh-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 21.794871794871796
- type: f1
value: 16.314588577641768
- type: precision
value: 14.962288221599962
- type: recall
value: 21.794871794871796
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hin-eng)
config: hin-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 93.5
- type: f1
value: 91.53333333333333
- type: precision
value: 90.58333333333333
- type: recall
value: 93.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (dsb-eng)
config: dsb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 12.526096033402922
- type: f1
value: 9.57488704957882
- type: precision
value: 8.943001322776725
- type: recall
value: 12.526096033402922
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ber-eng)
config: ber-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.9
- type: f1
value: 4.5770099528158
- type: precision
value: 4.166915172638407
- type: recall
value: 6.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tam-eng)
config: tam-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 81.75895765472313
- type: f1
value: 77.29641693811075
- type: precision
value: 75.3528773072747
- type: recall
value: 81.75895765472313
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (slk-eng)
config: slk-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 11
- type: f1
value: 8.522094712720397
- type: precision
value: 7.883076528738328
- type: recall
value: 11
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tgl-eng)
config: tgl-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 11.3
- type: f1
value: 8.626190704312432
- type: precision
value: 7.994434420637179
- type: recall
value: 11.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ast-eng)
config: ast-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 74.01574803149606
- type: f1
value: 68.16272965879266
- type: precision
value: 65.99737532808399
- type: recall
value: 74.01574803149606
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mkd-eng)
config: mkd-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 9
- type: f1
value: 6.189958106409719
- type: precision
value: 5.445330404889228
- type: recall
value: 9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (khm-eng)
config: khm-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 0.2770083102493075
- type: f1
value: 0.011664800298618888
- type: precision
value: 0.005957856811560036
- type: recall
value: 0.2770083102493075
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ces-eng)
config: ces-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.799999999999999
- type: f1
value: 5.636139438882621
- type: precision
value: 4.993972914553003
- type: recall
value: 8.799999999999999
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tzl-eng)
config: tzl-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 37.5
- type: f1
value: 31.31118881118881
- type: precision
value: 29.439102564102566
- type: recall
value: 37.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (urd-eng)
config: urd-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 74.5
- type: f1
value: 68.96380952380953
- type: precision
value: 66.67968253968255
- type: recall
value: 74.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ara-eng)
config: ara-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 89
- type: f1
value: 86.42523809523809
- type: precision
value: 85.28333333333332
- type: recall
value: 89
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kor-eng)
config: kor-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 17.2
- type: f1
value: 12.555081585081584
- type: precision
value: 11.292745310245309
- type: recall
value: 17.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (yid-eng)
config: yid-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 0.3537735849056604
- type: f1
value: 0.12010530448397783
- type: precision
value: 0.11902214818132154
- type: recall
value: 0.3537735849056604
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fin-eng)
config: fin-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5.8999999999999995
- type: f1
value: 4.26942162679512
- type: precision
value: 3.967144120536608
- type: recall
value: 5.8999999999999995
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tha-eng)
config: tha-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 2.737226277372263
- type: f1
value: 1.64474042578532
- type: precision
value: 1.567547886228932
- type: recall
value: 2.737226277372263
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (wuu-eng)
config: wuu-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 84.89999999999999
- type: f1
value: 81.17555555555555
- type: precision
value: 79.56416666666667
- type: recall
value: 84.89999999999999
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringP2P
name: MTEB ThuNewsClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 48.90675612551149
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringS2S
name: MTEB ThuNewsClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 48.33955538054993
- task:
type: Retrieval
dataset:
type: webis-touche2020
name: MTEB Touche2020
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 2.604
- type: map_at_10
value: 10.005
- type: map_at_100
value: 15.626999999999999
- type: map_at_1000
value: 16.974
- type: map_at_3
value: 5.333
- type: map_at_5
value: 7.031999999999999
- type: mrr_at_1
value: 30.612000000000002
- type: mrr_at_10
value: 45.324999999999996
- type: mrr_at_100
value: 46.261
- type: mrr_at_1000
value: 46.275
- type: mrr_at_3
value: 41.156
- type: mrr_at_5
value: 43.401
- type: ndcg_at_1
value: 28.571
- type: ndcg_at_10
value: 24.917
- type: ndcg_at_100
value: 35.304
- type: ndcg_at_1000
value: 45.973000000000006
- type: ndcg_at_3
value: 25.813000000000002
- type: ndcg_at_5
value: 24.627
- type: precision_at_1
value: 30.612000000000002
- type: precision_at_10
value: 23.061
- type: precision_at_100
value: 7.327
- type: precision_at_1000
value: 1.443
- type: precision_at_3
value: 27.211000000000002
- type: precision_at_5
value: 24.898
- type: recall_at_1
value: 2.604
- type: recall_at_10
value: 16.459
- type: recall_at_100
value: 45.344
- type: recall_at_1000
value: 77.437
- type: recall_at_3
value: 6.349
- type: recall_at_5
value: 9.487
- task:
type: Classification
dataset:
type: mteb/toxic_conversations_50k
name: MTEB ToxicConversationsClassification
config: default
split: test
revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c
metrics:
- type: accuracy
value: 72.01180000000001
- type: ap
value: 14.626345366340157
- type: f1
value: 55.341805198526096
- task:
type: Classification
dataset:
type: mteb/tweet_sentiment_extraction
name: MTEB TweetSentimentExtractionClassification
config: default
split: test
revision: d604517c81ca91fe16a244d1248fc021f9ecee7a
metrics:
- type: accuracy
value: 61.51103565365025
- type: f1
value: 61.90767326783032
- task:
type: Clustering
dataset:
type: mteb/twentynewsgroups-clustering
name: MTEB TwentyNewsgroupsClustering
config: default
split: test
revision: 6125ec4e24fa026cec8a478383ee943acfbd5449
metrics:
- type: v_measure
value: 39.80161553107969
- task:
type: PairClassification
dataset:
type: mteb/twittersemeval2015-pairclassification
name: MTEB TwitterSemEval2015
config: default
split: test
revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1
metrics:
- type: cos_sim_accuracy
value: 84.32377659891517
- type: cos_sim_ap
value: 69.1354481874608
- type: cos_sim_f1
value: 64.52149133222514
- type: cos_sim_precision
value: 58.65716753022453
- type: cos_sim_recall
value: 71.68865435356201
- type: dot_accuracy
value: 82.82172021219527
- type: dot_ap
value: 64.00853575391538
- type: dot_f1
value: 60.32341223341926
- type: dot_precision
value: 54.25801011804384
- type: dot_recall
value: 67.9155672823219
- type: euclidean_accuracy
value: 84.1151576563152
- type: euclidean_ap
value: 67.83576623331122
- type: euclidean_f1
value: 63.15157338457842
- type: euclidean_precision
value: 57.95855379188713
- type: euclidean_recall
value: 69.36675461741424
- type: manhattan_accuracy
value: 84.09727603266377
- type: manhattan_ap
value: 67.82849173216036
- type: manhattan_f1
value: 63.34376956793989
- type: manhattan_precision
value: 60.28605482717521
- type: manhattan_recall
value: 66.72823218997361
- type: max_accuracy
value: 84.32377659891517
- type: max_ap
value: 69.1354481874608
- type: max_f1
value: 64.52149133222514
- task:
type: PairClassification
dataset:
type: mteb/twitterurlcorpus-pairclassification
name: MTEB TwitterURLCorpus
config: default
split: test
revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf
metrics:
- type: cos_sim_accuracy
value: 88.90053168781775
- type: cos_sim_ap
value: 85.61513175543742
- type: cos_sim_f1
value: 78.12614999632001
- type: cos_sim_precision
value: 74.82729451571973
- type: cos_sim_recall
value: 81.72928857406838
- type: dot_accuracy
value: 88.3086894089339
- type: dot_ap
value: 83.12888443163673
- type: dot_f1
value: 77.2718948023882
- type: dot_precision
value: 73.69524208761266
- type: dot_recall
value: 81.21342777948875
- type: euclidean_accuracy
value: 88.51825978965343
- type: euclidean_ap
value: 84.99220411819988
- type: euclidean_f1
value: 77.30590577305905
- type: euclidean_precision
value: 74.16183335691045
- type: euclidean_recall
value: 80.72836464428703
- type: manhattan_accuracy
value: 88.54542632048744
- type: manhattan_ap
value: 84.98068073894048
- type: manhattan_f1
value: 77.28853696440466
- type: manhattan_precision
value: 74.39806240205158
- type: manhattan_recall
value: 80.41268863566368
- type: max_accuracy
value: 88.90053168781775
- type: max_ap
value: 85.61513175543742
- type: max_f1
value: 78.12614999632001
- task:
type: Retrieval
dataset:
type: C-MTEB/VideoRetrieval
name: MTEB VideoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 41.8
- type: map_at_10
value: 51.413
- type: map_at_100
value: 52.127
- type: map_at_1000
value: 52.168000000000006
- type: map_at_3
value: 49.25
- type: map_at_5
value: 50.425
- type: mrr_at_1
value: 41.699999999999996
- type: mrr_at_10
value: 51.363
- type: mrr_at_100
value: 52.077
- type: mrr_at_1000
value: 52.117999999999995
- type: mrr_at_3
value: 49.2
- type: mrr_at_5
value: 50.375
- type: ndcg_at_1
value: 41.8
- type: ndcg_at_10
value: 56.071000000000005
- type: ndcg_at_100
value: 59.58599999999999
- type: ndcg_at_1000
value: 60.718
- type: ndcg_at_3
value: 51.605999999999995
- type: ndcg_at_5
value: 53.714
- type: precision_at_1
value: 41.8
- type: precision_at_10
value: 7.07
- type: precision_at_100
value: 0.873
- type: precision_at_1000
value: 0.096
- type: precision_at_3
value: 19.467000000000002
- type: precision_at_5
value: 12.7
- type: recall_at_1
value: 41.8
- type: recall_at_10
value: 70.7
- type: recall_at_100
value: 87.3
- type: recall_at_1000
value: 96.39999999999999
- type: recall_at_3
value: 58.4
- type: recall_at_5
value: 63.5
- task:
type: Classification
dataset:
type: C-MTEB/waimai-classification
name: MTEB Waimai
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 82.67
- type: ap
value: 63.20621490084175
- type: f1
value: 80.81778523320692
Model Card for udever-bloom
udever-bloom-1b1 is finetuned from bigscience/bloom-1b1 via BitFit on MS MARCO Passage Ranking, SNLI and MultiNLI data.
It is a universal embedding model across tasks, natural and programming languages.
(From the technical view, udever is merely with some minor improvements to sgpt-bloom)
Model Details
Model Description
- Developed by: Alibaba Group
- Model type: Transformer-based Language Model (decoder-only)
- Language(s) (NLP): Multiple; see bloom training data
- Finetuned from model : bigscience/bloom-1b1
Model Sources
- Repository: github.com/izhx/uni-rep
- Paper : Language Models are Universal Embedders
- Training Date : 2023-06
How to Get Started with the Model
Use the code below to get started with the model.
import torch
from transformers import AutoTokenizer, BloomModel
tokenizer = AutoTokenizer.from_pretrained('izhx/udever-bloom-1b1')
model = BloomModel.from_pretrained('izhx/udever-bloom-1b1')
boq, eoq, bod, eod = '[BOQ]', '[EOQ]', '[BOD]', '[EOD]'
eoq_id, eod_id = tokenizer.convert_tokens_to_ids([eoq, eod])
if tokenizer.padding_side != 'left':
print('!!!', tokenizer.padding_side)
tokenizer.padding_side = 'left'
def encode(texts: list, is_query: bool = True, max_length=300):
bos = boq if is_query else bod
eos_id = eoq_id if is_query else eod_id
texts = [bos + t for t in texts]
encoding = tokenizer(
texts, truncation=True, max_length=max_length - 1, padding=True
)
for ids, mask in zip(encoding['input_ids'], encoding['attention_mask']):
ids.append(eos_id)
mask.append(1)
inputs = tokenizer.pad(encoding, return_tensors='pt')
with torch.inference_mode():
outputs = model(**inputs)
embeds = outputs.last_hidden_state[:, -1]
return embeds
encode(['I am Bert', 'You are Elmo'])
Training Details
Training Data
- MS MARCO Passage Ranking, retrieved by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86)
- SNLI and MultiNLI (https://sbert.net/datasets/AllNLI.tsv.gz)
Training Procedure
Preprocessing
MS MARCO hard negatives provided by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86). Negatives for SNLI and MultiNLI are randomly sampled.
Training Hyperparameters
- Training regime: tf32, BitFit
- Batch size: 1024
- Epochs: 3
- Optimizer: AdamW
- Learning rate: 1e-4
- Scheduler: constant with warmup.
- Warmup: 0.25 epoch
Evaluation
Table 1: Massive Text Embedding Benchmark MTEB
| MTEB | Avg. | Class. | Clust. | PairClass. | Rerank. | Retr. | STS | Summ. |
|---|---|---|---|---|---|---|---|---|
| #Datasets ➡️ | 56 | 12 | 11 | 3 | 4 | 15 | 10 | 1 |
| bge-large-en-v1.5 | 64.23 | 75.97 | 46.08 | 87.12 | 60.03 | 54.29 | 83.11 | 31.61 |
| bge-base-en-v1.5 | 63.55 | 75.53 | 45.77 | 86.55 | 58.86 | 53.25 | 82.4 | 31.07 |
| gte-large | 63.13 | 73.33 | 46.84 | 85 | 59.13 | 52.22 | 83.35 | 31.66 |
| gte-base | 62.39 | 73.01 | 46.2 | 84.57 | 58.61 | 51.14 | 82.3 | 31.17 |
| e5-large-v2 | 62.25 | 75.24 | 44.49 | 86.03 | 56.61 | 50.56 | 82.05 | 30.19 |
| instructor-xl | 61.79 | 73.12 | 44.74 | 86.62 | 57.29 | 49.26 | 83.06 | 32.32 |
| instructor-large | 61.59 | 73.86 | 45.29 | 85.89 | 57.54 | 47.57 | 83.15 | 31.84 |
| e5-base-v2 | 61.5 | 73.84 | 43.8 | 85.73 | 55.91 | 50.29 | 81.05 | 30.28 |
| e5-large | 61.42 | 73.14 | 43.33 | 85.94 | 56.53 | 49.99 | 82.06 | 30.97 |
| text-embedding-ada-002 (OpenAI API) | 60.99 | 70.93 | 45.9 | 84.89 | 56.32 | 49.25 | 80.97 | 30.8 |
| e5-base | 60.44 | 72.63 | 42.11 | 85.09 | 55.7 | 48.75 | 80.96 | 31.01 |
| SGPT-5.8B-msmarco | 58.93 | 68.13 | 40.34 | 82 | 56.56 | 50.25 | 78.1 | 31.46 |
| sgpt-bloom-7b1-msmarco | 57.59 | 66.19 | 38.93 | 81.9 | 55.65 | 48.22 | 77.74 | 33.6 |
| Udever-bloom-560m | 55.80 | 68.04 | 36.89 | 81.05 | 52.60 | 41.19 | 79.93 | 32.06 |
| Udever-bloom-1b1 | 58.28 | 70.18 | 39.11 | 83.11 | 54.28 | 45.27 | 81.52 | 31.10 |
| Udever-bloom-3b | 59.86 | 71.91 | 40.74 | 84.06 | 54.90 | 47.67 | 82.37 | 30.62 |
| Udever-bloom-7b1 | 60.63 | 72.13 | 40.81 | 85.40 | 55.91 | 49.34 | 83.01 | 30.97 |
Table 2: CodeSearchNet
| CodeSearchNet | Go | Ruby | Python | Java | JS | PHP | Avg. |
|---|---|---|---|---|---|---|---|
| CodeBERT | 69.3 | 70.6 | 84.0 | 86.8 | 74.8 | 70.6 | 76.0 |
| GraphCodeBERT | 84.1 | 73.2 | 87.9 | 75.7 | 71.1 | 72.5 | 77.4 |
| cpt-code S | 97.7 | 86.3 | 99.8 | 94.0 | 86.0 | 96.7 | 93.4 |
| cpt-code M | 97.5 | 85.5 | 99.9 | 94.4 | 86.5 | 97.2 | 93.5 |
| sgpt-bloom-7b1-msmarco | 76.79 | 69.25 | 95.68 | 77.93 | 70.35 | 73.45 | 77.24 |
| Udever-bloom-560m | 75.38 | 66.67 | 96.23 | 78.99 | 69.39 | 73.69 | 76.73 |
| Udever-bloom-1b1 | 78.76 | 72.85 | 97.67 | 82.77 | 74.38 | 78.97 | 80.90 |
| Udever-bloom-3b | 80.63 | 75.40 | 98.02 | 83.88 | 76.18 | 79.67 | 82.29 |
| Udever-bloom-7b1 | 79.37 | 76.59 | 98.38 | 84.68 | 77.49 | 80.03 | 82.76 |
Table 3: Chinese multi-domain retrieval Multi-cpr
| E-commerce | Entertainment video | Medical | ||||||
|---|---|---|---|---|---|---|---|---|
| Model | Train | Backbone | MRR@10 | Recall@1k | MRR@10 | Recall@1k | MRR@10 | Recall@1k |
| BM25 | - | - | 0.225 | 0.815 | 0.225 | 0.780 | 0.187 | 0.482 |
| Doc2Query | - | - | 0.239 | 0.826 | 0.238 | 0.794 | 0.210 | 0.505 |
| DPR-1 | In-Domain | BERT | 0.270 | 0.921 | 0.254 | 0.934 | 0.327 | 0.747 |
| DPR-2 | In-Domain | BERT-CT | 0.289 | 0.926 | 0.263 | 0.935 | 0.339 | 0.769 |
| text-embedding-ada-002 | General | GPT | 0.183 | 0.825 | 0.159 | 0.786 | 0.245 | 0.593 |
| sgpt-bloom-7b1-msmarco | General | BLOOM | 0.242 | 0.840 | 0.227 | 0.829 | 0.311 | 0.675 |
| Udever-bloom-560m | General | BLOOM | 0.156 | 0.802 | 0.149 | 0.749 | 0.245 | 0.571 |
| Udever-bloom-1b1 | General | BLOOM | 0.244 | 0.863 | 0.208 | 0.815 | 0.241 | 0.557 |
| Udever-bloom-3b | General | BLOOM | 0.267 | 0.871 | 0.228 | 0.836 | 0.288 | 0.619 |
| Udever-bloom-7b1 | General | BLOOM | 0.296 | 0.889 | 0.267 | 0.907 | 0.343 | 0.705 |
More results refer to paper section 3.
Technical Specifications
Model Architecture and Objective
- Model: bigscience/bloom-1b1.
- Objective: Constrastive loss with hard negatives (refer to paper section 2.2).
Compute Infrastructure
- Nvidia A100 SXM4 80GB.
- torch 2.0.0, transformers 4.29.2.
Citation
BibTeX:
@article{zhang2023language,
title={Language Models are Universal Embedders},
author={Zhang, Xin and Li, Zehan and Zhang, Yanzhao and Long, Dingkun and Xie, Pengjun and Zhang, Meishan and Zhang, Min},
journal={arXiv preprint arXiv:2310.08232},
year={2023}
}