metadata
pipeline_tag: sentence-similarity
tags:
- sentence-transformers
- feature-extraction
- sentence-similarity
- mteb
model-index:
- name: stella-mrl-large-zh-v3.5-1792d
results:
- task:
type: STS
dataset:
type: C-MTEB/AFQMC
name: MTEB AFQMC
config: default
split: validation
revision: None
metrics:
- type: cos_sim_pearson
value: 54.33822814973567
- type: cos_sim_spearman
value: 58.85457316132848
- type: euclidean_pearson
value: 57.57048145477383
- type: euclidean_spearman
value: 58.854593263425095
- type: manhattan_pearson
value: 57.55884028558309
- type: manhattan_spearman
value: 58.84474216217465
- task:
type: STS
dataset:
type: C-MTEB/ATEC
name: MTEB ATEC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 54.219652875381875
- type: cos_sim_spearman
value: 58.079506691583546
- type: euclidean_pearson
value: 61.646366330471736
- type: euclidean_spearman
value: 58.07951006894859
- type: manhattan_pearson
value: 61.64460832085762
- type: manhattan_spearman
value: 58.08054699349972
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (zh)
config: zh
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 46.593999999999994
- type: f1
value: 44.73150848183217
- task:
type: STS
dataset:
type: C-MTEB/BQ
name: MTEB BQ
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 69.16841007040091
- type: cos_sim_spearman
value: 71.04760904227217
- type: euclidean_pearson
value: 69.95126084376611
- type: euclidean_spearman
value: 71.04760904184589
- type: manhattan_pearson
value: 69.92512024129407
- type: manhattan_spearman
value: 71.02613161257672
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringP2P
name: MTEB CLSClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 43.032332399653306
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringS2S
name: MTEB CLSClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 40.41603958793544
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv1-reranking
name: MTEB CMedQAv1
config: default
split: test
revision: None
metrics:
- type: map
value: 89.33487924447584
- type: mrr
value: 91.34623015873017
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv2-reranking
name: MTEB CMedQAv2
config: default
split: test
revision: None
metrics:
- type: map
value: 89.17795270698021
- type: mrr
value: 91.0956746031746
- task:
type: Retrieval
dataset:
type: C-MTEB/CmedqaRetrieval
name: MTEB CmedqaRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 26.809
- type: map_at_10
value: 39.906000000000006
- type: map_at_100
value: 41.858000000000004
- type: map_at_1000
value: 41.954
- type: map_at_3
value: 35.435
- type: map_at_5
value: 37.978
- type: mrr_at_1
value: 40.660000000000004
- type: mrr_at_10
value: 48.787000000000006
- type: mrr_at_100
value: 49.796
- type: mrr_at_1000
value: 49.832
- type: mrr_at_3
value: 46.166000000000004
- type: mrr_at_5
value: 47.675
- type: ndcg_at_1
value: 40.660000000000004
- type: ndcg_at_10
value: 46.614
- type: ndcg_at_100
value: 54.037
- type: ndcg_at_1000
value: 55.654
- type: ndcg_at_3
value: 41.032000000000004
- type: ndcg_at_5
value: 43.464999999999996
- type: precision_at_1
value: 40.660000000000004
- type: precision_at_10
value: 10.35
- type: precision_at_100
value: 1.6340000000000001
- type: precision_at_1000
value: 0.184
- type: precision_at_3
value: 23.122
- type: precision_at_5
value: 16.944
- type: recall_at_1
value: 26.809
- type: recall_at_10
value: 57.474000000000004
- type: recall_at_100
value: 87.976
- type: recall_at_1000
value: 98.74199999999999
- type: recall_at_3
value: 40.819
- type: recall_at_5
value: 48.175000000000004
- task:
type: PairClassification
dataset:
type: C-MTEB/CMNLI
name: MTEB Cmnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 83.4996993385448
- type: cos_sim_ap
value: 90.66238348446467
- type: cos_sim_f1
value: 84.39077936333699
- type: cos_sim_precision
value: 79.53651975998345
- type: cos_sim_recall
value: 89.87608136544307
- type: dot_accuracy
value: 83.4996993385448
- type: dot_ap
value: 90.64660919236363
- type: dot_f1
value: 84.39077936333699
- type: dot_precision
value: 79.53651975998345
- type: dot_recall
value: 89.87608136544307
- type: euclidean_accuracy
value: 83.4996993385448
- type: euclidean_ap
value: 90.66238269557765
- type: euclidean_f1
value: 84.39077936333699
- type: euclidean_precision
value: 79.53651975998345
- type: euclidean_recall
value: 89.87608136544307
- type: manhattan_accuracy
value: 83.35538184004811
- type: manhattan_ap
value: 90.6446013420276
- type: manhattan_f1
value: 84.37465196569775
- type: manhattan_precision
value: 80.5614632071459
- type: manhattan_recall
value: 88.56675239653963
- type: max_accuracy
value: 83.4996993385448
- type: max_ap
value: 90.66238348446467
- type: max_f1
value: 84.39077936333699
- task:
type: Retrieval
dataset:
type: C-MTEB/CovidRetrieval
name: MTEB CovidRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 68.967
- type: map_at_10
value: 77.95299999999999
- type: map_at_100
value: 78.213
- type: map_at_1000
value: 78.21900000000001
- type: map_at_3
value: 76.30799999999999
- type: map_at_5
value: 77.316
- type: mrr_at_1
value: 69.125
- type: mrr_at_10
value: 77.886
- type: mrr_at_100
value: 78.141
- type: mrr_at_1000
value: 78.147
- type: mrr_at_3
value: 76.291
- type: mrr_at_5
value: 77.29700000000001
- type: ndcg_at_1
value: 69.231
- type: ndcg_at_10
value: 81.867
- type: ndcg_at_100
value: 82.982
- type: ndcg_at_1000
value: 83.12
- type: ndcg_at_3
value: 78.592
- type: ndcg_at_5
value: 80.39
- type: precision_at_1
value: 69.231
- type: precision_at_10
value: 9.494
- type: precision_at_100
value: 0.9990000000000001
- type: precision_at_1000
value: 0.101
- type: precision_at_3
value: 28.591
- type: precision_at_5
value: 18.061
- type: recall_at_1
value: 68.967
- type: recall_at_10
value: 93.941
- type: recall_at_100
value: 98.84100000000001
- type: recall_at_1000
value: 99.895
- type: recall_at_3
value: 85.142
- type: recall_at_5
value: 89.46300000000001
- task:
type: Retrieval
dataset:
type: C-MTEB/DuRetrieval
name: MTEB DuRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 25.824
- type: map_at_10
value: 79.396
- type: map_at_100
value: 82.253
- type: map_at_1000
value: 82.295
- type: map_at_3
value: 54.83
- type: map_at_5
value: 69.536
- type: mrr_at_1
value: 89.7
- type: mrr_at_10
value: 92.929
- type: mrr_at_100
value: 93.013
- type: mrr_at_1000
value: 93.015
- type: mrr_at_3
value: 92.658
- type: mrr_at_5
value: 92.841
- type: ndcg_at_1
value: 89.7
- type: ndcg_at_10
value: 86.797
- type: ndcg_at_100
value: 89.652
- type: ndcg_at_1000
value: 90.047
- type: ndcg_at_3
value: 85.651
- type: ndcg_at_5
value: 84.747
- type: precision_at_1
value: 89.7
- type: precision_at_10
value: 41.61
- type: precision_at_100
value: 4.788
- type: precision_at_1000
value: 0.488
- type: precision_at_3
value: 76.833
- type: precision_at_5
value: 65.14
- type: recall_at_1
value: 25.824
- type: recall_at_10
value: 87.896
- type: recall_at_100
value: 97.221
- type: recall_at_1000
value: 99.29599999999999
- type: recall_at_3
value: 57.178
- type: recall_at_5
value: 74.348
- task:
type: Retrieval
dataset:
type: C-MTEB/EcomRetrieval
name: MTEB EcomRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 52.5
- type: map_at_10
value: 63.04
- type: map_at_100
value: 63.548
- type: map_at_1000
value: 63.56
- type: map_at_3
value: 60.483
- type: map_at_5
value: 62.22800000000001
- type: mrr_at_1
value: 52.5
- type: mrr_at_10
value: 63.04
- type: mrr_at_100
value: 63.548
- type: mrr_at_1000
value: 63.56
- type: mrr_at_3
value: 60.483
- type: mrr_at_5
value: 62.22800000000001
- type: ndcg_at_1
value: 52.5
- type: ndcg_at_10
value: 68.099
- type: ndcg_at_100
value: 70.48400000000001
- type: ndcg_at_1000
value: 70.769
- type: ndcg_at_3
value: 63.01
- type: ndcg_at_5
value: 66.148
- type: precision_at_1
value: 52.5
- type: precision_at_10
value: 8.39
- type: precision_at_100
value: 0.9490000000000001
- type: precision_at_1000
value: 0.097
- type: precision_at_3
value: 23.433
- type: precision_at_5
value: 15.58
- type: recall_at_1
value: 52.5
- type: recall_at_10
value: 83.89999999999999
- type: recall_at_100
value: 94.89999999999999
- type: recall_at_1000
value: 97.1
- type: recall_at_3
value: 70.3
- type: recall_at_5
value: 77.9
- task:
type: Classification
dataset:
type: C-MTEB/IFlyTek-classification
name: MTEB IFlyTek
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 50.742593305117346
- type: f1
value: 38.7451988564002
- task:
type: Classification
dataset:
type: C-MTEB/JDReview-classification
name: MTEB JDReview
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 86.09756097560977
- type: ap
value: 54.39255221143281
- type: f1
value: 80.8326851537251
- task:
type: STS
dataset:
type: C-MTEB/LCQMC
name: MTEB LCQMC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 72.32408066246728
- type: cos_sim_spearman
value: 78.25773378380241
- type: euclidean_pearson
value: 77.87824677060661
- type: euclidean_spearman
value: 78.25773599854358
- type: manhattan_pearson
value: 77.86648277798515
- type: manhattan_spearman
value: 78.24642917155661
- task:
type: Reranking
dataset:
type: C-MTEB/Mmarco-reranking
name: MTEB MMarcoReranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 28.846601097874608
- type: mrr
value: 27.902777777777775
- task:
type: Retrieval
dataset:
type: C-MTEB/MMarcoRetrieval
name: MTEB MMarcoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 66.533
- type: map_at_10
value: 75.58399999999999
- type: map_at_100
value: 75.91
- type: map_at_1000
value: 75.921
- type: map_at_3
value: 73.847
- type: map_at_5
value: 74.929
- type: mrr_at_1
value: 68.854
- type: mrr_at_10
value: 76.20700000000001
- type: mrr_at_100
value: 76.498
- type: mrr_at_1000
value: 76.508
- type: mrr_at_3
value: 74.71600000000001
- type: mrr_at_5
value: 75.653
- type: ndcg_at_1
value: 68.854
- type: ndcg_at_10
value: 79.209
- type: ndcg_at_100
value: 80.67
- type: ndcg_at_1000
value: 80.95
- type: ndcg_at_3
value: 75.923
- type: ndcg_at_5
value: 77.74799999999999
- type: precision_at_1
value: 68.854
- type: precision_at_10
value: 9.547
- type: precision_at_100
value: 1.027
- type: precision_at_1000
value: 0.105
- type: precision_at_3
value: 28.582
- type: precision_at_5
value: 18.112000000000002
- type: recall_at_1
value: 66.533
- type: recall_at_10
value: 89.736
- type: recall_at_100
value: 96.34
- type: recall_at_1000
value: 98.52
- type: recall_at_3
value: 81.047
- type: recall_at_5
value: 85.38900000000001
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-CN)
config: zh-CN
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 73.27841291190316
- type: f1
value: 70.82287701665152
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-CN)
config: zh-CN
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 76.20040349697376
- type: f1
value: 75.92782428878164
- task:
type: Retrieval
dataset:
type: C-MTEB/MedicalRetrieval
name: MTEB MedicalRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 56.39999999999999
- type: map_at_10
value: 62.122
- type: map_at_100
value: 62.692
- type: map_at_1000
value: 62.739
- type: map_at_3
value: 60.617
- type: map_at_5
value: 61.582
- type: mrr_at_1
value: 56.39999999999999
- type: mrr_at_10
value: 62.125
- type: mrr_at_100
value: 62.696
- type: mrr_at_1000
value: 62.742
- type: mrr_at_3
value: 60.617
- type: mrr_at_5
value: 61.602000000000004
- type: ndcg_at_1
value: 56.39999999999999
- type: ndcg_at_10
value: 64.986
- type: ndcg_at_100
value: 67.889
- type: ndcg_at_1000
value: 69.16499999999999
- type: ndcg_at_3
value: 61.951
- type: ndcg_at_5
value: 63.685
- type: precision_at_1
value: 56.39999999999999
- type: precision_at_10
value: 7.3999999999999995
- type: precision_at_100
value: 0.8789999999999999
- type: precision_at_1000
value: 0.098
- type: precision_at_3
value: 21.933
- type: precision_at_5
value: 14.000000000000002
- type: recall_at_1
value: 56.39999999999999
- type: recall_at_10
value: 74
- type: recall_at_100
value: 87.9
- type: recall_at_1000
value: 98
- type: recall_at_3
value: 65.8
- type: recall_at_5
value: 70
- task:
type: Classification
dataset:
type: C-MTEB/MultilingualSentiment-classification
name: MTEB MultilingualSentiment
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 76.64
- type: f1
value: 76.5446299028248
- task:
type: PairClassification
dataset:
type: C-MTEB/OCNLI
name: MTEB Ocnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 82.34975636166757
- type: cos_sim_ap
value: 85.51352392694149
- type: cos_sim_f1
value: 83.53057199211045
- type: cos_sim_precision
value: 78.35337650323775
- type: cos_sim_recall
value: 89.44033790918691
- type: dot_accuracy
value: 82.34975636166757
- type: dot_ap
value: 85.51347115601486
- type: dot_f1
value: 83.53057199211045
- type: dot_precision
value: 78.35337650323775
- type: dot_recall
value: 89.44033790918691
- type: euclidean_accuracy
value: 82.34975636166757
- type: euclidean_ap
value: 85.51352392694149
- type: euclidean_f1
value: 83.53057199211045
- type: euclidean_precision
value: 78.35337650323775
- type: euclidean_recall
value: 89.44033790918691
- type: manhattan_accuracy
value: 82.34975636166757
- type: manhattan_ap
value: 85.48313896880585
- type: manhattan_f1
value: 83.52414136386261
- type: manhattan_precision
value: 79.00188323917138
- type: manhattan_recall
value: 88.59556494192185
- type: max_accuracy
value: 82.34975636166757
- type: max_ap
value: 85.51352392694149
- type: max_f1
value: 83.53057199211045
- task:
type: Classification
dataset:
type: C-MTEB/OnlineShopping-classification
name: MTEB OnlineShopping
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 93.39
- type: ap
value: 91.62127505252761
- type: f1
value: 93.38126146765326
- task:
type: STS
dataset:
type: C-MTEB/PAWSX
name: MTEB PAWSX
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 39.69424895486595
- type: cos_sim_spearman
value: 45.357868735202885
- type: euclidean_pearson
value: 44.85027304963503
- type: euclidean_spearman
value: 45.356945176162064
- type: manhattan_pearson
value: 44.866080721344744
- type: manhattan_spearman
value: 45.37053172312661
- task:
type: STS
dataset:
type: C-MTEB/QBQTC
name: MTEB QBQTC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 37.03908089465844
- type: cos_sim_spearman
value: 38.98314179826781
- type: euclidean_pearson
value: 37.189386019789545
- type: euclidean_spearman
value: 38.98311189555396
- type: manhattan_pearson
value: 37.14695118899785
- type: manhattan_spearman
value: 38.94957261261034
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh)
config: zh
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 65.08396305098712
- type: cos_sim_spearman
value: 66.26346934994216
- type: euclidean_pearson
value: 65.56501615370941
- type: euclidean_spearman
value: 66.26346934994216
- type: manhattan_pearson
value: 65.47984748172154
- type: manhattan_spearman
value: 66.25326746119808
- task:
type: STS
dataset:
type: C-MTEB/STSB
name: MTEB STSB
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 80.95965207330296
- type: cos_sim_spearman
value: 82.96149593569953
- type: euclidean_pearson
value: 82.67125448003975
- type: euclidean_spearman
value: 82.96141174550262
- type: manhattan_pearson
value: 82.64660468206361
- type: manhattan_spearman
value: 82.91756025324656
- task:
type: Reranking
dataset:
type: C-MTEB/T2Reranking
name: MTEB T2Reranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 66.43391960680063
- type: mrr
value: 76.078440855015
- task:
type: Retrieval
dataset:
type: C-MTEB/T2Retrieval
name: MTEB T2Retrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 28.29
- type: map_at_10
value: 78.441
- type: map_at_100
value: 82.043
- type: map_at_1000
value: 82.10499999999999
- type: map_at_3
value: 55.448
- type: map_at_5
value: 67.982
- type: mrr_at_1
value: 91.18
- type: mrr_at_10
value: 93.498
- type: mrr_at_100
value: 93.57
- type: mrr_at_1000
value: 93.572
- type: mrr_at_3
value: 93.112
- type: mrr_at_5
value: 93.351
- type: ndcg_at_1
value: 91.18
- type: ndcg_at_10
value: 85.849
- type: ndcg_at_100
value: 89.32600000000001
- type: ndcg_at_1000
value: 89.9
- type: ndcg_at_3
value: 87.333
- type: ndcg_at_5
value: 85.91499999999999
- type: precision_at_1
value: 91.18
- type: precision_at_10
value: 42.315000000000005
- type: precision_at_100
value: 5.029
- type: precision_at_1000
value: 0.517
- type: precision_at_3
value: 76.12400000000001
- type: precision_at_5
value: 63.690000000000005
- type: recall_at_1
value: 28.29
- type: recall_at_10
value: 84.679
- type: recall_at_100
value: 95.952
- type: recall_at_1000
value: 98.821
- type: recall_at_3
value: 56.987
- type: recall_at_5
value: 71.15599999999999
- task:
type: Classification
dataset:
type: C-MTEB/TNews-classification
name: MTEB TNews
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 53.09799999999999
- type: f1
value: 51.397192036892314
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringP2P
name: MTEB ThuNewsClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 70.59693805158501
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringS2S
name: MTEB ThuNewsClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 63.21127290121542
- task:
type: Retrieval
dataset:
type: C-MTEB/VideoRetrieval
name: MTEB VideoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 61.3
- type: map_at_10
value: 70.658
- type: map_at_100
value: 71.096
- type: map_at_1000
value: 71.108
- type: map_at_3
value: 69.15
- type: map_at_5
value: 70.125
- type: mrr_at_1
value: 61.3
- type: mrr_at_10
value: 70.658
- type: mrr_at_100
value: 71.096
- type: mrr_at_1000
value: 71.108
- type: mrr_at_3
value: 69.15
- type: mrr_at_5
value: 70.125
- type: ndcg_at_1
value: 61.3
- type: ndcg_at_10
value: 74.71
- type: ndcg_at_100
value: 76.783
- type: ndcg_at_1000
value: 77.09899999999999
- type: ndcg_at_3
value: 71.634
- type: ndcg_at_5
value: 73.399
- type: precision_at_1
value: 61.3
- type: precision_at_10
value: 8.72
- type: precision_at_100
value: 0.967
- type: precision_at_1000
value: 0.099
- type: precision_at_3
value: 26.267000000000003
- type: precision_at_5
value: 16.619999999999997
- type: recall_at_1
value: 61.3
- type: recall_at_10
value: 87.2
- type: recall_at_100
value: 96.7
- type: recall_at_1000
value: 99.2
- type: recall_at_3
value: 78.8
- type: recall_at_5
value: 83.1
- task:
type: Classification
dataset:
type: C-MTEB/waimai-classification
name: MTEB Waimai
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 88.01
- type: ap
value: 72.51537272974005
- type: f1
value: 86.49546025793478
license: mit
新闻 | News
[2024-04-06] 开源puff系列模型，专门针对检索和语义匹配任务，更多的考虑泛化性和私有通用测试集效果，向量维度可变，中英双语。
[2024-02-27] 开源stella-mrl-large-zh-v3.5-1792d模型，支持向量可变维度。
[2024-02-17] 开源stella v3系列、dialogue编码模型和相关训练数据。
[2023-10-19] 开源stella-base-en-v2 使用简单，不需要任何前缀文本。
[2023-10-12] 开源stella-base-zh-v2和stella-large-zh-v2, 效果更好且使用简单，不需要任何前缀文本。
[2023-09-11] 开源stella-base-zh和stella-large-zh
欢迎去本人主页查看最新模型，并提出您的宝贵意见！
1 开源模型
本次开源stella-mrl-large-zh-v3.5-1792d模型， 本模型是在stella-large-zh-v3-1792d的基础上使用MRL方法训练而成。 其主要特点是可变的向量维度。
2 使用方法
from sentence_transformers import SentenceTransformer
from sklearn.preprocessing import normalize
model = SentenceTransformer("infgrad/stella-mrl-large-zh-v3.5-1792d")
# 注意先不要normalize! 选取前n维后再normalize
vectors = model.encode(["text1", "text2"], normalize_embeddings=False)
print(vectors.shape) # shape is [2,1792]
# n_dims越大效果越好，但是时空消耗就越大。建议维度选取128的倍数，因为是这么训练的
n_dims = 768
cut_vecs = normalize(vectors[:, :n_dims])
3 不同向量维度的CMTEB得分
stella-mrl-large-zh-v3.5-1792d_1024 代表取前1024维。整体趋势是维度越大效果越好。
| Model | Retrieval | STS | PairClassification | Classification | Reranking | Clustering | CMTEB-Score |
|---|---|---|---|---|---|---|---|
| stella-mrl-large-zh-v3.5-1792d_128 | 70.01 | 62.17 | 87.99 | 70.67 | 66.77 | 53.55 | 67.16 |
| stella-mrl-large-zh-v3.5-1792d_256 | 72.19 | 62.41 | 88.09 | 71.22 | 68.32 | 53.38 | 68.02 |
| stella-mrl-large-zh-v3.5-1792d_384 | 72.77 | 62.43 | 88.26 | 71.34 | 68.31 | 53.87 | 68.25 |
| stella-mrl-large-zh-v3.5-1792d_512 | 73.11 | 62.45 | 88.16 | 71.46 | 68.32 | 53.28 | 68.29 |
| stella-mrl-large-zh-v3.5-1792d_640 | 73.27 | 62.49 | 88.21 | 71.46 | 68.69 | 53.63 | 68.42 |
| stella-mrl-large-zh-v3.5-1792d_768 | 73.38 | 62.5 | 88.19 | 71.49 | 68.64 | 53.77 | 68.47 |
| stella-mrl-large-zh-v3.5-1792d_896 | 73.37 | 62.5 | 88.14 | 71.51 | 68.44 | 54.13 | 68.49 |
| stella-mrl-large-zh-v3.5-1792d_1024 | 73.43 | 62.51 | 88.16 | 71.52 | 68.59 | 53.43 | 68.44 |
| stella-mrl-large-zh-v3.5-1792d_1152 | 73.46 | 62.49 | 88.16 | 71.57 | 68.55 | 53.67 | 68.49 |
| stella-mrl-large-zh-v3.5-1792d_1280 | 73.48 | 62.51 | 88.12 | 71.55 | 68.44 | 53.74 | 68.48 |
| stella-mrl-large-zh-v3.5-1792d_1408 | 73.48 | 62.51 | 88.14 | 71.58 | 68.46 | 53.69 | 68.48 |
| stella-mrl-large-zh-v3.5-1792d_1536 | 73.49 | 62.5 | 88.11 | 71.55 | 68.5 | 54.06 | 68.52 |
| stella-mrl-large-zh-v3.5-1792d_1664 | 73.56 | 62.49 | 88.06 | 71.56 | 68.47 | 54.28 | 68.56 |
| stella-mrl-large-zh-v3.5-1792d_1792 | 73.51 | 62.48 | 88.09 | 71.56 | 68.45 | 54.39 | 68.56 |
上述表格中stella-mrl-large-zh-v3.5-1792d_1792的得分为68.56和榜单68.55得分不一致，原因和权重类型有关，小差异请忽略不计。