Upload 130 files

.gitattributes

@@ -32,3 +32,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+qa_output/nq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/squad1-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/trivia-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/webq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text

beir_output/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.17639,0.2868,0.33452,0.39138,0.4242,0.44534,0.45186,0.45544,0.45917
+map,0.17639,0.2583,0.28476,0.30811,0.31727,0.32077,0.32136,0.32155,0.32163
+precision,0.17639,0.12328,0.09716,0.06622,0.03954,0.01792,0.00936,0.00481,0.00099
+recall,0.17639,0.36984,0.48578,0.66216,0.7909,0.89616,0.93599,0.96159,0.9936
+mrr,0.18208,0.26008,0.2871,0.31037,0.31959,0.32309,0.32368,0.32386,0.32395
+recall_cap,0.18208,0.36771,0.48578,0.66145,0.7909,0.89616,0.93599,0.96159,0.9936
+hole,0.66358,0.67923,0.67895,0.68236,0.72671,0.80683,0.8423,0.86566,0.87831

beir_output/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.17639,
+            "NDCG@3": 0.2868,
+            "NDCG@5": 0.33452,
+            "NDCG@10": 0.39138,
+            "NDCG@20": 0.4242,
+            "NDCG@50": 0.44534,
+            "NDCG@100": 0.45186,
+            "NDCG@200": 0.45544,
+            "NDCG@1000": 0.45917
+        },
+        "map": {
+            "MAP@1": 0.17639,
+            "MAP@3": 0.2583,
+            "MAP@5": 0.28476,
+            "MAP@10": 0.30811,
+            "MAP@20": 0.31727,
+            "MAP@50": 0.32077,
+            "MAP@100": 0.32136,
+            "MAP@200": 0.32155,
+            "MAP@1000": 0.32163
+        },
+        "precision": {
+            "P@1": 0.17639,
+            "P@3": 0.12328,
+            "P@5": 0.09716,
+            "P@10": 0.06622,
+            "P@20": 0.03954,
+            "P@50": 0.01792,
+            "P@100": 0.00936,
+            "P@200": 0.00481,
+            "P@1000": 0.00099
+        },
+        "recall": {
+            "Recall@1": 0.17639,
+            "Recall@3": 0.36984,
+            "Recall@5": 0.48578,
+            "Recall@10": 0.66216,
+            "Recall@20": 0.7909,
+            "Recall@50": 0.89616,
+            "Recall@100": 0.93599,
+            "Recall@200": 0.96159,
+            "Recall@1000": 0.9936
+        },
+        "mrr": {
+            "MRR@1": 0.18208,
+            "MRR@3": 0.26008,
+            "MRR@5": 0.2871,
+            "MRR@10": 0.31037,
+            "MRR@20": 0.31959,
+            "MRR@50": 0.32309,
+            "MRR@100": 0.32368,
+            "MRR@200": 0.32386,
+            "MRR@1000": 0.32395
+        },
+        "recall_cap": {
+            "R_cap@1": 0.18208,
+            "R_cap@3": 0.36771,
+            "R_cap@5": 0.48578,
+            "R_cap@10": 0.66145,
+            "R_cap@20": 0.7909,
+            "R_cap@50": 0.89616,
+            "R_cap@100": 0.93599,
+            "R_cap@200": 0.96159,
+            "R_cap@1000": 0.9936
+        },
+        "hole": {
+            "Hole@1": 0.66358,
+            "Hole@3": 0.67923,
+            "Hole@5": 0.67895,
+            "Hole@10": 0.68236,
+            "Hole@20": 0.72671,
+            "Hole@50": 0.80683,
+            "Hole@100": 0.8423,
+            "Hole@200": 0.86566,
+            "Hole@1000": 0.87831
+        }
+    }
+}

beir_output/beir.json

@@ -0,0 +1,26 @@
+{
+    "eval_beir-msmarco_ndcg@10": 0.20565,
+    "eval_beir-msmarco_recall@10": 0.35615,
+    "eval_beir-msmarco_recall@20": 0.45691,
+    "eval_beir-msmarco_recall@100": 0.67104,
+    "eval_beir-dbpedia-entity_ndcg@10": 0.28985,
+    "eval_beir-dbpedia-entity_recall@10": 0.19138,
+    "eval_beir-dbpedia-entity_recall@20": 0.25266,
+    "eval_beir-dbpedia-entity_recall@100": 0.44355,
+    "eval_beir-fever_ndcg@10": 0.59681,
+    "eval_beir-fever_recall@10": 0.78033,
+    "eval_beir-fever_recall@20": 0.84084,
+    "eval_beir-fever_recall@100": 0.91255,
+    "eval_beir-climate-fever_ndcg@10": 0.17682,
+    "eval_beir-climate-fever_recall@10": 0.21882,
+    "eval_beir-climate-fever_recall@20": 0.28607,
+    "eval_beir-climate-fever_recall@100": 0.46955,
+    "eval_beir-hotpotqa_ndcg@10": 0.47864,
+    "eval_beir-hotpotqa_recall@10": 0.51803,
+    "eval_beir-hotpotqa_recall@20": 0.57259,
+    "eval_beir-hotpotqa_recall@100": 0.68575,
+    "eval_beir-avg_ndcg@10": 0.349554,
+    "eval_beir-avg_recall@10": 0.412942,
+    "eval_beir-avg_recall@20": 0.4818140000000001,
+    "eval_beir-avg_recall@100": 0.636488
+}

beir_output/climate-fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.16287,0.1398,0.15203,0.17682,0.20024,0.22878,0.24699,0.26335,0.29112
+map,0.07742,0.10256,0.11234,0.12242,0.12946,0.13485,0.13695,0.13816,0.13916
+precision,0.16287,0.09924,0.07739,0.05472,0.03691,0.02098,0.01293,0.00778,0.0021
+recall,0.07742,0.12836,0.16251,0.21882,0.28607,0.38942,0.46955,0.5517,0.72127
+mrr,0.16287,0.21086,0.22555,0.23942,0.24681,0.25121,0.25246,0.25295,0.25318
+recall_cap,0.16287,0.13702,0.16251,0.21882,0.28607,0.38942,0.46955,0.5517,0.72127
+hole,0.61824,0.67991,0.71101,0.74267,0.77085,0.80521,0.8344,0.86118,0.91954

beir_output/climate-fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "climate-fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.16287,
+            "NDCG@3": 0.1398,
+            "NDCG@5": 0.15203,
+            "NDCG@10": 0.17682,
+            "NDCG@20": 0.20024,
+            "NDCG@50": 0.22878,
+            "NDCG@100": 0.24699,
+            "NDCG@200": 0.26335,
+            "NDCG@1000": 0.29112
+        },
+        "map": {
+            "MAP@1": 0.07742,
+            "MAP@3": 0.10256,
+            "MAP@5": 0.11234,
+            "MAP@10": 0.12242,
+            "MAP@20": 0.12946,
+            "MAP@50": 0.13485,
+            "MAP@100": 0.13695,
+            "MAP@200": 0.13816,
+            "MAP@1000": 0.13916
+        },
+        "precision": {
+            "P@1": 0.16287,
+            "P@3": 0.09924,
+            "P@5": 0.07739,
+            "P@10": 0.05472,
+            "P@20": 0.03691,
+            "P@50": 0.02098,
+            "P@100": 0.01293,
+            "P@200": 0.00778,
+            "P@1000": 0.0021
+        },
+        "recall": {
+            "Recall@1": 0.07742,
+            "Recall@3": 0.12836,
+            "Recall@5": 0.16251,
+            "Recall@10": 0.21882,
+            "Recall@20": 0.28607,
+            "Recall@50": 0.38942,
+            "Recall@100": 0.46955,
+            "Recall@200": 0.5517,
+            "Recall@1000": 0.72127
+        },
+        "mrr": {
+            "MRR@1": 0.16287,
+            "MRR@3": 0.21086,
+            "MRR@5": 0.22555,
+            "MRR@10": 0.23942,
+            "MRR@20": 0.24681,
+            "MRR@50": 0.25121,
+            "MRR@100": 0.25246,
+            "MRR@200": 0.25295,
+            "MRR@1000": 0.25318
+        },
+        "recall_cap": {
+            "R_cap@1": 0.16287,
+            "R_cap@3": 0.13702,
+            "R_cap@5": 0.16251,
+            "R_cap@10": 0.21882,
+            "R_cap@20": 0.28607,
+            "R_cap@50": 0.38942,
+            "R_cap@100": 0.46955,
+            "R_cap@200": 0.5517,
+            "R_cap@1000": 0.72127
+        },
+        "hole": {
+            "Hole@1": 0.61824,
+            "Hole@3": 0.67991,
+            "Hole@5": 0.71101,
+            "Hole@10": 0.74267,
+            "Hole@20": 0.77085,
+            "Hole@50": 0.80521,
+            "Hole@100": 0.8344,
+            "Hole@200": 0.86118,
+            "Hole@1000": 0.91954
+        }
+    }
+}

beir_output/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.19809666666666667,0.23345500000000005,0.2504741666666667,0.27104833333333334,0.2886541666666667,0.30863666666666667,0.3212183333333334,0.33208166666666666,0.3511625
+map,0.1654,0.20855,0.22002999999999998,0.22955166666666668,0.23495,0.23854166666666665,0.23986416666666666,0.24057000000000003,0.2411416666666667
+precision,0.19809666666666667,0.108695,0.07816833333333333,0.048558333333333335,0.02945083333333333,0.014938333333333333,0.008753333333333333,0.005055833333333333,0.0013183333333333335
+recall,0.1654,0.25717249999999997,0.3013466666666666,0.363245,0.4280566666666667,0.5195883333333333,0.5892308333333333,0.6581233333333333,0.8043724999999999
+mrr,0.19809666666666667,0.24325333333333332,0.2542508333333333,0.26318499999999995,0.26818166666666665,0.27129416666666667,0.2722625,0.2727116666666667,0.2730533333333333
+recall_cap,0.19809666666666667,0.2632625,0.3038458333333333,0.3642008333333333,0.4284308333333333,0.5197316666666667,0.589295,0.6581308333333333,0.8043724999999999
+hole,0.7261233333333333,0.8101108333333334,0.8371108333333334,0.8659116666666665,0.8852641666666666,0.9005008333333332,0.9082033333333333,0.914135,0.9236383333333332

beir_output/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.19809666666666667,
+            "NDCG@3": 0.23345500000000005,
+            "NDCG@5": 0.2504741666666667,
+            "NDCG@10": 0.27104833333333334,
+            "NDCG@20": 0.2886541666666667,
+            "NDCG@50": 0.30863666666666667,
+            "NDCG@100": 0.3212183333333334,
+            "NDCG@200": 0.33208166666666666,
+            "NDCG@1000": 0.3511625
+        },
+        "map": {
+            "MAP@1": 0.1654,
+            "MAP@3": 0.20855,
+            "MAP@5": 0.22002999999999998,
+            "MAP@10": 0.22955166666666668,
+            "MAP@20": 0.23495,
+            "MAP@50": 0.23854166666666665,
+            "MAP@100": 0.23986416666666666,
+            "MAP@200": 0.24057000000000003,
+            "MAP@1000": 0.2411416666666667
+        },
+        "precision": {
+            "P@1": 0.19809666666666667,
+            "P@3": 0.108695,
+            "P@5": 0.07816833333333333,
+            "P@10": 0.048558333333333335,
+            "P@20": 0.02945083333333333,
+            "P@50": 0.014938333333333333,
+            "P@100": 0.008753333333333333,
+            "P@200": 0.005055833333333333,
+            "P@1000": 0.0013183333333333335
+        },
+        "recall": {
+            "Recall@1": 0.1654,
+            "Recall@3": 0.25717249999999997,
+            "Recall@5": 0.3013466666666666,
+            "Recall@10": 0.363245,
+            "Recall@20": 0.4280566666666667,
+            "Recall@50": 0.5195883333333333,
+            "Recall@100": 0.5892308333333333,
+            "Recall@200": 0.6581233333333333,
+            "Recall@1000": 0.8043724999999999
+        },
+        "mrr": {
+            "MRR@1": 0.19809666666666667,
+            "MRR@3": 0.24325333333333332,
+            "MRR@5": 0.2542508333333333,
+            "MRR@10": 0.26318499999999995,
+            "MRR@20": 0.26818166666666665,
+            "MRR@50": 0.27129416666666667,
+            "MRR@100": 0.2722625,
+            "MRR@200": 0.2727116666666667,
+            "MRR@1000": 0.2730533333333333
+        },
+        "recall_cap": {
+            "R_cap@1": 0.19809666666666667,
+            "R_cap@3": 0.2632625,
+            "R_cap@5": 0.3038458333333333,
+            "R_cap@10": 0.3642008333333333,
+            "R_cap@20": 0.4284308333333333,
+            "R_cap@50": 0.5197316666666667,
+            "R_cap@100": 0.589295,
+            "R_cap@200": 0.6581308333333333,
+            "R_cap@1000": 0.8043724999999999
+        },
+        "hole": {
+            "Hole@1": 0.7261233333333333,
+            "Hole@3": 0.8101108333333334,
+            "Hole@5": 0.8371108333333334,
+            "Hole@10": 0.8659116666666665,
+            "Hole@20": 0.8852641666666666,
+            "Hole@50": 0.9005008333333332,
+            "Hole@100": 0.9082033333333333,
+            "Hole@200": 0.914135,
+            "Hole@1000": 0.9236383333333332
+        }
+    }
+}

beir_output/dbpedia-entity.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.35125,0.30661,0.29124,0.28985,0.28968,0.31716,0.34731,0.37493,0.42513
+map,0.06002,0.09776,0.11083,0.13231,0.15364,0.17903,0.19106,0.19786,0.20483
+precision,0.46,0.34667,0.293,0.2405,0.183,0.12175,0.08248,0.05306,0.01806
+recall,0.06002,0.11359,0.14184,0.19138,0.25266,0.3608,0.44355,0.52683,0.69595
+mrr,0.46,0.54833,0.56608,0.57606,0.57978,0.58172,0.58202,0.58218,0.58221
+recall_cap,0.46,0.36208,0.32767,0.31245,0.30796,0.37084,0.44754,0.52919,0.69641
+hole,0.3325,0.425,0.4715,0.54375,0.623,0.72875,0.79775,0.85532,0.934

beir_output/dbpedia-entity.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "dbpedia-entity",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.35125,
+            "NDCG@3": 0.30661,
+            "NDCG@5": 0.29124,
+            "NDCG@10": 0.28985,
+            "NDCG@20": 0.28968,
+            "NDCG@50": 0.31716,
+            "NDCG@100": 0.34731,
+            "NDCG@200": 0.37493,
+            "NDCG@1000": 0.42513
+        },
+        "map": {
+            "MAP@1": 0.06002,
+            "MAP@3": 0.09776,
+            "MAP@5": 0.11083,
+            "MAP@10": 0.13231,
+            "MAP@20": 0.15364,
+            "MAP@50": 0.17903,
+            "MAP@100": 0.19106,
+            "MAP@200": 0.19786,
+            "MAP@1000": 0.20483
+        },
+        "precision": {
+            "P@1": 0.46,
+            "P@3": 0.34667,
+            "P@5": 0.293,
+            "P@10": 0.2405,
+            "P@20": 0.183,
+            "P@50": 0.12175,
+            "P@100": 0.08248,
+            "P@200": 0.05306,
+            "P@1000": 0.01806
+        },
+        "recall": {
+            "Recall@1": 0.06002,
+            "Recall@3": 0.11359,
+            "Recall@5": 0.14184,
+            "Recall@10": 0.19138,
+            "Recall@20": 0.25266,
+            "Recall@50": 0.3608,
+            "Recall@100": 0.44355,
+            "Recall@200": 0.52683,
+            "Recall@1000": 0.69595
+        },
+        "mrr": {
+            "MRR@1": 0.46,
+            "MRR@3": 0.54833,
+            "MRR@5": 0.56608,
+            "MRR@10": 0.57606,
+            "MRR@20": 0.57978,
+            "MRR@50": 0.58172,
+            "MRR@100": 0.58202,
+            "MRR@200": 0.58218,
+            "MRR@1000": 0.58221
+        },
+        "recall_cap": {
+            "R_cap@1": 0.46,
+            "R_cap@3": 0.36208,
+            "R_cap@5": 0.32767,
+            "R_cap@10": 0.31245,
+            "R_cap@20": 0.30796,
+            "R_cap@50": 0.37084,
+            "R_cap@100": 0.44754,
+            "R_cap@200": 0.52919,
+            "R_cap@1000": 0.69641
+        },
+        "hole": {
+            "Hole@1": 0.3325,
+            "Hole@3": 0.425,
+            "Hole@5": 0.4715,
+            "Hole@10": 0.54375,
+            "Hole@20": 0.623,
+            "Hole@50": 0.72875,
+            "Hole@100": 0.79775,
+            "Hole@200": 0.85532,
+            "Hole@1000": 0.934
+        }
+    }
+}

beir_output/fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.42634,0.53524,0.56871,0.59681,0.61266,0.62281,0.62696,0.62946,0.63291
+map,0.39788,0.49607,0.5153,0.52749,0.53206,0.53388,0.5343,0.53445,0.53455
+precision,0.42634,0.22282,0.15164,0.08533,0.04622,0.01968,0.01015,0.00519,0.00108
+recall,0.39788,0.61552,0.69633,0.78033,0.84084,0.88871,0.91255,0.92907,0.95554
+mrr,0.42634,0.52773,0.54718,0.55917,0.56332,0.5649,0.56521,0.56532,0.56537
+recall_cap,0.42634,0.61681,0.69666,0.78035,0.84084,0.88871,0.91255,0.92907,0.95554
+hole,0.5483,0.75113,0.82298,0.89395,0.93869,0.97085,0.98315,0.98984,0.99606

beir_output/fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.42634,
+            "NDCG@3": 0.53524,
+            "NDCG@5": 0.56871,
+            "NDCG@10": 0.59681,
+            "NDCG@20": 0.61266,
+            "NDCG@50": 0.62281,
+            "NDCG@100": 0.62696,
+            "NDCG@200": 0.62946,
+            "NDCG@1000": 0.63291
+        },
+        "map": {
+            "MAP@1": 0.39788,
+            "MAP@3": 0.49607,
+            "MAP@5": 0.5153,
+            "MAP@10": 0.52749,
+            "MAP@20": 0.53206,
+            "MAP@50": 0.53388,
+            "MAP@100": 0.5343,
+            "MAP@200": 0.53445,
+            "MAP@1000": 0.53455
+        },
+        "precision": {
+            "P@1": 0.42634,
+            "P@3": 0.22282,
+            "P@5": 0.15164,
+            "P@10": 0.08533,
+            "P@20": 0.04622,
+            "P@50": 0.01968,
+            "P@100": 0.01015,
+            "P@200": 0.00519,
+            "P@1000": 0.00108
+        },
+        "recall": {
+            "Recall@1": 0.39788,
+            "Recall@3": 0.61552,
+            "Recall@5": 0.69633,
+            "Recall@10": 0.78033,
+            "Recall@20": 0.84084,
+            "Recall@50": 0.88871,
+            "Recall@100": 0.91255,
+            "Recall@200": 0.92907,
+            "Recall@1000": 0.95554
+        },
+        "mrr": {
+            "MRR@1": 0.42634,
+            "MRR@3": 0.52773,
+            "MRR@5": 0.54718,
+            "MRR@10": 0.55917,
+            "MRR@20": 0.56332,
+            "MRR@50": 0.5649,
+            "MRR@100": 0.56521,
+            "MRR@200": 0.56532,
+            "MRR@1000": 0.56537
+        },
+        "recall_cap": {
+            "R_cap@1": 0.42634,
+            "R_cap@3": 0.61681,
+            "R_cap@5": 0.69666,
+            "R_cap@10": 0.78035,
+            "R_cap@20": 0.84084,
+            "R_cap@50": 0.88871,
+            "R_cap@100": 0.91255,
+            "R_cap@200": 0.92907,
+            "R_cap@1000": 0.95554
+        },
+        "hole": {
+            "Hole@1": 0.5483,
+            "Hole@3": 0.75113,
+            "Hole@5": 0.82298,
+            "Hole@10": 0.89395,
+            "Hole@20": 0.93869,
+            "Hole@50": 0.97085,
+            "Hole@100": 0.98315,
+            "Hole@200": 0.98984,
+            "Hole@1000": 0.99606
+        }
+    }
+}

beir_output/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.19599,0.19067,0.20212,0.22343,0.24275,0.2715,0.28951,0.30503,0.33352
+map,0.10225,0.14666,0.15792,0.16937,0.1754,0.18108,0.18323,0.18436,0.18536
+precision,0.19599,0.12346,0.09383,0.0625,0.03927,0.02133,0.01307,0.00758,0.00206
+recall,0.10225,0.1808,0.22063,0.28095,0.3412,0.45427,0.53699,0.62439,0.80906
+mrr,0.19599,0.24306,0.2591,0.2699,0.27566,0.27965,0.28089,0.28139,0.28168
+recall_cap,0.19599,0.19856,0.22629,0.28144,0.3412,0.45427,0.53699,0.62439,0.80906
+hole,0.75926,0.82767,0.85031,0.88009,0.90255,0.9196,0.92806,0.93346,0.9413

beir_output/fiqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fiqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.19599,
+            "NDCG@3": 0.19067,
+            "NDCG@5": 0.20212,
+            "NDCG@10": 0.22343,
+            "NDCG@20": 0.24275,
+            "NDCG@50": 0.2715,
+            "NDCG@100": 0.28951,
+            "NDCG@200": 0.30503,
+            "NDCG@1000": 0.33352
+        },
+        "map": {
+            "MAP@1": 0.10225,
+            "MAP@3": 0.14666,
+            "MAP@5": 0.15792,
+            "MAP@10": 0.16937,
+            "MAP@20": 0.1754,
+            "MAP@50": 0.18108,
+            "MAP@100": 0.18323,
+            "MAP@200": 0.18436,
+            "MAP@1000": 0.18536
+        },
+        "precision": {
+            "P@1": 0.19599,
+            "P@3": 0.12346,
+            "P@5": 0.09383,
+            "P@10": 0.0625,
+            "P@20": 0.03927,
+            "P@50": 0.02133,
+            "P@100": 0.01307,
+            "P@200": 0.00758,
+            "P@1000": 0.00206
+        },
+        "recall": {
+            "Recall@1": 0.10225,
+            "Recall@3": 0.1808,
+            "Recall@5": 0.22063,
+            "Recall@10": 0.28095,
+            "Recall@20": 0.3412,
+            "Recall@50": 0.45427,
+            "Recall@100": 0.53699,
+            "Recall@200": 0.62439,
+            "Recall@1000": 0.80906
+        },
+        "mrr": {
+            "MRR@1": 0.19599,
+            "MRR@3": 0.24306,
+            "MRR@5": 0.2591,
+            "MRR@10": 0.2699,
+            "MRR@20": 0.27566,
+            "MRR@50": 0.27965,
+            "MRR@100": 0.28089,
+            "MRR@200": 0.28139,
+            "MRR@1000": 0.28168
+        },
+        "recall_cap": {
+            "R_cap@1": 0.19599,
+            "R_cap@3": 0.19856,
+            "R_cap@5": 0.22629,
+            "R_cap@10": 0.28144,
+            "R_cap@20": 0.3412,
+            "R_cap@50": 0.45427,
+            "R_cap@100": 0.53699,
+            "R_cap@200": 0.62439,
+            "R_cap@1000": 0.80906
+        },
+        "hole": {
+            "Hole@1": 0.75926,
+            "Hole@3": 0.82767,
+            "Hole@5": 0.85031,
+            "Hole@10": 0.88009,
+            "Hole@20": 0.90255,
+            "Hole@50": 0.9196,
+            "Hole@100": 0.92806,
+            "Hole@200": 0.93346,
+            "Hole@1000": 0.9413
+        }
+    }
+}

beir_output/hotpotqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.54004,0.42766,0.45377,0.47864,0.49556,0.51232,0.52124,0.52844,0.54145
+map,0.27002,0.35448,0.3716,0.38458,0.39057,0.39425,0.39536,0.39589,0.39632
+precision,0.54004,0.26937,0.18231,0.10361,0.05726,0.02564,0.01372,0.00728,0.00164
+recall,0.27002,0.40405,0.45577,0.51803,0.57259,0.64099,0.68575,0.72762,0.82026
+mrr,0.54004,0.60943,0.62243,0.6303,0.63347,0.63504,0.63541,0.63555,0.63565
+recall_cap,0.54004,0.40405,0.45577,0.51803,0.57259,0.64099,0.68575,0.72762,0.82026
+hole,0.45226,0.71767,0.80438,0.88365,0.93007,0.96269,0.97545,0.98261,0.98986

beir_output/hotpotqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "hotpotqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.54004,
+            "NDCG@3": 0.42766,
+            "NDCG@5": 0.45377,
+            "NDCG@10": 0.47864,
+            "NDCG@20": 0.49556,
+            "NDCG@50": 0.51232,
+            "NDCG@100": 0.52124,
+            "NDCG@200": 0.52844,
+            "NDCG@1000": 0.54145
+        },
+        "map": {
+            "MAP@1": 0.27002,
+            "MAP@3": 0.35448,
+            "MAP@5": 0.3716,
+            "MAP@10": 0.38458,
+            "MAP@20": 0.39057,
+            "MAP@50": 0.39425,
+            "MAP@100": 0.39536,
+            "MAP@200": 0.39589,
+            "MAP@1000": 0.39632
+        },
+        "precision": {
+            "P@1": 0.54004,
+            "P@3": 0.26937,
+            "P@5": 0.18231,
+            "P@10": 0.10361,
+            "P@20": 0.05726,
+            "P@50": 0.02564,
+            "P@100": 0.01372,
+            "P@200": 0.00728,
+            "P@1000": 0.00164
+        },
+        "recall": {
+            "Recall@1": 0.27002,
+            "Recall@3": 0.40405,
+            "Recall@5": 0.45577,
+            "Recall@10": 0.51803,
+            "Recall@20": 0.57259,
+            "Recall@50": 0.64099,
+            "Recall@100": 0.68575,
+            "Recall@200": 0.72762,
+            "Recall@1000": 0.82026
+        },
+        "mrr": {
+            "MRR@1": 0.54004,
+            "MRR@3": 0.60943,
+            "MRR@5": 0.62243,
+            "MRR@10": 0.6303,
+            "MRR@20": 0.63347,
+            "MRR@50": 0.63504,
+            "MRR@100": 0.63541,
+            "MRR@200": 0.63555,
+            "MRR@1000": 0.63565
+        },
+        "recall_cap": {
+            "R_cap@1": 0.54004,
+            "R_cap@3": 0.40405,
+            "R_cap@5": 0.45577,
+            "R_cap@10": 0.51803,
+            "R_cap@20": 0.57259,
+            "R_cap@50": 0.64099,
+            "R_cap@100": 0.68575,
+            "R_cap@200": 0.72762,
+            "R_cap@1000": 0.82026
+        },
+        "hole": {
+            "Hole@1": 0.45226,
+            "Hole@3": 0.71767,
+            "Hole@5": 0.80438,
+            "Hole@10": 0.88365,
+            "Hole@20": 0.93007,
+            "Hole@50": 0.96269,
+            "Hole@100": 0.97545,
+            "Hole@200": 0.98261,
+            "Hole@1000": 0.98986
+        }
+    }
+}

beir_output/msmarco.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.08596,0.14458,0.17299,0.20565,0.23143,0.25642,0.27131,0.2829,0.29902
+map,0.08398,0.12868,0.14448,0.15808,0.1652,0.16926,0.17062,0.17123,0.17161
+precision,0.08596,0.06471,0.05318,0.03706,0.02387,0.01217,0.00706,0.00397,0.00094
+recall,0.08398,0.18774,0.25644,0.35615,0.45691,0.58102,0.67104,0.75199,0.88916
+mrr,0.08596,0.13159,0.14726,0.16081,0.16787,0.17191,0.1732,0.17379,0.17414
+recall_cap,0.08596,0.18775,0.25644,0.35615,0.45691,0.58102,0.67104,0.75199,0.88916
+hole,0.91261,0.93443,0.94573,0.96191,0.97512,0.9867,0.99184,0.99494,0.99802

beir_output/msmarco.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "msmarco",
+    "split": "dev",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.08596,
+            "NDCG@3": 0.14458,
+            "NDCG@5": 0.17299,
+            "NDCG@10": 0.20565,
+            "NDCG@20": 0.23143,
+            "NDCG@50": 0.25642,
+            "NDCG@100": 0.27131,
+            "NDCG@200": 0.2829,
+            "NDCG@1000": 0.29902
+        },
+        "map": {
+            "MAP@1": 0.08398,
+            "MAP@3": 0.12868,
+            "MAP@5": 0.14448,
+            "MAP@10": 0.15808,
+            "MAP@20": 0.1652,
+            "MAP@50": 0.16926,
+            "MAP@100": 0.17062,
+            "MAP@200": 0.17123,
+            "MAP@1000": 0.17161
+        },
+        "precision": {
+            "P@1": 0.08596,
+            "P@3": 0.06471,
+            "P@5": 0.05318,
+            "P@10": 0.03706,
+            "P@20": 0.02387,
+            "P@50": 0.01217,
+            "P@100": 0.00706,
+            "P@200": 0.00397,
+            "P@1000": 0.00094
+        },
+        "recall": {
+            "Recall@1": 0.08398,
+            "Recall@3": 0.18774,
+            "Recall@5": 0.25644,
+            "Recall@10": 0.35615,
+            "Recall@20": 0.45691,
+            "Recall@50": 0.58102,
+            "Recall@100": 0.67104,
+            "Recall@200": 0.75199,
+            "Recall@1000": 0.88916
+        },
+        "mrr": {
+            "MRR@1": 0.08596,
+            "MRR@3": 0.13159,
+            "MRR@5": 0.14726,
+            "MRR@10": 0.16081,
+            "MRR@20": 0.16787,
+            "MRR@50": 0.17191,
+            "MRR@100": 0.1732,
+            "MRR@200": 0.17379,
+            "MRR@1000": 0.17414
+        },
+        "recall_cap": {
+            "R_cap@1": 0.08596,
+            "R_cap@3": 0.18775,
+            "R_cap@5": 0.25644,
+            "R_cap@10": 0.35615,
+            "R_cap@20": 0.45691,
+            "R_cap@50": 0.58102,
+            "R_cap@100": 0.67104,
+            "R_cap@200": 0.75199,
+            "R_cap@1000": 0.88916
+        },
+        "hole": {
+            "Hole@1": 0.91261,
+            "Hole@3": 0.93443,
+            "Hole@5": 0.94573,
+            "Hole@10": 0.96191,
+            "Hole@20": 0.97512,
+            "Hole@50": 0.9867,
+            "Hole@100": 0.99184,
+            "Hole@200": 0.99494,
+            "Hole@1000": 0.99802
+        }
+    }
+}

beir_output/nfcorpus.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.39628,0.35563,0.33507,0.30319,0.28103,0.27022,0.27658,0.29045,0.36604
+map,0.04598,0.08066,0.09348,0.109,0.11899,0.12836,0.13388,0.13829,0.14703
+precision,0.40867,0.33643,0.28916,0.22105,0.16316,0.10483,0.07115,0.04785,0.01998
+recall,0.04598,0.09097,0.1183,0.1525,0.18001,0.2298,0.27586,0.33212,0.60087
+mrr,0.41486,0.48607,0.49985,0.50818,0.51178,0.51365,0.51418,0.51443,0.51466
+recall_cap,0.41486,0.35294,0.32626,0.28389,0.25688,0.26052,0.2899,0.33708,0.60087
+hole,0.07121,0.07327,0.08111,0.09783,0.10913,0.11622,0.12105,0.12596,0.13346

beir_output/nfcorpus.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nfcorpus",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.39628,
+            "NDCG@3": 0.35563,
+            "NDCG@5": 0.33507,
+            "NDCG@10": 0.30319,
+            "NDCG@20": 0.28103,
+            "NDCG@50": 0.27022,
+            "NDCG@100": 0.27658,
+            "NDCG@200": 0.29045,
+            "NDCG@1000": 0.36604
+        },
+        "map": {
+            "MAP@1": 0.04598,
+            "MAP@3": 0.08066,
+            "MAP@5": 0.09348,
+            "MAP@10": 0.109,
+            "MAP@20": 0.11899,
+            "MAP@50": 0.12836,
+            "MAP@100": 0.13388,
+            "MAP@200": 0.13829,
+            "MAP@1000": 0.14703
+        },
+        "precision": {
+            "P@1": 0.40867,
+            "P@3": 0.33643,
+            "P@5": 0.28916,
+            "P@10": 0.22105,
+            "P@20": 0.16316,
+            "P@50": 0.10483,
+            "P@100": 0.07115,
+            "P@200": 0.04785,
+            "P@1000": 0.01998
+        },
+        "recall": {
+            "Recall@1": 0.04598,
+            "Recall@3": 0.09097,
+            "Recall@5": 0.1183,
+            "Recall@10": 0.1525,
+            "Recall@20": 0.18001,
+            "Recall@50": 0.2298,
+            "Recall@100": 0.27586,
+            "Recall@200": 0.33212,
+            "Recall@1000": 0.60087
+        },
+        "mrr": {
+            "MRR@1": 0.41486,
+            "MRR@3": 0.48607,
+            "MRR@5": 0.49985,
+            "MRR@10": 0.50818,
+            "MRR@20": 0.51178,
+            "MRR@50": 0.51365,
+            "MRR@100": 0.51418,
+            "MRR@200": 0.51443,
+            "MRR@1000": 0.51466
+        },
+        "recall_cap": {
+            "R_cap@1": 0.41486,
+            "R_cap@3": 0.35294,
+            "R_cap@5": 0.32626,
+            "R_cap@10": 0.28389,
+            "R_cap@20": 0.25688,
+            "R_cap@50": 0.26052,
+            "R_cap@100": 0.2899,
+            "R_cap@200": 0.33708,
+            "R_cap@1000": 0.60087
+        },
+        "hole": {
+            "Hole@1": 0.07121,
+            "Hole@3": 0.07327,
+            "Hole@5": 0.08111,
+            "Hole@10": 0.09783,
+            "Hole@20": 0.10913,
+            "Hole@50": 0.11622,
+            "Hole@100": 0.12105,
+            "Hole@200": 0.12596,
+            "Hole@1000": 0.13346
+        }
+    }
+}

beir_output/nq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.13094,0.19415,0.22982,0.27168,0.30332,0.33339,0.34724,0.35617,0.36695
+map,0.1146,0.17089,0.19116,0.20918,0.21847,0.22378,0.22513,0.22564,0.22593
+precision,0.13094,0.09347,0.07625,0.0529,0.03379,0.01706,0.00954,0.00515,0.00114
+recall,0.1146,0.24244,0.32525,0.44928,0.56817,0.7114,0.79179,0.85187,0.93909
+mrr,0.13094,0.19047,0.21053,0.22749,0.23559,0.24003,0.24115,0.24157,0.24178
+recall_cap,0.13094,0.24252,0.32525,0.44928,0.56817,0.7114,0.79179,0.85187,0.93909
+hole,0.8653,0.90353,0.9201,0.94354,0.96283,0.97976,0.98731,0.99186,0.99622

beir_output/nq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.13094,
+            "NDCG@3": 0.19415,
+            "NDCG@5": 0.22982,
+            "NDCG@10": 0.27168,
+            "NDCG@20": 0.30332,
+            "NDCG@50": 0.33339,
+            "NDCG@100": 0.34724,
+            "NDCG@200": 0.35617,
+            "NDCG@1000": 0.36695
+        },
+        "map": {
+            "MAP@1": 0.1146,
+            "MAP@3": 0.17089,
+            "MAP@5": 0.19116,
+            "MAP@10": 0.20918,
+            "MAP@20": 0.21847,
+            "MAP@50": 0.22378,
+            "MAP@100": 0.22513,
+            "MAP@200": 0.22564,
+            "MAP@1000": 0.22593
+        },
+        "precision": {
+            "P@1": 0.13094,
+            "P@3": 0.09347,
+            "P@5": 0.07625,
+            "P@10": 0.0529,
+            "P@20": 0.03379,
+            "P@50": 0.01706,
+            "P@100": 0.00954,
+            "P@200": 0.00515,
+            "P@1000": 0.00114
+        },
+        "recall": {
+            "Recall@1": 0.1146,
+            "Recall@3": 0.24244,
+            "Recall@5": 0.32525,
+            "Recall@10": 0.44928,
+            "Recall@20": 0.56817,
+            "Recall@50": 0.7114,
+            "Recall@100": 0.79179,
+            "Recall@200": 0.85187,
+            "Recall@1000": 0.93909
+        },
+        "mrr": {
+            "MRR@1": 0.13094,
+            "MRR@3": 0.19047,
+            "MRR@5": 0.21053,
+            "MRR@10": 0.22749,
+            "MRR@20": 0.23559,
+            "MRR@50": 0.24003,
+            "MRR@100": 0.24115,
+            "MRR@200": 0.24157,
+            "MRR@1000": 0.24178
+        },
+        "recall_cap": {
+            "R_cap@1": 0.13094,
+            "R_cap@3": 0.24252,
+            "R_cap@5": 0.32525,
+            "R_cap@10": 0.44928,
+            "R_cap@20": 0.56817,
+            "R_cap@50": 0.7114,
+            "R_cap@100": 0.79179,
+            "R_cap@200": 0.85187,
+            "R_cap@1000": 0.93909
+        },
+        "hole": {
+            "Hole@1": 0.8653,
+            "Hole@3": 0.90353,
+            "Hole@5": 0.9201,
+            "Hole@10": 0.94354,
+            "Hole@20": 0.96283,
+            "Hole@50": 0.97976,
+            "Hole@100": 0.98731,
+            "Hole@200": 0.99186,
+            "Hole@1000": 0.99622
+        }
+    }
+}

beir_output/quora.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.7432,0.79063,0.80987,0.82685,0.83677,0.84336,0.8458,0.84709,0.84815
+map,0.64738,0.75051,0.76936,0.78116,0.78602,0.78822,0.78872,0.7889,0.78899
+precision,0.7432,0.34417,0.2278,0.1256,0.06758,0.02883,0.01487,0.00759,0.00156
+recall,0.64738,0.81602,0.86876,0.91814,0.95101,0.97574,0.98627,0.99241,0.99806
+mrr,0.7428,0.80358,0.81272,0.81777,0.81933,0.81978,0.81985,0.81987,0.81988
+recall_cap,0.7428,0.82975,0.87403,0.9196,0.95133,0.97577,0.98627,0.99241,0.99806
+hole,0.2415,0.61983,0.73188,0.83085,0.8874,0.92607,0.94109,0.94999,0.95891

beir_output/quora.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "quora",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.7432,
+            "NDCG@3": 0.79063,
+            "NDCG@5": 0.80987,
+            "NDCG@10": 0.82685,
+            "NDCG@20": 0.83677,
+            "NDCG@50": 0.84336,
+            "NDCG@100": 0.8458,
+            "NDCG@200": 0.84709,
+            "NDCG@1000": 0.84815
+        },
+        "map": {
+            "MAP@1": 0.64738,
+            "MAP@3": 0.75051,
+            "MAP@5": 0.76936,
+            "MAP@10": 0.78116,
+            "MAP@20": 0.78602,
+            "MAP@50": 0.78822,
+            "MAP@100": 0.78872,
+            "MAP@200": 0.7889,
+            "MAP@1000": 0.78899
+        },
+        "precision": {
+            "P@1": 0.7432,
+            "P@3": 0.34417,
+            "P@5": 0.2278,
+            "P@10": 0.1256,
+            "P@20": 0.06758,
+            "P@50": 0.02883,
+            "P@100": 0.01487,
+            "P@200": 0.00759,
+            "P@1000": 0.00156
+        },
+        "recall": {
+            "Recall@1": 0.64738,
+            "Recall@3": 0.81602,
+            "Recall@5": 0.86876,
+            "Recall@10": 0.91814,
+            "Recall@20": 0.95101,
+            "Recall@50": 0.97574,
+            "Recall@100": 0.98627,
+            "Recall@200": 0.99241,
+            "Recall@1000": 0.99806
+        },
+        "mrr": {
+            "MRR@1": 0.7428,
+            "MRR@3": 0.80358,
+            "MRR@5": 0.81272,
+            "MRR@10": 0.81777,
+            "MRR@20": 0.81933,
+            "MRR@50": 0.81978,
+            "MRR@100": 0.81985,
+            "MRR@200": 0.81987,
+            "MRR@1000": 0.81988
+        },
+        "recall_cap": {
+            "R_cap@1": 0.7428,
+            "R_cap@3": 0.82975,
+            "R_cap@5": 0.87403,
+            "R_cap@10": 0.9196,
+            "R_cap@20": 0.95133,
+            "R_cap@50": 0.97577,
+            "R_cap@100": 0.98627,
+            "R_cap@200": 0.99241,
+            "R_cap@1000": 0.99806
+        },
+        "hole": {
+            "Hole@1": 0.2415,
+            "Hole@3": 0.61983,
+            "Hole@5": 0.73188,
+            "Hole@10": 0.83085,
+            "Hole@20": 0.8874,
+            "Hole@50": 0.92607,
+            "Hole@100": 0.94109,
+            "Hole@200": 0.94999,
+            "Hole@1000": 0.95891
+        }
+    }
+}

beir_output/scidocs.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.169,0.13739,0.12099,0.14703,0.17228,0.19858,0.21515,0.23246,0.26731
+map,0.03438,0.06,0.07152,0.08275,0.09146,0.09698,0.09924,0.10063,0.1019
+precision,0.169,0.12767,0.106,0.0766,0.05295,0.02898,0.01748,0.01055,0.003
+recall,0.03438,0.07768,0.10748,0.15518,0.21462,0.29352,0.35427,0.42755,0.60878
+mrr,0.169,0.23117,0.24962,0.26426,0.27122,0.27454,0.27545,0.27598,0.27629
+recall_cap,0.169,0.12767,0.10748,0.15518,0.21462,0.29352,0.35427,0.42755,0.60878
+hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

beir_output/scidocs.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scidocs",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.169,
+            "NDCG@3": 0.13739,
+            "NDCG@5": 0.12099,
+            "NDCG@10": 0.14703,
+            "NDCG@20": 0.17228,
+            "NDCG@50": 0.19858,
+            "NDCG@100": 0.21515,
+            "NDCG@200": 0.23246,
+            "NDCG@1000": 0.26731
+        },
+        "map": {
+            "MAP@1": 0.03438,
+            "MAP@3": 0.06,
+            "MAP@5": 0.07152,
+            "MAP@10": 0.08275,
+            "MAP@20": 0.09146,
+            "MAP@50": 0.09698,
+            "MAP@100": 0.09924,
+            "MAP@200": 0.10063,
+            "MAP@1000": 0.1019
+        },
+        "precision": {
+            "P@1": 0.169,
+            "P@3": 0.12767,
+            "P@5": 0.106,
+            "P@10": 0.0766,
+            "P@20": 0.05295,
+            "P@50": 0.02898,
+            "P@100": 0.01748,
+            "P@200": 0.01055,
+            "P@1000": 0.003
+        },
+        "recall": {
+            "Recall@1": 0.03438,
+            "Recall@3": 0.07768,
+            "Recall@5": 0.10748,
+            "Recall@10": 0.15518,
+            "Recall@20": 0.21462,
+            "Recall@50": 0.29352,
+            "Recall@100": 0.35427,
+            "Recall@200": 0.42755,
+            "Recall@1000": 0.60878
+        },
+        "mrr": {
+            "MRR@1": 0.169,
+            "MRR@3": 0.23117,
+            "MRR@5": 0.24962,
+            "MRR@10": 0.26426,
+            "MRR@20": 0.27122,
+            "MRR@50": 0.27454,
+            "MRR@100": 0.27545,
+            "MRR@200": 0.27598,
+            "MRR@1000": 0.27629
+        },
+        "recall_cap": {
+            "R_cap@1": 0.169,
+            "R_cap@3": 0.12767,
+            "R_cap@5": 0.10748,
+            "R_cap@10": 0.15518,
+            "R_cap@20": 0.21462,
+            "R_cap@50": 0.29352,
+            "R_cap@100": 0.35427,
+            "R_cap@200": 0.42755,
+            "R_cap@1000": 0.60878
+        },
+        "hole": {
+            "Hole@1": 0.0,
+            "Hole@3": 0.0,
+            "Hole@5": 0.0,
+            "Hole@10": 0.0,
+            "Hole@20": 0.0,
+            "Hole@50": 0.0,
+            "Hole@100": 0.0,
+            "Hole@200": 0.0,
+            "Hole@1000": 0.0
+        }
+    }
+}

beir_output/scifact.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.51333,0.58892,0.62212,0.64459,0.65859,0.67069,0.67661,0.6789,0.68402
+map,0.48983,0.56126,0.58223,0.59276,0.5972,0.59931,0.59991,0.60004,0.60017
+precision,0.51333,0.23111,0.15867,0.08767,0.047,0.0202,0.0105,0.00535,0.00112
+recall,0.48983,0.64033,0.72283,0.78733,0.84011,0.89922,0.93422,0.94989,0.99333
+mrr,0.51333,0.58222,0.60022,0.60863,0.61191,0.61367,0.61411,0.61422,0.61429
+recall_cap,0.51333,0.64333,0.72283,0.78733,0.84011,0.89922,0.93422,0.94989,0.99333
+hole,0.44667,0.71667,0.78933,0.85767,0.89383,0.92227,0.93133,0.93717,0.94093

beir_output/scifact.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scifact",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.51333,
+            "NDCG@3": 0.58892,
+            "NDCG@5": 0.62212,
+            "NDCG@10": 0.64459,
+            "NDCG@20": 0.65859,
+            "NDCG@50": 0.67069,
+            "NDCG@100": 0.67661,
+            "NDCG@200": 0.6789,
+            "NDCG@1000": 0.68402
+        },
+        "map": {
+            "MAP@1": 0.48983,
+            "MAP@3": 0.56126,
+            "MAP@5": 0.58223,
+            "MAP@10": 0.59276,
+            "MAP@20": 0.5972,
+            "MAP@50": 0.59931,
+            "MAP@100": 0.59991,
+            "MAP@200": 0.60004,
+            "MAP@1000": 0.60017
+        },
+        "precision": {
+            "P@1": 0.51333,
+            "P@3": 0.23111,
+            "P@5": 0.15867,
+            "P@10": 0.08767,
+            "P@20": 0.047,
+            "P@50": 0.0202,
+            "P@100": 0.0105,
+            "P@200": 0.00535,
+            "P@1000": 0.00112
+        },
+        "recall": {
+            "Recall@1": 0.48983,
+            "Recall@3": 0.64033,
+            "Recall@5": 0.72283,
+            "Recall@10": 0.78733,
+            "Recall@20": 0.84011,
+            "Recall@50": 0.89922,
+            "Recall@100": 0.93422,
+            "Recall@200": 0.94989,
+            "Recall@1000": 0.99333
+        },
+        "mrr": {
+            "MRR@1": 0.51333,
+            "MRR@3": 0.58222,
+            "MRR@5": 0.60022,
+            "MRR@10": 0.60863,
+            "MRR@20": 0.61191,
+            "MRR@50": 0.61367,
+            "MRR@100": 0.61411,
+            "MRR@200": 0.61422,
+            "MRR@1000": 0.61429
+        },
+        "recall_cap": {
+            "R_cap@1": 0.51333,
+            "R_cap@3": 0.64333,
+            "R_cap@5": 0.72283,
+            "R_cap@10": 0.78733,
+            "R_cap@20": 0.84011,
+            "R_cap@50": 0.89922,
+            "R_cap@100": 0.93422,
+            "R_cap@200": 0.94989,
+            "R_cap@1000": 0.99333
+        },
+        "hole": {
+            "Hole@1": 0.44667,
+            "Hole@3": 0.71667,
+            "Hole@5": 0.78933,
+            "Hole@10": 0.85767,
+            "Hole@20": 0.89383,
+            "Hole@50": 0.92227,
+            "Hole@100": 0.93133,
+            "Hole@200": 0.93717,
+            "Hole@1000": 0.94093
+        }
+    }
+}

beir_output/senteval-core.json

@@ -0,0 +1,5 @@
+{
+    "eval_senteval-stsb_spearman": 0.7572938795946844,
+    "eval_senteval-sickr_spearman": 0.6974671648657597,
+    "eval_senteval-avg_sts": 0.7273805222302221
+}

beir_output/senteval.json

@@ -0,0 +1,99 @@
+{
+    "STSBenchmark": {
+        "train": {
+            "pearson": [
+                0.7450494648920531,
+                0.0
+            ],
+            "spearman": [
+                0.718852932944271,
+                0.0
+            ],
+            "nsamples": 5749
+        },
+        "dev": {
+            "pearson": [
+                0.7506528902322174,
+                6.334785308459189e-272
+            ],
+            "spearman": [
+                0.7572938795946844,
+                1.7723543437102127e-279
+            ],
+            "nsamples": 1500
+        },
+        "test": {
+            "pearson": [
+                0.7350173167948336,
+                1.2934482649472062e-234
+            ],
+            "spearman": [
+                0.7263841319575137,
+                1.6287041037900951e-226
+            ],
+            "nsamples": 1379
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7417281269891771,
+                "mean": 0.7435732239730347,
+                "wmean": 0.7444202119694965
+            },
+            "spearman": {
+                "all": 0.731134165691079,
+                "mean": 0.7341769814988229,
+                "wmean": 0.7267396904100664
+            }
+        }
+    },
+    "SICKRelatedness": {
+        "train": {
+            "pearson": [
+                0.7406122998418166,
+                0.0
+            ],
+            "spearman": [
+                0.6737860533258943,
+                0.0
+            ],
+            "nsamples": 4500
+        },
+        "dev": {
+            "pearson": [
+                0.7491355419681752,
+                3.992619644206497e-91
+            ],
+            "spearman": [
+                0.6974671648657597,
+                4.380991542566403e-74
+            ],
+            "nsamples": 500
+        },
+        "test": {
+            "pearson": [
+                0.7341897490069431,
+                0.0
+            ],
+            "spearman": [
+                0.6637347101284393,
+                0.0
+            ],
+            "nsamples": 4927
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7377595629576476,
+                "mean": 0.7413125302723117,
+                "wmean": 0.7378539350890975
+            },
+            "spearman": {
+                "all": 0.6699463146611633,
+                "mean": 0.6783293094400311,
+                "wmean": 0.6699901016623577
+            }
+        }
+    },
+    "eval_senteval-stsb_spearman": 0.7572938795946844,
+    "eval_senteval-sickr_spearman": 0.6974671648657597,
+    "eval_senteval-avg_sts": 0.7273805222302221
+}

beir_output/trec-covid.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.63,0.5805,0.56313,0.53589,0.50601,0.45539,0.40739,0.35146,0.37269
+map,0.00182,0.00447,0.00664,0.01187,0.02102,0.04092,0.0648,0.09522,0.1588
+precision,0.64,0.60667,0.596,0.574,0.542,0.4828,0.4254,0.3541,0.1715
+recall,0.00182,0.00483,0.00768,0.01456,0.0274,0.05786,0.09788,0.15687,0.3557
+mrr,0.62,0.72,0.746,0.74933,0.75058,0.75058,0.75058,0.75058,0.75058
+recall_cap,0.62,0.60667,0.592,0.572,0.542,0.4824,0.4254,0.35664,0.35705
+hole,0.16,0.12667,0.132,0.134,0.135,0.1568,0.1958,0.2358,0.36202

beir_output/trec-covid.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "trec-covid",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.63,
+            "NDCG@3": 0.5805,
+            "NDCG@5": 0.56313,
+            "NDCG@10": 0.53589,
+            "NDCG@20": 0.50601,
+            "NDCG@50": 0.45539,
+            "NDCG@100": 0.40739,
+            "NDCG@200": 0.35146,
+            "NDCG@1000": 0.37269
+        },
+        "map": {
+            "MAP@1": 0.00182,
+            "MAP@3": 0.00447,
+            "MAP@5": 0.00664,
+            "MAP@10": 0.01187,
+            "MAP@20": 0.02102,
+            "MAP@50": 0.04092,
+            "MAP@100": 0.0648,
+            "MAP@200": 0.09522,
+            "MAP@1000": 0.1588
+        },
+        "precision": {
+            "P@1": 0.64,
+            "P@3": 0.60667,
+            "P@5": 0.596,
+            "P@10": 0.574,
+            "P@20": 0.542,
+            "P@50": 0.4828,
+            "P@100": 0.4254,
+            "P@200": 0.3541,
+            "P@1000": 0.1715
+        },
+        "recall": {
+            "Recall@1": 0.00182,
+            "Recall@3": 0.00483,
+            "Recall@5": 0.00768,
+            "Recall@10": 0.01456,
+            "Recall@20": 0.0274,
+            "Recall@50": 0.05786,
+            "Recall@100": 0.09788,
+            "Recall@200": 0.15687,
+            "Recall@1000": 0.3557
+        },
+        "mrr": {
+            "MRR@1": 0.62,
+            "MRR@3": 0.72,
+            "MRR@5": 0.746,
+            "MRR@10": 0.74933,
+            "MRR@20": 0.75058,
+            "MRR@50": 0.75058,
+            "MRR@100": 0.75058,
+            "MRR@200": 0.75058,
+            "MRR@1000": 0.75058
+        },
+        "recall_cap": {
+            "R_cap@1": 0.62,
+            "R_cap@3": 0.60667,
+            "R_cap@5": 0.592,
+            "R_cap@10": 0.572,
+            "R_cap@20": 0.542,
+            "R_cap@50": 0.4824,
+            "R_cap@100": 0.4254,
+            "R_cap@200": 0.35664,
+            "R_cap@1000": 0.35705
+        },
+        "hole": {
+            "Hole@1": 0.16,
+            "Hole@3": 0.12667,
+            "Hole@5": 0.132,
+            "Hole@10": 0.134,
+            "Hole@20": 0.135,
+            "Hole@50": 0.1568,
+            "Hole@100": 0.1958,
+            "Hole@200": 0.2358,
+            "Hole@1000": 0.36202
+        }
+    }
+}

beir_output/webis-touche2020.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.15306,0.16559,0.19102,0.18996,0.2098,0.25659,0.31192,0.3514,0.42487
+map,0.0147,0.03302,0.05037,0.07144,0.09435,0.11205,0.12454,0.13191,0.13968
+precision,0.18367,0.18367,0.21224,0.18367,0.15306,0.09592,0.07041,0.04541,0.01435
+recall,0.0147,0.04432,0.08342,0.13654,0.21521,0.31708,0.44399,0.54667,0.78305
+mrr,0.18367,0.28571,0.33878,0.35322,0.35978,0.36181,0.3622,0.3622,0.36238
+recall_cap,0.18367,0.18367,0.21224,0.2013,0.22388,0.31708,0.44399,0.54667,0.78305
+hole,0.67347,0.73469,0.71429,0.76939,0.80102,0.86286,0.89367,0.92388,0.96686

beir_output/webis-touche2020.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "webis-touche2020",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.15306,
+            "NDCG@3": 0.16559,
+            "NDCG@5": 0.19102,
+            "NDCG@10": 0.18996,
+            "NDCG@20": 0.2098,
+            "NDCG@50": 0.25659,
+            "NDCG@100": 0.31192,
+            "NDCG@200": 0.3514,
+            "NDCG@1000": 0.42487
+        },
+        "map": {
+            "MAP@1": 0.0147,
+            "MAP@3": 0.03302,
+            "MAP@5": 0.05037,
+            "MAP@10": 0.07144,
+            "MAP@20": 0.09435,
+            "MAP@50": 0.11205,
+            "MAP@100": 0.12454,
+            "MAP@200": 0.13191,
+            "MAP@1000": 0.13968
+        },
+        "precision": {
+            "P@1": 0.18367,
+            "P@3": 0.18367,
+            "P@5": 0.21224,
+            "P@10": 0.18367,
+            "P@20": 0.15306,
+            "P@50": 0.09592,
+            "P@100": 0.07041,
+            "P@200": 0.04541,
+            "P@1000": 0.01435
+        },
+        "recall": {
+            "Recall@1": 0.0147,
+            "Recall@3": 0.04432,
+            "Recall@5": 0.08342,
+            "Recall@10": 0.13654,
+            "Recall@20": 0.21521,
+            "Recall@50": 0.31708,
+            "Recall@100": 0.44399,
+            "Recall@200": 0.54667,
+            "Recall@1000": 0.78305
+        },
+        "mrr": {
+            "MRR@1": 0.18367,
+            "MRR@3": 0.28571,
+            "MRR@5": 0.33878,
+            "MRR@10": 0.35322,
+            "MRR@20": 0.35978,
+            "MRR@50": 0.36181,
+            "MRR@100": 0.3622,
+            "MRR@200": 0.3622,
+            "MRR@1000": 0.36238
+        },
+        "recall_cap": {
+            "R_cap@1": 0.18367,
+            "R_cap@3": 0.18367,
+            "R_cap@5": 0.21224,
+            "R_cap@10": 0.2013,
+            "R_cap@20": 0.22388,
+            "R_cap@50": 0.31708,
+            "R_cap@100": 0.44399,
+            "R_cap@200": 0.54667,
+            "R_cap@1000": 0.78305
+        },
+        "hole": {
+            "Hole@1": 0.67347,
+            "Hole@3": 0.73469,
+            "Hole@5": 0.71429,
+            "Hole@10": 0.76939,
+            "Hole@20": 0.80102,
+            "Hole@50": 0.86286,
+            "Hole@100": 0.89367,
+            "Hole@200": 0.92388,
+            "Hole@1000": 0.96686
+        }
+    }
+}

cc.ExtQ50-plm.moco.bs1024.gpu8.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export LOCAL_RANK=0
+export WORLD_SIZE=8
+
+export TOKENIZERS_PARALLELISM=true
+export NUM_WORKER=8
+export MAX_STEPS=100000
+
+export CUDA_LAUNCH_BLOCKING=1
+export NCCL_DEBUG=INFO
+
+export EXP_NAME=cc.ExtQ50-plm-chunk16.seed477.moco-2e14.bert-base-uncased.avg.dot.q128d256.step100k.bs1024.lr5e5
+export PROJECT_DIR=output_dir/$EXP_NAME
+mkdir -p $PROJECT_DIR
+cp "$0" $PROJECT_DIR  # copy bash to project dir
+echo $PROJECT_DIR
+
+export WANDB_NAME=$EXP_NAME
+export WANDB_PROJECT=unsup_retrieval_cc
+export WANDB_DIR=$PROJECT_DIR
+mkdir -p $WANDB_DIR/wandb
+
+nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=3112 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type moco --q_select google/t5-small-lm-adapt --num_random_chunk 16 --q_select_ratio 0.5 --train_file pile_Pile-CC --dev_file /export/home/data/pretrain/wiki2021_structure/wiki_psgs_w100.dev.tail2e13.tsv --data_type hf --data_pipeline_name contriever256 --remove_unused_columns False --sim_type dot --queue_size 16384 --momentum 0.9995 --output_dir $PROJECT_DIR --overwrite_output_dir --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --warmup_steps 10000 --logging_steps 100 --eval_steps 10000 --save_steps 1000000 --per_device_train_batch_size 128 --per_device_eval_batch_size 128 --beir_batch_size 256 --dataloader_num_workers $NUM_WORKER --learning_rate 5e-5 --max_q_tokens 128 --max_d_tokens 256 --evaluation_strategy steps --load_best_model_at_end --do_train --do_eval --run_name $EXP_NAME --fp16 --seed 477 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup.log 2>&1 & echo $! > run.pid
+
+#export WANDB_RUN_ID=370bk36z
+#nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=3112 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type inbatch --reload_model_from $PROJECT_DIR --remove_unused_columns False --sim_type dot --output_dir $PROJECT_DIR --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --do_eval --run_name $EXP_NAME --fp16 --seed 477 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup-eval.log 2>&1 & echo $! > run.pid
+

eval_output/checkpoint-100000/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.17639,0.2868,0.33452,0.39138,0.4242,0.44534,0.45186,0.45544,0.45917
+map,0.17639,0.2583,0.28476,0.30811,0.31727,0.32077,0.32136,0.32155,0.32163
+precision,0.17639,0.12328,0.09716,0.06622,0.03954,0.01792,0.00936,0.00481,0.00099
+recall,0.17639,0.36984,0.48578,0.66216,0.7909,0.89616,0.93599,0.96159,0.9936
+mrr,0.18208,0.26008,0.2871,0.31037,0.31959,0.32309,0.32368,0.32386,0.32395
+recall_cap,0.18208,0.36771,0.48578,0.66145,0.7909,0.89616,0.93599,0.96159,0.9936
+hole,0.66358,0.67923,0.67895,0.68236,0.72671,0.80683,0.8423,0.86566,0.87831

eval_output/checkpoint-100000/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.17639,
+            "NDCG@3": 0.2868,
+            "NDCG@5": 0.33452,
+            "NDCG@10": 0.39138,
+            "NDCG@20": 0.4242,
+            "NDCG@50": 0.44534,
+            "NDCG@100": 0.45186,
+            "NDCG@200": 0.45544,
+            "NDCG@1000": 0.45917
+        },
+        "map": {
+            "MAP@1": 0.17639,
+            "MAP@3": 0.2583,
+            "MAP@5": 0.28476,
+            "MAP@10": 0.30811,
+            "MAP@20": 0.31727,
+            "MAP@50": 0.32077,
+            "MAP@100": 0.32136,
+            "MAP@200": 0.32155,
+            "MAP@1000": 0.32163
+        },
+        "precision": {
+            "P@1": 0.17639,
+            "P@3": 0.12328,
+            "P@5": 0.09716,
+            "P@10": 0.06622,
+            "P@20": 0.03954,
+            "P@50": 0.01792,
+            "P@100": 0.00936,
+            "P@200": 0.00481,
+            "P@1000": 0.00099
+        },
+        "recall": {
+            "Recall@1": 0.17639,
+            "Recall@3": 0.36984,
+            "Recall@5": 0.48578,
+            "Recall@10": 0.66216,
+            "Recall@20": 0.7909,
+            "Recall@50": 0.89616,
+            "Recall@100": 0.93599,
+            "Recall@200": 0.96159,
+            "Recall@1000": 0.9936
+        },
+        "mrr": {
+            "MRR@1": 0.18208,
+            "MRR@3": 0.26008,
+            "MRR@5": 0.2871,
+            "MRR@10": 0.31037,
+            "MRR@20": 0.31959,
+            "MRR@50": 0.32309,
+            "MRR@100": 0.32368,
+            "MRR@200": 0.32386,
+            "MRR@1000": 0.32395
+        },
+        "recall_cap": {
+            "R_cap@1": 0.18208,
+            "R_cap@3": 0.36771,
+            "R_cap@5": 0.48578,
+            "R_cap@10": 0.66145,
+            "R_cap@20": 0.7909,
+            "R_cap@50": 0.89616,
+            "R_cap@100": 0.93599,
+            "R_cap@200": 0.96159,
+            "R_cap@1000": 0.9936
+        },
+        "hole": {
+            "Hole@1": 0.66358,
+            "Hole@3": 0.67923,
+            "Hole@5": 0.67895,
+            "Hole@10": 0.68236,
+            "Hole@20": 0.72671,
+            "Hole@50": 0.80683,
+            "Hole@100": 0.8423,
+            "Hole@200": 0.86566,
+            "Hole@1000": 0.87831
+        }
+    }
+}

eval_output/checkpoint-100000/beir.json

@@ -0,0 +1,46 @@
+{
+    "eval_beir-nfcorpus_ndcg@10": 0.30319,
+    "eval_beir-nfcorpus_recall@10": 0.1525,
+    "eval_beir-nfcorpus_recall@20": 0.18001,
+    "eval_beir-nfcorpus_recall@100": 0.27586,
+    "eval_beir-fiqa_ndcg@10": 0.22343,
+    "eval_beir-fiqa_recall@10": 0.28095,
+    "eval_beir-fiqa_recall@20": 0.3412,
+    "eval_beir-fiqa_recall@100": 0.53699,
+    "eval_beir-arguana_ndcg@10": 0.39138,
+    "eval_beir-arguana_recall@10": 0.66216,
+    "eval_beir-arguana_recall@20": 0.7909,
+    "eval_beir-arguana_recall@100": 0.93599,
+    "eval_beir-scidocs_ndcg@10": 0.14703,
+    "eval_beir-scidocs_recall@10": 0.15518,
+    "eval_beir-scidocs_recall@20": 0.21462,
+    "eval_beir-scidocs_recall@100": 0.35427,
+    "eval_beir-scifact_ndcg@10": 0.64459,
+    "eval_beir-scifact_recall@10": 0.78733,
+    "eval_beir-scifact_recall@20": 0.84011,
+    "eval_beir-scifact_recall@100": 0.93422,
+    "eval_beir-webis-touche2020_ndcg@10": 0.18996,
+    "eval_beir-webis-touche2020_recall@10": 0.13654,
+    "eval_beir-webis-touche2020_recall@20": 0.21521,
+    "eval_beir-webis-touche2020_recall@100": 0.44399,
+    "eval_beir-cqadupstack_ndcg@10": 0.27104833333333334,
+    "eval_beir-cqadupstack_recall@10": 0.363245,
+    "eval_beir-cqadupstack_recall@20": 0.4280566666666667,
+    "eval_beir-cqadupstack_recall@100": 0.5892308333333333,
+    "eval_beir-trec-covid_ndcg@10": 0.53589,
+    "eval_beir-trec-covid_recall@10": 0.572,
+    "eval_beir-trec-covid_recall@20": 0.542,
+    "eval_beir-trec-covid_recall@100": 0.4254,
+    "eval_beir-quora_ndcg@10": 0.82685,
+    "eval_beir-quora_recall@10": 0.91814,
+    "eval_beir-quora_recall@20": 0.95101,
+    "eval_beir-quora_recall@100": 0.98627,
+    "eval_beir-nq_ndcg@10": 0.27168,
+    "eval_beir-nq_recall@10": 0.44928,
+    "eval_beir-nq_recall@20": 0.56817,
+    "eval_beir-nq_recall@100": 0.79179,
+    "eval_beir-avg_ndcg@10": 0.3805048333333333,
+    "eval_beir-avg_recall@10": 0.4477325,
+    "eval_beir-avg_recall@20": 0.5071286666666668,
+    "eval_beir-avg_recall@100": 0.6274010833333333
+}

eval_output/checkpoint-100000/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.19809666666666667,0.23345500000000005,0.2504741666666667,0.27104833333333334,0.2886541666666667,0.30863666666666667,0.3212183333333334,0.33208166666666666,0.3511625
+map,0.1654,0.20855,0.22002999999999998,0.22955166666666668,0.23495,0.23854166666666665,0.23986416666666666,0.24057000000000003,0.2411416666666667
+precision,0.19809666666666667,0.108695,0.07816833333333333,0.048558333333333335,0.02945083333333333,0.014938333333333333,0.008753333333333333,0.005055833333333333,0.0013183333333333335
+recall,0.1654,0.25717249999999997,0.3013466666666666,0.363245,0.4280566666666667,0.5195883333333333,0.5892308333333333,0.6581233333333333,0.8043724999999999
+mrr,0.19809666666666667,0.24325333333333332,0.2542508333333333,0.26318499999999995,0.26818166666666665,0.27129416666666667,0.2722625,0.2727116666666667,0.2730533333333333
+recall_cap,0.19809666666666667,0.2632625,0.3038458333333333,0.3642008333333333,0.4284308333333333,0.5197316666666667,0.589295,0.6581308333333333,0.8043724999999999
+hole,0.7261233333333333,0.8101108333333334,0.8371108333333334,0.8659116666666665,0.8852641666666666,0.9005008333333332,0.9082033333333333,0.914135,0.9236383333333332

eval_output/checkpoint-100000/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.19809666666666667,
+            "NDCG@3": 0.23345500000000005,
+            "NDCG@5": 0.2504741666666667,
+            "NDCG@10": 0.27104833333333334,
+            "NDCG@20": 0.2886541666666667,
+            "NDCG@50": 0.30863666666666667,
+            "NDCG@100": 0.3212183333333334,
+            "NDCG@200": 0.33208166666666666,
+            "NDCG@1000": 0.3511625
+        },
+        "map": {
+            "MAP@1": 0.1654,
+            "MAP@3": 0.20855,
+            "MAP@5": 0.22002999999999998,
+            "MAP@10": 0.22955166666666668,
+            "MAP@20": 0.23495,
+            "MAP@50": 0.23854166666666665,
+            "MAP@100": 0.23986416666666666,
+            "MAP@200": 0.24057000000000003,
+            "MAP@1000": 0.2411416666666667
+        },
+        "precision": {
+            "P@1": 0.19809666666666667,
+            "P@3": 0.108695,
+            "P@5": 0.07816833333333333,
+            "P@10": 0.048558333333333335,
+            "P@20": 0.02945083333333333,
+            "P@50": 0.014938333333333333,
+            "P@100": 0.008753333333333333,
+            "P@200": 0.005055833333333333,
+            "P@1000": 0.0013183333333333335
+        },
+        "recall": {
+            "Recall@1": 0.1654,
+            "Recall@3": 0.25717249999999997,
+            "Recall@5": 0.3013466666666666,
+            "Recall@10": 0.363245,
+            "Recall@20": 0.4280566666666667,
+            "Recall@50": 0.5195883333333333,
+            "Recall@100": 0.5892308333333333,
+            "Recall@200": 0.6581233333333333,
+            "Recall@1000": 0.8043724999999999
+        },
+        "mrr": {
+            "MRR@1": 0.19809666666666667,
+            "MRR@3": 0.24325333333333332,
+            "MRR@5": 0.2542508333333333,
+            "MRR@10": 0.26318499999999995,
+            "MRR@20": 0.26818166666666665,
+            "MRR@50": 0.27129416666666667,
+            "MRR@100": 0.2722625,
+            "MRR@200": 0.2727116666666667,
+            "MRR@1000": 0.2730533333333333
+        },
+        "recall_cap": {
+            "R_cap@1": 0.19809666666666667,
+            "R_cap@3": 0.2632625,
+            "R_cap@5": 0.3038458333333333,
+            "R_cap@10": 0.3642008333333333,
+            "R_cap@20": 0.4284308333333333,
+            "R_cap@50": 0.5197316666666667,
+            "R_cap@100": 0.589295,
+            "R_cap@200": 0.6581308333333333,
+            "R_cap@1000": 0.8043724999999999
+        },
+        "hole": {
+            "Hole@1": 0.7261233333333333,
+            "Hole@3": 0.8101108333333334,
+            "Hole@5": 0.8371108333333334,
+            "Hole@10": 0.8659116666666665,
+            "Hole@20": 0.8852641666666666,
+            "Hole@50": 0.9005008333333332,
+            "Hole@100": 0.9082033333333333,
+            "Hole@200": 0.914135,
+            "Hole@1000": 0.9236383333333332
+        }
+    }
+}

eval_output/checkpoint-100000/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.19599,0.19067,0.20212,0.22343,0.24275,0.2715,0.28951,0.30503,0.33352
+map,0.10225,0.14666,0.15792,0.16937,0.1754,0.18108,0.18323,0.18436,0.18536
+precision,0.19599,0.12346,0.09383,0.0625,0.03927,0.02133,0.01307,0.00758,0.00206
+recall,0.10225,0.1808,0.22063,0.28095,0.3412,0.45427,0.53699,0.62439,0.80906
+mrr,0.19599,0.24306,0.2591,0.2699,0.27566,0.27965,0.28089,0.28139,0.28168
+recall_cap,0.19599,0.19856,0.22629,0.28144,0.3412,0.45427,0.53699,0.62439,0.80906
+hole,0.75926,0.82767,0.85031,0.88009,0.90255,0.9196,0.92806,0.93346,0.9413

eval_output/checkpoint-100000/fiqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fiqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.19599,
+            "NDCG@3": 0.19067,
+            "NDCG@5": 0.20212,
+            "NDCG@10": 0.22343,
+            "NDCG@20": 0.24275,
+            "NDCG@50": 0.2715,
+            "NDCG@100": 0.28951,
+            "NDCG@200": 0.30503,
+            "NDCG@1000": 0.33352
+        },
+        "map": {
+            "MAP@1": 0.10225,
+            "MAP@3": 0.14666,
+            "MAP@5": 0.15792,
+            "MAP@10": 0.16937,
+            "MAP@20": 0.1754,
+            "MAP@50": 0.18108,
+            "MAP@100": 0.18323,
+            "MAP@200": 0.18436,
+            "MAP@1000": 0.18536
+        },
+        "precision": {
+            "P@1": 0.19599,
+            "P@3": 0.12346,
+            "P@5": 0.09383,
+            "P@10": 0.0625,
+            "P@20": 0.03927,
+            "P@50": 0.02133,
+            "P@100": 0.01307,
+            "P@200": 0.00758,
+            "P@1000": 0.00206
+        },
+        "recall": {
+            "Recall@1": 0.10225,
+            "Recall@3": 0.1808,
+            "Recall@5": 0.22063,
+            "Recall@10": 0.28095,
+            "Recall@20": 0.3412,
+            "Recall@50": 0.45427,
+            "Recall@100": 0.53699,
+            "Recall@200": 0.62439,
+            "Recall@1000": 0.80906
+        },
+        "mrr": {
+            "MRR@1": 0.19599,
+            "MRR@3": 0.24306,
+            "MRR@5": 0.2591,
+            "MRR@10": 0.2699,
+            "MRR@20": 0.27566,
+            "MRR@50": 0.27965,
+            "MRR@100": 0.28089,
+            "MRR@200": 0.28139,
+            "MRR@1000": 0.28168
+        },
+        "recall_cap": {
+            "R_cap@1": 0.19599,
+            "R_cap@3": 0.19856,
+            "R_cap@5": 0.22629,
+            "R_cap@10": 0.28144,
+            "R_cap@20": 0.3412,
+            "R_cap@50": 0.45427,
+            "R_cap@100": 0.53699,
+            "R_cap@200": 0.62439,
+            "R_cap@1000": 0.80906
+        },
+        "hole": {
+            "Hole@1": 0.75926,
+            "Hole@3": 0.82767,
+            "Hole@5": 0.85031,
+            "Hole@10": 0.88009,
+            "Hole@20": 0.90255,
+            "Hole@50": 0.9196,
+            "Hole@100": 0.92806,
+            "Hole@200": 0.93346,
+            "Hole@1000": 0.9413
+        }
+    }
+}

eval_output/checkpoint-100000/nfcorpus.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.39628,0.35563,0.33507,0.30319,0.28103,0.27022,0.27658,0.29045,0.36604
+map,0.04598,0.08066,0.09348,0.109,0.11899,0.12836,0.13388,0.13829,0.14703
+precision,0.40867,0.33643,0.28916,0.22105,0.16316,0.10483,0.07115,0.04785,0.01998
+recall,0.04598,0.09097,0.1183,0.1525,0.18001,0.2298,0.27586,0.33212,0.60087
+mrr,0.41486,0.48607,0.49985,0.50818,0.51178,0.51365,0.51418,0.51443,0.51466
+recall_cap,0.41486,0.35294,0.32626,0.28389,0.25688,0.26052,0.2899,0.33708,0.60087
+hole,0.07121,0.07327,0.08111,0.09783,0.10913,0.11622,0.12105,0.12596,0.13346

eval_output/checkpoint-100000/nfcorpus.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nfcorpus",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.39628,
+            "NDCG@3": 0.35563,
+            "NDCG@5": 0.33507,
+            "NDCG@10": 0.30319,
+            "NDCG@20": 0.28103,
+            "NDCG@50": 0.27022,
+            "NDCG@100": 0.27658,
+            "NDCG@200": 0.29045,
+            "NDCG@1000": 0.36604
+        },
+        "map": {
+            "MAP@1": 0.04598,
+            "MAP@3": 0.08066,
+            "MAP@5": 0.09348,
+            "MAP@10": 0.109,
+            "MAP@20": 0.11899,
+            "MAP@50": 0.12836,
+            "MAP@100": 0.13388,
+            "MAP@200": 0.13829,
+            "MAP@1000": 0.14703
+        },
+        "precision": {
+            "P@1": 0.40867,
+            "P@3": 0.33643,
+            "P@5": 0.28916,
+            "P@10": 0.22105,
+            "P@20": 0.16316,
+            "P@50": 0.10483,
+            "P@100": 0.07115,
+            "P@200": 0.04785,
+            "P@1000": 0.01998
+        },
+        "recall": {
+            "Recall@1": 0.04598,
+            "Recall@3": 0.09097,
+            "Recall@5": 0.1183,
+            "Recall@10": 0.1525,
+            "Recall@20": 0.18001,
+            "Recall@50": 0.2298,
+            "Recall@100": 0.27586,
+            "Recall@200": 0.33212,
+            "Recall@1000": 0.60087
+        },
+        "mrr": {
+            "MRR@1": 0.41486,
+            "MRR@3": 0.48607,
+            "MRR@5": 0.49985,
+            "MRR@10": 0.50818,
+            "MRR@20": 0.51178,
+            "MRR@50": 0.51365,
+            "MRR@100": 0.51418,
+            "MRR@200": 0.51443,
+            "MRR@1000": 0.51466
+        },
+        "recall_cap": {
+            "R_cap@1": 0.41486,
+            "R_cap@3": 0.35294,
+            "R_cap@5": 0.32626,
+            "R_cap@10": 0.28389,
+            "R_cap@20": 0.25688,
+            "R_cap@50": 0.26052,
+            "R_cap@100": 0.2899,
+            "R_cap@200": 0.33708,
+            "R_cap@1000": 0.60087
+        },
+        "hole": {
+            "Hole@1": 0.07121,
+            "Hole@3": 0.07327,
+            "Hole@5": 0.08111,
+            "Hole@10": 0.09783,
+            "Hole@20": 0.10913,
+            "Hole@50": 0.11622,
+            "Hole@100": 0.12105,
+            "Hole@200": 0.12596,
+            "Hole@1000": 0.13346
+        }
+    }
+}

eval_output/checkpoint-100000/nq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.13094,0.19415,0.22982,0.27168,0.30332,0.33339,0.34724,0.35617,0.36695
+map,0.1146,0.17089,0.19116,0.20918,0.21847,0.22378,0.22513,0.22564,0.22593
+precision,0.13094,0.09347,0.07625,0.0529,0.03379,0.01706,0.00954,0.00515,0.00114
+recall,0.1146,0.24244,0.32525,0.44928,0.56817,0.7114,0.79179,0.85187,0.93909
+mrr,0.13094,0.19047,0.21053,0.22749,0.23559,0.24003,0.24115,0.24157,0.24178
+recall_cap,0.13094,0.24252,0.32525,0.44928,0.56817,0.7114,0.79179,0.85187,0.93909
+hole,0.8653,0.90353,0.9201,0.94354,0.96283,0.97976,0.98731,0.99186,0.99622

eval_output/checkpoint-100000/nq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.13094,
+            "NDCG@3": 0.19415,
+            "NDCG@5": 0.22982,
+            "NDCG@10": 0.27168,
+            "NDCG@20": 0.30332,
+            "NDCG@50": 0.33339,
+            "NDCG@100": 0.34724,
+            "NDCG@200": 0.35617,
+            "NDCG@1000": 0.36695
+        },
+        "map": {
+            "MAP@1": 0.1146,
+            "MAP@3": 0.17089,
+            "MAP@5": 0.19116,
+            "MAP@10": 0.20918,
+            "MAP@20": 0.21847,
+            "MAP@50": 0.22378,
+            "MAP@100": 0.22513,
+            "MAP@200": 0.22564,
+            "MAP@1000": 0.22593
+        },
+        "precision": {
+            "P@1": 0.13094,
+            "P@3": 0.09347,
+            "P@5": 0.07625,
+            "P@10": 0.0529,
+            "P@20": 0.03379,
+            "P@50": 0.01706,
+            "P@100": 0.00954,
+            "P@200": 0.00515,
+            "P@1000": 0.00114
+        },
+        "recall": {
+            "Recall@1": 0.1146,
+            "Recall@3": 0.24244,
+            "Recall@5": 0.32525,
+            "Recall@10": 0.44928,
+            "Recall@20": 0.56817,
+            "Recall@50": 0.7114,
+            "Recall@100": 0.79179,
+            "Recall@200": 0.85187,
+            "Recall@1000": 0.93909
+        },
+        "mrr": {
+            "MRR@1": 0.13094,
+            "MRR@3": 0.19047,
+            "MRR@5": 0.21053,
+            "MRR@10": 0.22749,
+            "MRR@20": 0.23559,
+            "MRR@50": 0.24003,
+            "MRR@100": 0.24115,
+            "MRR@200": 0.24157,
+            "MRR@1000": 0.24178
+        },
+        "recall_cap": {
+            "R_cap@1": 0.13094,
+            "R_cap@3": 0.24252,
+            "R_cap@5": 0.32525,
+            "R_cap@10": 0.44928,
+            "R_cap@20": 0.56817,
+            "R_cap@50": 0.7114,
+            "R_cap@100": 0.79179,
+            "R_cap@200": 0.85187,
+            "R_cap@1000": 0.93909
+        },
+        "hole": {
+            "Hole@1": 0.8653,
+            "Hole@3": 0.90353,
+            "Hole@5": 0.9201,
+            "Hole@10": 0.94354,
+            "Hole@20": 0.96283,
+            "Hole@50": 0.97976,
+            "Hole@100": 0.98731,
+            "Hole@200": 0.99186,
+            "Hole@1000": 0.99622
+        }
+    }
+}

eval_output/checkpoint-100000/quora.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.7432,0.79063,0.80987,0.82685,0.83677,0.84336,0.8458,0.84709,0.84815
+map,0.64738,0.75051,0.76936,0.78116,0.78602,0.78822,0.78872,0.7889,0.78899
+precision,0.7432,0.34417,0.2278,0.1256,0.06758,0.02883,0.01487,0.00759,0.00156
+recall,0.64738,0.81602,0.86876,0.91814,0.95101,0.97574,0.98627,0.99241,0.99806
+mrr,0.7428,0.80358,0.81272,0.81777,0.81933,0.81978,0.81985,0.81987,0.81988
+recall_cap,0.7428,0.82975,0.87403,0.9196,0.95133,0.97577,0.98627,0.99241,0.99806
+hole,0.2415,0.61983,0.73188,0.83085,0.8874,0.92607,0.94109,0.94999,0.95891

eval_output/checkpoint-100000/quora.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "quora",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.7432,
+            "NDCG@3": 0.79063,
+            "NDCG@5": 0.80987,
+            "NDCG@10": 0.82685,
+            "NDCG@20": 0.83677,
+            "NDCG@50": 0.84336,
+            "NDCG@100": 0.8458,
+            "NDCG@200": 0.84709,
+            "NDCG@1000": 0.84815
+        },
+        "map": {
+            "MAP@1": 0.64738,
+            "MAP@3": 0.75051,
+            "MAP@5": 0.76936,
+            "MAP@10": 0.78116,
+            "MAP@20": 0.78602,
+            "MAP@50": 0.78822,
+            "MAP@100": 0.78872,
+            "MAP@200": 0.7889,
+            "MAP@1000": 0.78899
+        },
+        "precision": {
+            "P@1": 0.7432,
+            "P@3": 0.34417,
+            "P@5": 0.2278,
+            "P@10": 0.1256,
+            "P@20": 0.06758,
+            "P@50": 0.02883,
+            "P@100": 0.01487,
+            "P@200": 0.00759,
+            "P@1000": 0.00156
+        },
+        "recall": {
+            "Recall@1": 0.64738,
+            "Recall@3": 0.81602,
+            "Recall@5": 0.86876,
+            "Recall@10": 0.91814,
+            "Recall@20": 0.95101,
+            "Recall@50": 0.97574,
+            "Recall@100": 0.98627,
+            "Recall@200": 0.99241,
+            "Recall@1000": 0.99806
+        },
+        "mrr": {
+            "MRR@1": 0.7428,
+            "MRR@3": 0.80358,
+            "MRR@5": 0.81272,
+            "MRR@10": 0.81777,
+            "MRR@20": 0.81933,
+            "MRR@50": 0.81978,
+            "MRR@100": 0.81985,
+            "MRR@200": 0.81987,
+            "MRR@1000": 0.81988
+        },
+        "recall_cap": {
+            "R_cap@1": 0.7428,
+            "R_cap@3": 0.82975,
+            "R_cap@5": 0.87403,
+            "R_cap@10": 0.9196,
+            "R_cap@20": 0.95133,
+            "R_cap@50": 0.97577,
+            "R_cap@100": 0.98627,
+            "R_cap@200": 0.99241,
+            "R_cap@1000": 0.99806
+        },
+        "hole": {
+            "Hole@1": 0.2415,
+            "Hole@3": 0.61983,
+            "Hole@5": 0.73188,
+            "Hole@10": 0.83085,
+            "Hole@20": 0.8874,
+            "Hole@50": 0.92607,
+            "Hole@100": 0.94109,
+            "Hole@200": 0.94999,
+            "Hole@1000": 0.95891
+        }
+    }
+}

eval_output/checkpoint-100000/scidocs.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.169,0.13739,0.12099,0.14703,0.17228,0.19858,0.21515,0.23246,0.26731
+map,0.03438,0.06,0.07152,0.08275,0.09146,0.09698,0.09924,0.10063,0.1019
+precision,0.169,0.12767,0.106,0.0766,0.05295,0.02898,0.01748,0.01055,0.003
+recall,0.03438,0.07768,0.10748,0.15518,0.21462,0.29352,0.35427,0.42755,0.60878
+mrr,0.169,0.23117,0.24962,0.26426,0.27122,0.27454,0.27545,0.27598,0.27629
+recall_cap,0.169,0.12767,0.10748,0.15518,0.21462,0.29352,0.35427,0.42755,0.60878
+hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0