| conditions: | |
| - name: bm25-default | |
| display: BM25 original passage (k1=0.9, b=0.4) | |
| display-html: BM25 original passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (1a) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage --topics $topics --output $output --bm25 | |
| topics: | |
| - topic_key: msmarco-v2-passage-dev | |
| eval_key: msmarco-v2-passage-dev | |
| scores: | |
| - MRR@100: 0.0719 | |
| R@1K: 0.5733 | |
| - topic_key: msmarco-v2-passage-dev2 | |
| eval_key: msmarco-v2-passage-dev2 | |
| scores: | |
| - MRR@100: 0.0802 | |
| R@1K: 0.5839 | |
| - topic_key: dl21 | |
| eval_key: dl21-passage | |
| scores: | |
| - MAP@100: 0.1357 | |
| nDCG@10: 0.4458 | |
| MRR@100: 0.5060 | |
| R@100: 0.3261 | |
| R@1K: 0.6149 | |
| - name: bm25-augmented-default | |
| display: BM25 augmented passage (k1=0.9, b=0.4) | |
| display-html: BM25 augmented passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (1b) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-augmented --topics $topics --output $output --bm25 | |
| topics: | |
| - topic_key: msmarco-v2-passage-dev | |
| eval_key: msmarco-v2-passage-dev | |
| scores: | |
| - MRR@100: 0.0872 | |
| R@1K: 0.6925 | |
| - topic_key: msmarco-v2-passage-dev2 | |
| eval_key: msmarco-v2-passage-dev2 | |
| scores: | |
| - MRR@100: 0.0917 | |
| R@1K: 0.6933 | |
| - topic_key: dl21 | |
| eval_key: dl21-passage | |
| scores: | |
| - MAP@100: 0.0977 | |
| nDCG@10: 0.3977 | |
| MRR@100: 0.5303 | |
| R@100: 0.2709 | |
| R@1K: 0.5835 | |
| - name: bm25-rm3-default | |
| display: BM25+RM3 original passage (k1=0.9, b=0.4) | |
| display-html: BM25+RM3 original passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (1c) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage --topics $topics --output $output --bm25 --rm3 | |
| topics: | |
| - topic_key: msmarco-v2-passage-dev | |
| eval_key: msmarco-v2-passage-dev | |
| scores: | |
| - MRR@100: 0.0630 | |
| R@1K: 0.5947 | |
| - topic_key: msmarco-v2-passage-dev2 | |
| eval_key: msmarco-v2-passage-dev2 | |
| scores: | |
| - MRR@100: 0.0659 | |
| R@1K: 0.6062 | |
| - topic_key: dl21 | |
| eval_key: dl21-passage | |
| scores: | |
| - MAP@100: 0.1666 | |
| nDCG@10: 0.4455 | |
| MRR@100: 0.5202 | |
| R@100: 0.3499 | |
| R@1K: 0.6616 | |
| - name: bm25-rm3-augmented-default | |
| display: BM25+RM3 augmented passage (k1=0.9, b=0.4) | |
| display-html: BM25+RM3 augmented passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (1d) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-augmented --topics $topics --output $output --bm25 --rm3 | |
| topics: | |
| - topic_key: msmarco-v2-passage-dev | |
| eval_key: msmarco-v2-passage-dev | |
| scores: | |
| - MRR@100: 0.0667 | |
| R@1K: 0.6857 | |
| - topic_key: msmarco-v2-passage-dev2 | |
| eval_key: msmarco-v2-passage-dev2 | |
| scores: | |
| - MRR@100: 0.0700 | |
| R@1K: 0.6826 | |
| - topic_key: dl21 | |
| eval_key: dl21-passage | |
| scores: | |
| - MAP@100: 0.1050 | |
| nDCG@10: 0.3869 | |
| MRR@100: 0.4915 | |
| R@100: 0.2807 | |
| R@1K: 0.6298 | |
| - name: bm25-d2q-t5-default | |
| display: BM25 w/ doc2query-T5 original passage (k1=0.9, b=0.4) | |
| display-html: BM25 w/ doc2query-T5 original passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (2a) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-d2q-t5 --topics $topics --output $output --bm25 | |
| topics: | |
| - topic_key: msmarco-v2-passage-dev | |
| eval_key: msmarco-v2-passage-dev | |
| scores: | |
| - MRR@100: 0.1072 | |
| R@1K: 0.7083 | |
| - topic_key: msmarco-v2-passage-dev2 | |
| eval_key: msmarco-v2-passage-dev2 | |
| scores: | |
| - MRR@100: 0.1123 | |
| R@1K: 0.7151 | |
| - topic_key: dl21 | |
| eval_key: dl21-passage | |
| scores: | |
| - MAP@100: 0.1874 | |
| nDCG@10: 0.4816 | |
| MRR@100: 0.6848 | |
| R@100: 0.4076 | |
| R@1K: 0.7078 | |
| - name: bm25-d2q-t5-augmented-default | |
| display: BM25 w/ doc2query-T5 augmented passage (k1=0.9, b=0.4) | |
| display-html: BM25 w/ doc2query-T5 augmented passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (2b) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-augmented-d2q-t5 --topics $topics --output $output --bm25 | |
| topics: | |
| - topic_key: msmarco-v2-passage-dev | |
| eval_key: msmarco-v2-passage-dev | |
| scores: | |
| - MRR@100: 0.1172 | |
| R@1K: 0.7647 | |
| - topic_key: msmarco-v2-passage-dev2 | |
| eval_key: msmarco-v2-passage-dev2 | |
| scores: | |
| - MRR@100: 0.1170 | |
| R@1K: 0.7659 | |
| - topic_key: dl21 | |
| eval_key: dl21-passage | |
| scores: | |
| - MAP@100: 0.1649 | |
| nDCG@10: 0.4702 | |
| MRR@100: 0.6391 | |
| R@100: 0.3883 | |
| R@1K: 0.6962 | |
| - name: bm25-rm3-d2q-t5-default | |
| display: BM25+RM3 w/ doc2query-T5 original passage (k1=0.9, b=0.4) | |
| display-html: BM25+RM3 w/ doc2query-T5 original passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (2c) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 | |
| topics: | |
| - topic_key: msmarco-v2-passage-dev | |
| eval_key: msmarco-v2-passage-dev | |
| scores: | |
| - MRR@100: 0.0947 | |
| R@1K: 0.7181 | |
| - topic_key: msmarco-v2-passage-dev2 | |
| eval_key: msmarco-v2-passage-dev2 | |
| scores: | |
| - MRR@100: 0.0984 | |
| R@1K: 0.7222 | |
| - topic_key: dl21 | |
| eval_key: dl21-passage | |
| scores: | |
| - MAP@100: 0.2285 | |
| nDCG@10: 0.5098 | |
| MRR@100: 0.6548 | |
| R@100: 0.4499 | |
| R@1K: 0.7537 | |
| - name: bm25-rm3-d2q-t5-augmented-default | |
| display: BM25+RM3 w/ doc2query-T5 augmented passage (k1=0.9, b=0.4) | |
| display-html: BM25+RM3 w/ doc2query-T5 augmented passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (2d) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-augmented-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 | |
| topics: | |
| - topic_key: msmarco-v2-passage-dev | |
| eval_key: msmarco-v2-passage-dev | |
| scores: | |
| - MRR@100: 0.0883 | |
| R@1K: 0.7607 | |
| - topic_key: msmarco-v2-passage-dev2 | |
| eval_key: msmarco-v2-passage-dev2 | |
| scores: | |
| - MRR@100: 0.0904 | |
| R@1K: 0.7649 | |
| - topic_key: dl21 | |
| eval_key: dl21-passage | |
| scores: | |
| - MAP@100: 0.1930 | |
| nDCG@10: 0.4812 | |
| MRR@100: 0.5958 | |
| R@100: 0.4321 | |
| R@1K: 0.7672 | |
| - name: unicoil | |
| display: "uniCOIL (w/ doc2query-T5): pre-encoded" | |
| display-html: "uniCOIL (w/ doc2query-T5): pre-encoded queries" | |
| display-row: (3b) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-unicoil-0shot --topics $topics --output $output --hits 1000 --impact | |
| topics: | |
| - topic_key: msmarco-v2-passage-dev-unicoil | |
| eval_key: msmarco-v2-passage-dev | |
| scores: | |
| - MRR@100: 0.1499 | |
| R@1K: 0.7616 | |
| - topic_key: msmarco-v2-passage-dev2-unicoil | |
| eval_key: msmarco-v2-passage-dev2 | |
| scores: | |
| - MRR@100: 0.1577 | |
| R@1K: 0.7671 | |
| - topic_key: dl21-unicoil | |
| eval_key: dl21-passage | |
| scores: | |
| - MAP@100: 0.2538 | |
| nDCG@10: 0.6159 | |
| MRR@100: 0.7311 | |
| R@100: 0.4731 | |
| R@1K: 0.7551 | |
| - name: unicoil-otf | |
| display: "uniCOIL (w/ doc2query-T5): query inference with PyTorch" | |
| display-html: "uniCOIL (w/ doc2query-T5): query inference with PyTorch" | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-unicoil-0shot --topics $topics --encoder castorini/unicoil-msmarco-passage --output $output --hits 1000 --impact | |
| topics: | |
| - topic_key: msmarco-v2-passage-dev | |
| eval_key: msmarco-v2-passage-dev | |
| scores: | |
| - MRR@100: 0.1501 | |
| R@1K: 0.7613 | |
| - topic_key: msmarco-v2-passage-dev2 | |
| eval_key: msmarco-v2-passage-dev2 | |
| scores: | |
| - MRR@100: 0.1576 | |
| R@1K: 0.7676 | |
| - topic_key: dl21 | |
| eval_key: dl21-passage | |
| scores: | |
| - MAP@100: 0.2539 | |
| nDCG@10: 0.6160 | |
| MRR@100: 0.7311 | |
| R@100: 0.4723 | |
| R@1K: 0.7560 | |
| - name: unicoil-noexp | |
| display: "uniCOIL (noexp): pre-encoded" | |
| display-html: "uniCOIL (noexp): pre-encoded queries" | |
| display-row: (3a) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-unicoil-noexp-0shot --topics $topics --output $output --hits 1000 --impact | |
| topics: | |
| - topic_key: msmarco-v2-passage-dev-unicoil-noexp | |
| eval_key: msmarco-v2-passage-dev | |
| scores: | |
| - MRR@100: 0.1342 | |
| R@1K: 0.7010 | |
| - topic_key: msmarco-v2-passage-dev2-unicoil-noexp | |
| eval_key: msmarco-v2-passage-dev2 | |
| scores: | |
| - MRR@100: 0.1385 | |
| R@1K: 0.7114 | |
| - topic_key: dl21-unicoil-noexp | |
| eval_key: dl21-passage | |
| scores: | |
| - MAP@100: 0.2193 | |
| nDCG@10: 0.5756 | |
| MRR@100: 0.6991 | |
| R@100: 0.4246 | |
| R@1K: 0.6897 | |
| - name: unicoil-noexp-otf | |
| display: "uniCOIL (noexp): query inference with PyTorch" | |
| display-html: "uniCOIL (noexp): query inference with PyTorch" | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-unicoil-noexp-0shot --topics $topics --encoder castorini/unicoil-noexp-msmarco-passage --output $output --hits 1000 --impact | |
| topics: | |
| - topic_key: msmarco-v2-passage-dev | |
| eval_key: msmarco-v2-passage-dev | |
| scores: | |
| - MRR@100: 0.1343 | |
| R@1K: 0.7010 | |
| - topic_key: msmarco-v2-passage-dev2 | |
| eval_key: msmarco-v2-passage-dev2 | |
| scores: | |
| - MRR@100: 0.1385 | |
| R@1K: 0.7114 | |
| - topic_key: dl21 | |
| eval_key: dl21-passage | |
| scores: | |
| - MAP@100: 0.2194 | |
| nDCG@10: 0.5759 | |
| MRR@100: 0.6991 | |
| R@100: 0.4247 | |
| R@1K: 0.6893 | |