PyTerrier demonstration for msmarcov2_document

In [ ]:
import pyterrier as pt
if not pt.started():
    pt.init()

systems=[]
names=[]
dataset = pt.get_dataset('msmarcov2_document')
        
In [ ]:
bm25_terrier_stemmed = pt.BatchRetrieve.from_dataset('msmarcov2_document', 'terrier_stemmed', wmodel='BM25')
systems.append(bm25_terrier_stemmed)
names.append('bm25_terrier_stemmed')


dph_terrier_stemmed = pt.BatchRetrieve.from_dataset('msmarcov2_document', 'terrier_stemmed', wmodel='DPH')
systems.append(dph_terrier_stemmed)
names.append('dph_terrier_stemmed')


dph_bo1_terrier_stemmed = dph_terrier_stemmed >> pt.rewrite.Bo1QueryExpansion(pt.get_dataset('msmarcov2_document').get_index('terrier_stemmed')) >> dph_terrier_stemmed
systems.append(dph_bo1_terrier_stemmed)
names.append('dph_bo1_terrier_stemmed')

Evaluation on valid1 topics and qrels

43 topics used in the TREC 2019 Deep Learning track, with deep judgements

In [ ]:
pt.Experiment(
    systems,
    pt.get_dataset('msmarcov2_document').get_topics('valid1'),
    pt.get_dataset('msmarcov2_document').get_qrels('valid1'),
    batch_size=200,
    drop_unused=True,
    eval_metrics=['ndcg_cut_10'],
    names=names)
        

Evaluation on valid2 topics and qrels

54 topics used in the TREC 2020 Deep Learning track, with deep judgements

In [ ]:
pt.Experiment(
    systems,
    pt.get_dataset('msmarcov2_document').get_topics('valid2'),
    pt.get_dataset('msmarcov2_document').get_qrels('valid2'),
    batch_size=200,
    drop_unused=True,
    eval_metrics=['ndcg_cut_10'],
    names=names)
        

Evaluation on dev1 topics and qrels

4,552 topics with sparse judgements

In [ ]:
pt.Experiment(
    systems,
    pt.get_dataset('msmarcov2_document').get_topics('dev1'),
    pt.get_dataset('msmarcov2_document').get_qrels('dev1'),
    batch_size=200,
    drop_unused=True,
    eval_metrics=['recip_rank'],
    names=names)