Skip to content

Commit

Permalink
Merge pull request #9 from embeddings-benchmark/add-bm25s-results
Browse files Browse the repository at this point in the history 
add BM25s english retrieval results
  • Loading branch information
@Muennighoff
Muennighoff authored Jul 17, 2024
2 parents 1cc8a8e + fea2423 commit 1c32043
Show file tree
Hide file tree
Showing 27 changed files with 5,905 additions and 0 deletions.
158 changes: 158 additions & 0 deletions results/bm25s/0_1_10/ArguAna.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a",
"evaluation_time": 5.660568475723267,
"kg_co2_emissions": null,
"mteb_version": "1.12.22",
"scores": {
"test": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.49276,
"map_at_1": 0.24609,
"map_at_10": 0.40338,
"map_at_100": 0.41342,
"map_at_1000": 0.41355,
"map_at_20": 0.41109,
"map_at_3": 0.351,
"map_at_5": 0.38129,
"mrr_at_1": 0.25604551920341395,
"mrr_at_10": 0.40720748944433144,
"mrr_at_100": 0.4170380307552231,
"mrr_at_1000": 0.41717558754366546,
"mrr_at_20": 0.4146724769107446,
"mrr_at_3": 0.3560929350403027,
"mrr_at_5": 0.3846491228070172,
"nauc_map_at_1000_diff1": 0.021886370060854626,
"nauc_map_at_1000_max": -0.050589239073000766,
"nauc_map_at_1000_std": -0.047255238785213136,
"nauc_map_at_100_diff1": 0.021991848534549694,
"nauc_map_at_100_max": -0.05036037681015677,
"nauc_map_at_100_std": -0.046991639959645434,
"nauc_map_at_10_diff1": 0.01944695569349535,
"nauc_map_at_10_max": -0.05135259543708522,
"nauc_map_at_10_std": -0.04932007591740704,
"nauc_map_at_1_diff1": 0.03701924027832177,
"nauc_map_at_1_max": -0.10073437912329636,
"nauc_map_at_1_std": -0.09714228089183849,
"nauc_map_at_20_diff1": 0.021074707557638035,
"nauc_map_at_20_max": -0.04862337290352727,
"nauc_map_at_20_std": -0.04543320111520716,
"nauc_map_at_3_diff1": 0.018847825117445496,
"nauc_map_at_3_max": -0.0633251115176233,
"nauc_map_at_3_std": -0.06117393852247131,
"nauc_map_at_5_diff1": 0.020660933587840602,
"nauc_map_at_5_max": -0.05322561221432506,
"nauc_map_at_5_std": -0.05039034499825575,
"nauc_mrr_at_1000_diff1": -0.0069916774432518996,
"nauc_mrr_at_1000_max": -0.05333586260967066,
"nauc_mrr_at_1000_std": -0.04418644510256895,
"nauc_mrr_at_100_diff1": -0.006874087285323051,
"nauc_mrr_at_100_max": -0.05310772579098728,
"nauc_mrr_at_100_std": -0.0439264910005182,
"nauc_mrr_at_10_diff1": -0.008377656435482347,
"nauc_mrr_at_10_max": -0.05392075810974482,
"nauc_mrr_at_10_std": -0.04651614968325181,
"nauc_mrr_at_1_diff1": 0.005327314532673115,
"nauc_mrr_at_1_max": -0.08512604423534934,
"nauc_mrr_at_1_std": -0.07834121981070495,
"nauc_mrr_at_20_diff1": -0.0077329431928073195,
"nauc_mrr_at_20_max": -0.05152246898546906,
"nauc_mrr_at_20_std": -0.04257929430502551,
"nauc_mrr_at_3_diff1": -0.004856572445127299,
"nauc_mrr_at_3_max": -0.06240231043434506,
"nauc_mrr_at_3_std": -0.055716396330877466,
"nauc_mrr_at_5_diff1": -0.006755969959898004,
"nauc_mrr_at_5_max": -0.056478414555520336,
"nauc_mrr_at_5_std": -0.04801654234480827,
"nauc_ndcg_at_1000_diff1": 0.022385904672540317,
"nauc_ndcg_at_1000_max": -0.037668490148391166,
"nauc_ndcg_at_1000_std": -0.032296367627030594,
"nauc_ndcg_at_100_diff1": 0.026383707483814727,
"nauc_ndcg_at_100_max": -0.02961668256385589,
"nauc_ndcg_at_100_std": -0.02343964973523178,
"nauc_ndcg_at_10_diff1": 0.013312910506790721,
"nauc_ndcg_at_10_max": -0.029936910520231143,
"nauc_ndcg_at_10_std": -0.030239455934052435,
"nauc_ndcg_at_1_diff1": 0.03701924027832177,
"nauc_ndcg_at_1_max": -0.10073437912329636,
"nauc_ndcg_at_1_std": -0.09714228089183849,
"nauc_ndcg_at_20_diff1": 0.020332954448972276,
"nauc_ndcg_at_20_max": -0.01659652124941782,
"nauc_ndcg_at_20_std": -0.012171448030433511,
"nauc_ndcg_at_3_diff1": 0.014181473782307842,
"nauc_ndcg_at_3_max": -0.055017566324620126,
"nauc_ndcg_at_3_std": -0.05379142189559777,
"nauc_ndcg_at_5_diff1": 0.016594438418392792,
"nauc_ndcg_at_5_max": -0.03635193601664064,
"nauc_ndcg_at_5_std": -0.03396739766745262,
"nauc_precision_at_1000_diff1": 0.04548813525954615,
"nauc_precision_at_1000_max": -0.03741762492642651,
"nauc_precision_at_1000_std": 0.3434893045428403,
"nauc_precision_at_100_diff1": 0.26131642925368015,
"nauc_precision_at_100_max": 0.427745624881254,
"nauc_precision_at_100_std": 0.5507113012498237,
"nauc_precision_at_10_diff1": -0.015344585938079884,
"nauc_precision_at_10_max": 0.0717330694399166,
"nauc_precision_at_10_std": 0.05857802419902309,
"nauc_precision_at_1_diff1": 0.03701924027832177,
"nauc_precision_at_1_max": -0.10073437912329636,
"nauc_precision_at_1_std": -0.09714228089183849,
"nauc_precision_at_20_diff1": 0.030689672237653447,
"nauc_precision_at_20_max": 0.27286736721211624,
"nauc_precision_at_20_std": 0.29357201667169436,
"nauc_precision_at_3_diff1": 0.0015611753676343357,
"nauc_precision_at_3_max": -0.03300014690374674,
"nauc_precision_at_3_std": -0.034468293667097305,
"nauc_precision_at_5_diff1": 0.004303597872377514,
"nauc_precision_at_5_max": 0.01723887778063657,
"nauc_precision_at_5_std": 0.01822794811792894,
"nauc_recall_at_1000_diff1": 0.045488135259533935,
"nauc_recall_at_1000_max": -0.0374176249263954,
"nauc_recall_at_1000_std": 0.3434893045428401,
"nauc_recall_at_100_diff1": 0.26131642925368626,
"nauc_recall_at_100_max": 0.42774562488126144,
"nauc_recall_at_100_std": 0.550711301249821,
"nauc_recall_at_10_diff1": -0.015344585938079393,
"nauc_recall_at_10_max": 0.07173306943991657,
"nauc_recall_at_10_std": 0.05857802419902259,
"nauc_recall_at_1_diff1": 0.03701924027832177,
"nauc_recall_at_1_max": -0.10073437912329636,
"nauc_recall_at_1_std": -0.09714228089183849,
"nauc_recall_at_20_diff1": 0.030689672237654887,
"nauc_recall_at_20_max": 0.27286736721211635,
"nauc_recall_at_20_std": 0.29357201667169563,
"nauc_recall_at_3_diff1": 0.0015611753676343357,
"nauc_recall_at_3_max": -0.033000146903746364,
"nauc_recall_at_3_std": -0.03446829366709721,
"nauc_recall_at_5_diff1": 0.004303597872377939,
"nauc_recall_at_5_max": 0.017238877780636937,
"nauc_recall_at_5_std": 0.018227948117928935,
"ndcg_at_1": 0.24609,
"ndcg_at_10": 0.49276,
"ndcg_at_100": 0.53527,
"ndcg_at_1000": 0.53883,
"ndcg_at_20": 0.51975,
"ndcg_at_3": 0.38507,
"ndcg_at_5": 0.43946,
"precision_at_1": 0.24609,
"precision_at_10": 0.07795,
"precision_at_100": 0.00965,
"precision_at_1000": 0.00099,
"precision_at_20": 0.0442,
"precision_at_3": 0.16121,
"precision_at_5": 0.12304,
"recall_at_1": 0.24609,
"recall_at_10": 0.77952,
"recall_at_100": 0.96515,
"recall_at_1000": 0.99289,
"recall_at_20": 0.88407,
"recall_at_3": 0.48364,
"recall_at_5": 0.61522
}
]
},
"task_name": "ArguAna"
}
158 changes: 158 additions & 0 deletions results/bm25s/0_1_10/CQADupstackAndroidRetrieval.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "f46a197baaae43b4f621051089b82a364682dfeb",
"evaluation_time": 4.685648202896118,
"kg_co2_emissions": null,
"mteb_version": "1.12.80",
"scores": {
"test": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.39693,
"map_at_1": 0.25945,
"map_at_10": 0.34614,
"map_at_100": 0.35768,
"map_at_1000": 0.35892,
"map_at_20": 0.35218,
"map_at_3": 0.31913,
"map_at_5": 0.33394,
"mrr_at_1": 0.3218884120171674,
"mrr_at_10": 0.39754524604310026,
"mrr_at_100": 0.4049561715182821,
"mrr_at_1000": 0.40555047405524475,
"mrr_at_20": 0.4013601001878152,
"mrr_at_3": 0.3745827372436815,
"mrr_at_5": 0.3881020505484025,
"nauc_map_at_1000_diff1": 0.5401042788373607,
"nauc_map_at_1000_max": 0.46446941292616584,
"nauc_map_at_1000_std": 0.1448534325440925,
"nauc_map_at_100_diff1": 0.5399915323525181,
"nauc_map_at_100_max": 0.4641570929554848,
"nauc_map_at_100_std": 0.1444170242192052,
"nauc_map_at_10_diff1": 0.544123002626852,
"nauc_map_at_10_max": 0.46633938701722494,
"nauc_map_at_10_std": 0.14139150808244658,
"nauc_map_at_1_diff1": 0.576256760811547,
"nauc_map_at_1_max": 0.4539253680333111,
"nauc_map_at_1_std": 0.13886111230903941,
"nauc_map_at_20_diff1": 0.5406412193228595,
"nauc_map_at_20_max": 0.4640427163014816,
"nauc_map_at_20_std": 0.14192478286159718,
"nauc_map_at_3_diff1": 0.5565388392220246,
"nauc_map_at_3_max": 0.46657852693639296,
"nauc_map_at_3_std": 0.1414241365632173,
"nauc_map_at_5_diff1": 0.5493623117253394,
"nauc_map_at_5_max": 0.4661875641780336,
"nauc_map_at_5_std": 0.1398507592644627,
"nauc_mrr_at_1000_diff1": 0.5444244071501962,
"nauc_mrr_at_1000_max": 0.4693117952655278,
"nauc_mrr_at_1000_std": 0.15224424700571026,
"nauc_mrr_at_100_diff1": 0.5442781921808686,
"nauc_mrr_at_100_max": 0.4689664786770939,
"nauc_mrr_at_100_std": 0.15180516785703468,
"nauc_mrr_at_10_diff1": 0.546510598770559,
"nauc_mrr_at_10_max": 0.47178794029012106,
"nauc_mrr_at_10_std": 0.1514325118602757,
"nauc_mrr_at_1_diff1": 0.587458620032547,
"nauc_mrr_at_1_max": 0.4839179169052132,
"nauc_mrr_at_1_std": 0.15314613502076443,
"nauc_mrr_at_20_diff1": 0.5438642084003957,
"nauc_mrr_at_20_max": 0.4685892515983097,
"nauc_mrr_at_20_std": 0.15000004328956787,
"nauc_mrr_at_3_diff1": 0.5611595454121813,
"nauc_mrr_at_3_max": 0.4819413050014325,
"nauc_mrr_at_3_std": 0.1622140602258072,
"nauc_mrr_at_5_diff1": 0.5503681989476072,
"nauc_mrr_at_5_max": 0.47437329907861125,
"nauc_mrr_at_5_std": 0.1545662953584148,
"nauc_ndcg_at_1000_diff1": 0.5166912488184727,
"nauc_ndcg_at_1000_max": 0.46091945324253136,
"nauc_ndcg_at_1000_std": 0.1646539581699839,
"nauc_ndcg_at_100_diff1": 0.510840931307867,
"nauc_ndcg_at_100_max": 0.44987548346310813,
"nauc_ndcg_at_100_std": 0.1524670393894962,
"nauc_ndcg_at_10_diff1": 0.521250964842959,
"nauc_ndcg_at_10_max": 0.4589169947078968,
"nauc_ndcg_at_10_std": 0.13809521772037603,
"nauc_ndcg_at_1_diff1": 0.587458620032547,
"nauc_ndcg_at_1_max": 0.4839179169052132,
"nauc_ndcg_at_1_std": 0.15314613502076443,
"nauc_ndcg_at_20_diff1": 0.513078444102445,
"nauc_ndcg_at_20_max": 0.44854573945143655,
"nauc_ndcg_at_20_std": 0.1383467153046283,
"nauc_ndcg_at_3_diff1": 0.5489683241106224,
"nauc_ndcg_at_3_max": 0.47346936137111983,
"nauc_ndcg_at_3_std": 0.15026629693286153,
"nauc_ndcg_at_5_diff1": 0.532738240644177,
"nauc_ndcg_at_5_max": 0.46327370065068596,
"nauc_ndcg_at_5_std": 0.13948065084537742,
"nauc_precision_at_1000_diff1": -0.016452908247665064,
"nauc_precision_at_1000_max": -0.012001214641875885,
"nauc_precision_at_1000_std": -0.007429596498602477,
"nauc_precision_at_100_diff1": 0.043750177743398136,
"nauc_precision_at_100_max": 0.11840841801001449,
"nauc_precision_at_100_std": 0.09423170869894978,
"nauc_precision_at_10_diff1": 0.25775160299406186,
"nauc_precision_at_10_max": 0.31370734595865013,
"nauc_precision_at_10_std": 0.11623518438348741,
"nauc_precision_at_1_diff1": 0.587458620032547,
"nauc_precision_at_1_max": 0.4839179169052132,
"nauc_precision_at_1_std": 0.15314613502076443,
"nauc_precision_at_20_diff1": 0.18275415322892,
"nauc_precision_at_20_max": 0.24214692223337636,
"nauc_precision_at_20_std": 0.1026496751611631,
"nauc_precision_at_3_diff1": 0.4372477308856027,
"nauc_precision_at_3_max": 0.4206846005073202,
"nauc_precision_at_3_std": 0.14288990628895132,
"nauc_precision_at_5_diff1": 0.35601343726042023,
"nauc_precision_at_5_max": 0.3695758503618295,
"nauc_precision_at_5_std": 0.12143486994349435,
"nauc_recall_at_1000_diff1": 0.3290972545754859,
"nauc_recall_at_1000_max": 0.47809942001313116,
"nauc_recall_at_1000_std": 0.4554009253629691,
"nauc_recall_at_100_diff1": 0.3441698585821832,
"nauc_recall_at_100_max": 0.34250128812089764,
"nauc_recall_at_100_std": 0.17695741822462932,
"nauc_recall_at_10_diff1": 0.4326595941825119,
"nauc_recall_at_10_max": 0.4000349281281342,
"nauc_recall_at_10_std": 0.10268244219902738,
"nauc_recall_at_1_diff1": 0.576256760811547,
"nauc_recall_at_1_max": 0.4539253680333111,
"nauc_recall_at_1_std": 0.13886111230903941,
"nauc_recall_at_20_diff1": 0.39204221492763575,
"nauc_recall_at_20_max": 0.3566711669315542,
"nauc_recall_at_20_std": 0.10167314401440865,
"nauc_recall_at_3_diff1": 0.5100005611427266,
"nauc_recall_at_3_max": 0.44693489202542486,
"nauc_recall_at_3_std": 0.14034968317991875,
"nauc_recall_at_5_diff1": 0.4730348111968922,
"nauc_recall_at_5_max": 0.42656953405586634,
"nauc_recall_at_5_std": 0.12175169946828732,
"ndcg_at_1": 0.32189,
"ndcg_at_10": 0.39693,
"ndcg_at_100": 0.44568,
"ndcg_at_1000": 0.47072,
"ndcg_at_20": 0.41324,
"ndcg_at_3": 0.35782,
"ndcg_at_5": 0.37523,
"precision_at_1": 0.32189,
"precision_at_10": 0.07382,
"precision_at_100": 0.01209,
"precision_at_1000": 0.00173,
"precision_at_20": 0.04313,
"precision_at_3": 0.17024,
"precision_at_5": 0.1216,
"recall_at_1": 0.25945,
"recall_at_10": 0.49321,
"recall_at_100": 0.70705,
"recall_at_1000": 0.87218,
"recall_at_20": 0.55432,
"recall_at_3": 0.37485,
"recall_at_5": 0.42614
}
]
},
"task_name": "CQADupstackAndroidRetrieval"
}
Loading

0 comments on commit 1c32043

Please sign in to comment.