{ "arxivqa_test_subsampled": { "ndcg_at_1": 0.844, "ndcg_at_3": 0.88524, "ndcg_at_5": 0.88954, "ndcg_at_10": 0.89512, "ndcg_at_20": 0.90085, "ndcg_at_50": 0.90479, "ndcg_at_100": 0.90578, "map_at_1": 0.844, "map_at_3": 0.87467, "map_at_5": 0.87717, "map_at_10": 0.87933, "map_at_20": 0.88099, "map_at_50": 0.88161, "map_at_100": 0.8817, "recall_at_1": 0.844, "recall_at_3": 0.916, "recall_at_5": 0.926, "recall_at_10": 0.944, "recall_at_20": 0.966, "recall_at_50": 0.986, "recall_at_100": 0.992, "precision_at_1": 0.844, "precision_at_3": 0.30533, "precision_at_5": 0.1852, "precision_at_10": 0.0944, "precision_at_20": 0.0483, "precision_at_50": 0.01972, "precision_at_100": 0.00992, "mrr_at_1": 0.844, "mrr_at_3": 0.8746666666666665, "mrr_at_5": 0.8771666666666665, "mrr_at_10": 0.8793301587301586, "mrr_at_20": 0.880986183261183, "mrr_at_50": 0.8816066058267283, "mrr_at_100": 0.8816959272950264, "naucs_at_1_max": 0.7413901379085128, "naucs_at_1_std": 0.3454872013866209, "naucs_at_1_diff1": 0.9600906830113787, "naucs_at_3_max": 0.7713307545240329, "naucs_at_3_std": 0.4801698457160663, "naucs_at_3_diff1": 0.9489240140500664, "naucs_at_5_max": 0.7514699573523106, "naucs_at_5_std": 0.4375552022610836, "naucs_at_5_diff1": 0.9526206879148043, "naucs_at_10_max": 0.8086901427237575, "naucs_at_10_std": 0.5144891289849284, "naucs_at_10_diff1": 0.9513972255568919, "naucs_at_20_max": 0.907453177349375, "naucs_at_20_std": 0.5683802932937894, "naucs_at_20_diff1": 0.9692425990003846, "naucs_at_50_max": 0.8709483793517359, "naucs_at_50_std": 0.7055488862211612, "naucs_at_50_diff1": 0.9626517273576126, "naucs_at_100_max": 0.8068394024276366, "naucs_at_100_std": 0.7076330532212914, "naucs_at_100_diff1": 0.9673202614378978 }, "docvqa_test_subsampled": { "ndcg_at_1": 0.52328, "ndcg_at_3": 0.5841, "ndcg_at_5": 0.59975, "ndcg_at_10": 0.62669, "ndcg_at_20": 0.64245, "ndcg_at_50": 0.65661, "ndcg_at_100": 0.66492, "map_at_1": 0.52328, "map_at_3": 0.56911, "map_at_5": 0.57786, "map_at_10": 0.58881, "map_at_20": 0.59317, "map_at_50": 0.59548, "map_at_100": 0.59622, "recall_at_1": 0.52328, "recall_at_3": 0.62749, "recall_at_5": 0.66519, "recall_at_10": 0.74945, "recall_at_20": 0.81153, "recall_at_50": 0.88248, "recall_at_100": 0.93348, "precision_at_1": 0.52328, "precision_at_3": 0.20916, "precision_at_5": 0.13304, "precision_at_10": 0.07494, "precision_at_20": 0.04058, "precision_at_50": 0.01765, "precision_at_100": 0.00933, "mrr_at_1": 0.5232815964523282, "mrr_at_3": 0.5691056910569108, "mrr_at_5": 0.5778640059127865, "mrr_at_10": 0.5888132193010243, "mrr_at_20": 0.5931663069177401, "mrr_at_50": 0.5954783504735428, "mrr_at_100": 0.5962169799244146, "naucs_at_1_max": 0.46089368028029637, "naucs_at_1_std": 0.19359243300005127, "naucs_at_1_diff1": 0.8483527783001977, "naucs_at_3_max": 0.4640279399849662, "naucs_at_3_std": 0.1814509120980464, "naucs_at_3_diff1": 0.7719022256243834, "naucs_at_5_max": 0.45716016762761796, "naucs_at_5_std": 0.16428980258139747, "naucs_at_5_diff1": 0.750196647594659, "naucs_at_10_max": 0.3956528364820721, "naucs_at_10_std": 0.09973122080056422, "naucs_at_10_diff1": 0.7237863238311393, "naucs_at_20_max": 0.35927664451426317, "naucs_at_20_std": 0.09080366240903168, "naucs_at_20_diff1": 0.6946736504983693, "naucs_at_50_max": 0.3626447370884348, "naucs_at_50_std": 0.2775120087087966, "naucs_at_50_diff1": 0.6534710933108262, "naucs_at_100_max": 0.32155287639122004, "naucs_at_100_std": 0.3495021025151782, "naucs_at_100_diff1": 0.6165810885563539 }, "infovqa_test_subsampled": { "ndcg_at_1": 0.90283, "ndcg_at_3": 0.93062, "ndcg_at_5": 0.93567, "ndcg_at_10": 0.93969, "ndcg_at_20": 0.94324, "ndcg_at_50": 0.94401, "ndcg_at_100": 0.945, "map_at_1": 0.90283, "map_at_3": 0.92409, "map_at_5": 0.92692, "map_at_10": 0.92863, "map_at_20": 0.92959, "map_at_50": 0.9297, "map_at_100": 0.92979, "recall_at_1": 0.90283, "recall_at_3": 0.94939, "recall_at_5": 0.96154, "recall_at_10": 0.97368, "recall_at_20": 0.98785, "recall_at_50": 0.9919, "recall_at_100": 0.99798, "precision_at_1": 0.90283, "precision_at_3": 0.31646, "precision_at_5": 0.19231, "precision_at_10": 0.09737, "precision_at_20": 0.04939, "precision_at_50": 0.01984, "precision_at_100": 0.00998, "mrr_at_1": 0.902834008097166, "mrr_at_3": 0.9240890688259108, "mrr_at_5": 0.9269230769230767, "mrr_at_10": 0.9286316753422016, "mrr_at_20": 0.9295898610333593, "mrr_at_50": 0.929699602843506, "mrr_at_100": 0.929788457049907, "naucs_at_1_max": 0.6026903076230651, "naucs_at_1_std": 0.261936050485784, "naucs_at_1_diff1": 0.9396804875719484, "naucs_at_3_max": 0.7565375225904929, "naucs_at_3_std": 0.45980620999702715, "naucs_at_3_diff1": 0.9534218386220948, "naucs_at_5_max": 0.8235249494008307, "naucs_at_5_std": 0.5316999544043512, "naucs_at_5_diff1": 0.9524604670358964, "naucs_at_10_max": 0.8684766575602219, "naucs_at_10_std": 0.5944713216706646, "naucs_at_10_diff1": 0.9405654098266761, "naucs_at_20_max": 0.7830887900175995, "naucs_at_20_std": 0.5643438299512757, "naucs_at_20_diff1": 0.8929919636352566, "naucs_at_50_max": 0.7072835485426375, "naucs_at_50_std": 0.5764614839135555, "naucs_at_50_diff1": 0.8394879454528887, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "tabfquad_test_subsampled": { "ndcg_at_1": 0.9, "ndcg_at_3": 0.94685, "ndcg_at_5": 0.95131, "ndcg_at_10": 0.95366, "ndcg_at_20": 0.95455, "ndcg_at_50": 0.9553, "ndcg_at_100": 0.9553, "map_at_1": 0.9, "map_at_3": 0.9369, "map_at_5": 0.9394, "map_at_10": 0.9404, "map_at_20": 0.94063, "map_at_50": 0.94077, "map_at_100": 0.94077, "recall_at_1": 0.9, "recall_at_3": 0.975, "recall_at_5": 0.98571, "recall_at_10": 0.99286, "recall_at_20": 0.99643, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.9, "precision_at_3": 0.325, "precision_at_5": 0.19714, "precision_at_10": 0.09929, "precision_at_20": 0.04982, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.9, "mrr_at_3": 0.936904761904762, "mrr_at_5": 0.9394047619047617, "mrr_at_10": 0.9403968253968255, "mrr_at_20": 0.9406349206349207, "mrr_at_50": 0.9407722832722833, "mrr_at_100": 0.9407722832722833, "naucs_at_1_max": 0.39284046952114193, "naucs_at_1_std": 0.06274176337201544, "naucs_at_1_diff1": 0.9321395224756563, "naucs_at_3_max": 0.98132586367881, "naucs_at_3_std": 0.9042950513538718, "naucs_at_3_diff1": 0.98132586367881, "naucs_at_5_max": 0.967320261437913, "naucs_at_5_std": 0.8978758169934754, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 0.9346405228758269, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "tatdqa_test": { "ndcg_at_1": 0.68834, "ndcg_at_3": 0.7834, "ndcg_at_5": 0.80344, "ndcg_at_10": 0.81851, "ndcg_at_20": 0.82469, "ndcg_at_50": 0.82852, "ndcg_at_100": 0.82981, "map_at_1": 0.68834, "map_at_3": 0.76073, "map_at_5": 0.772, "map_at_10": 0.7783, "map_at_20": 0.78002, "map_at_50": 0.78067, "map_at_100": 0.78079, "recall_at_1": 0.68834, "recall_at_3": 0.84872, "recall_at_5": 0.89672, "recall_at_10": 0.94289, "recall_at_20": 0.96719, "recall_at_50": 0.98603, "recall_at_100": 0.99392, "precision_at_1": 0.68834, "precision_at_3": 0.28291, "precision_at_5": 0.17934, "precision_at_10": 0.09429, "precision_at_20": 0.04836, "precision_at_50": 0.01972, "precision_at_100": 0.00994, "mrr_at_1": 0.6865127582017011, "mrr_at_3": 0.7598217901984609, "mrr_at_5": 0.7710307816929933, "mrr_at_10": 0.7773322532739296, "mrr_at_20": 0.7790656715075932, "mrr_at_50": 0.7797137179788176, "mrr_at_100": 0.7798294471430899, "naucs_at_1_max": 0.19289339347399329, "naucs_at_1_std": -0.05373436574034402, "naucs_at_1_diff1": 0.8118815353915732, "naucs_at_3_max": 0.24444248974914928, "naucs_at_3_std": 0.012951438245694854, "naucs_at_3_diff1": 0.7252009696977523, "naucs_at_5_max": 0.27477480629269946, "naucs_at_5_std": 0.10687833140288663, "naucs_at_5_diff1": 0.7019146338300569, "naucs_at_10_max": 0.23474834180340118, "naucs_at_10_std": 0.13375117651376378, "naucs_at_10_diff1": 0.6766342016471449, "naucs_at_20_max": 0.3762582961131715, "naucs_at_20_std": 0.29216428469292166, "naucs_at_20_diff1": 0.6564671335087516, "naucs_at_50_max": 0.4691053847445, "naucs_at_50_std": 0.4359718488363951, "naucs_at_50_diff1": 0.7152604718494652, "naucs_at_100_max": 0.5259975902909616, "naucs_at_100_std": 0.651086653120611, "naucs_at_100_diff1": 0.7663843453532901 }, "shiftproject_test": { "ndcg_at_1": 0.85, "ndcg_at_3": 0.91917, "ndcg_at_5": 0.92347, "ndcg_at_10": 0.92949, "ndcg_at_20": 0.92949, "ndcg_at_50": 0.92949, "ndcg_at_100": 0.92949, "map_at_1": 0.85, "map_at_3": 0.90167, "map_at_5": 0.90417, "map_at_10": 0.90639, "map_at_20": 0.90639, "map_at_50": 0.90639, "map_at_100": 0.90639, "recall_at_1": 0.85, "recall_at_3": 0.97, "recall_at_5": 0.98, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.85, "precision_at_3": 0.32333, "precision_at_5": 0.196, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.85, "mrr_at_3": 0.9016666666666666, "mrr_at_5": 0.9041666666666666, "mrr_at_10": 0.9063888888888889, "mrr_at_20": 0.9063888888888889, "mrr_at_50": 0.9063888888888889, "mrr_at_100": 0.9063888888888889, "naucs_at_1_max": 0.029189716889034732, "naucs_at_1_std": -0.37507321835340074, "naucs_at_1_diff1": 0.7931012040351454, "naucs_at_3_max": 0.5589791472144446, "naucs_at_3_std": 0.09056956115779448, "naucs_at_3_diff1": 0.9564270152505466, "naucs_at_5_max": 0.3384687208216692, "naucs_at_5_std": -0.2987861811391239, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "syntheticDocQA_artificial_intelligence_test": { "ndcg_at_1": 0.98, "ndcg_at_3": 0.99262, "ndcg_at_5": 0.99262, "ndcg_at_10": 0.99262, "ndcg_at_20": 0.99262, "ndcg_at_50": 0.99262, "ndcg_at_100": 0.99262, "map_at_1": 0.98, "map_at_3": 0.99, "map_at_5": 0.99, "map_at_10": 0.99, "map_at_20": 0.99, "map_at_50": 0.99, "map_at_100": 0.99, "recall_at_1": 0.98, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.98, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.98, "mrr_at_3": 0.99, "mrr_at_5": 0.99, "mrr_at_10": 0.99, "mrr_at_20": 0.99, "mrr_at_50": 0.99, "mrr_at_100": 0.99, "naucs_at_1_max": 0.540149393090569, "naucs_at_1_std": 0.3384687208216605, "naucs_at_1_diff1": 0.9346405228758133, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "syntheticDocQA_energy_test": { "ndcg_at_1": 0.95, "ndcg_at_3": 0.96762, "ndcg_at_5": 0.96762, "ndcg_at_10": 0.97118, "ndcg_at_20": 0.97118, "ndcg_at_50": 0.973, "ndcg_at_100": 0.973, "map_at_1": 0.95, "map_at_3": 0.96333, "map_at_5": 0.96333, "map_at_10": 0.965, "map_at_20": 0.965, "map_at_50": 0.96523, "map_at_100": 0.96523, "recall_at_1": 0.95, "recall_at_3": 0.98, "recall_at_5": 0.98, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.95, "precision_at_3": 0.32667, "precision_at_5": 0.196, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.95, "mrr_at_3": 0.9633333333333333, "mrr_at_5": 0.9633333333333333, "mrr_at_10": 0.965, "mrr_at_20": 0.965, "mrr_at_50": 0.9652272727272727, "mrr_at_100": 0.9652272727272727, "naucs_at_1_max": 0.42726423902894384, "naucs_at_1_std": -0.4889822595704953, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 0.6136788048552655, "naucs_at_3_std": -0.6909430438842241, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.6136788048552745, "naucs_at_5_std": -0.690943043884218, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.8692810457516413, "naucs_at_10_std": 0.35807656395891135, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.8692810457516413, "naucs_at_20_std": 0.35807656395891135, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "syntheticDocQA_government_reports_test": { "ndcg_at_1": 0.93, "ndcg_at_3": 0.96524, "ndcg_at_5": 0.96954, "ndcg_at_10": 0.96954, "ndcg_at_20": 0.96954, "ndcg_at_50": 0.96954, "ndcg_at_100": 0.96954, "map_at_1": 0.93, "map_at_3": 0.95667, "map_at_5": 0.95917, "map_at_10": 0.95917, "map_at_20": 0.95917, "map_at_50": 0.95917, "map_at_100": 0.95917, "recall_at_1": 0.93, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.93, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.93, "mrr_at_3": 0.9566666666666667, "mrr_at_5": 0.9591666666666667, "mrr_at_10": 0.9591666666666667, "mrr_at_20": 0.9591666666666667, "mrr_at_50": 0.9591666666666667, "mrr_at_100": 0.9591666666666667, "naucs_at_1_max": 0.6809390422835813, "naucs_at_1_std": 0.5458850206749362, "naucs_at_1_diff1": 0.9229691876750709, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "syntheticDocQA_healthcare_industry_test": { "ndcg_at_1": 0.96, "ndcg_at_3": 0.98393, "ndcg_at_5": 0.98393, "ndcg_at_10": 0.98393, "ndcg_at_20": 0.98393, "ndcg_at_50": 0.98393, "ndcg_at_100": 0.98393, "map_at_1": 0.96, "map_at_3": 0.97833, "map_at_5": 0.97833, "map_at_10": 0.97833, "map_at_20": 0.97833, "map_at_50": 0.97833, "map_at_100": 0.97833, "recall_at_1": 0.96, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.96, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.96, "mrr_at_3": 0.9783333333333333, "mrr_at_5": 0.9783333333333333, "mrr_at_10": 0.9783333333333333, "mrr_at_20": 0.9783333333333333, "mrr_at_50": 0.9783333333333333, "mrr_at_100": 0.9783333333333333, "naucs_at_1_max": 0.7047152194211012, "naucs_at_1_std": 0.32037815126050734, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null } }