{"generatedAt":"2026-09-17T04:26:54.841Z","dataset":{"rows":1554,"testRows":1195,"devRows":359,"passRate":0.494,"tasks":{"ap_grade_school":{"n":321,"pass":0.64,"family":"aider","grader":"rubric","split":"test"},"ap_pov":{"n":317,"pass":0.68,"family":"aider","grader":"rubric","split":"test"},"ap_dot_dsl":{"n":295,"pass":0.27,"family":"aider","grader":"rubric","split":"test"},"ap_bottle_song":{"n":120,"pass":0.42,"family":"aider","grader":"rubric","split":"test"},"ap_beer_song":{"n":85,"pass":0.48,"family":"aider","grader":"rubric","split":"test"},"ap_connect":{"n":57,"pass":0.25,"family":"aider","grader":"rubric","split":"test"},"ap_hangman":{"n":34,"pass":0.03,"family":"aider","grader":"rubric","split":"dev"},"ap_pig_latin":{"n":25,"pass":0.72,"family":"aider","grader":"rubric","split":"dev"},"drop_off_by_one":{"n":25,"pass":1,"family":"toolz","grader":"rubric","split":"dev"},"ap_affine_cipher":{"n":22,"pass":0.55,"family":"aider","grader":"rubric","split":"dev"},"ap_paasio":{"n":21,"pass":0.1,"family":"aider","grader":"rubric","split":"dev"},"ap_book_store":{"n":19,"pass":0.11,"family":"aider","grader":"rubric","split":"dev"},"take_off_by_one":{"n":17,"pass":1,"family":"toolz","grader":"rubric","split":"dev"},"tail_slice":{"n":16,"pass":1,"family":"toolz","grader":"rubric","split":"dev"},"ap_go_counting":{"n":15,"pass":0,"family":"aider","grader":"rubric","split":"dev"},"ap_forth":{"n":15,"pass":0,"family":"aider","grader":"rubric","split":"dev"},"take_nth_offset":{"n":15,"pass":1,"family":"toolz","grader":"rubric","split":"dev"},"ap_list_ops":{"n":14,"pass":0.07,"family":"aider","grader":"rubric","split":"dev"},"ap_grep":{"n":13,"pass":0.08,"family":"aider","grader":"rubric","split":"dev"},"ap_food_chain":{"n":13,"pass":0.54,"family":"aider","grader":"rubric","split":"dev"},"ap_dominoes":{"n":12,"pass":0.67,"family":"aider","grader":"rubric","split":"dev"},"ap_poker":{"n":10,"pass":0.3,"family":"aider","grader":"rubric","split":"dev"},"ap_phone_number":{"n":9,"pass":0,"family":"aider","grader":"rubric","split":"dev"},"ap_bowling":{"n":8,"pass":0,"family":"aider","grader":"rubric","split":"dev"},"sliding_window_drops_first":{"n":6,"pass":0.83,"family":"toolz","grader":"rubric","split":"dev"},"t01":{"n":5,"pass":0.4,"family":"sensitivity","grader":"rubric","split":"dev"},"t04":{"n":4,"pass":0.5,"family":"sensitivity","grader":"rubric","split":"dev"},"t03":{"n":4,"pass":0.5,"family":"sensitivity","grader":"rubric","split":"dev"},"t08":{"n":4,"pass":0.5,"family":"sensitivity","grader":"rubric","split":"dev"},"t02":{"n":4,"pass":0.5,"family":"sensitivity","grader":"rubric","split":"dev"},"t10":{"n":4,"pass":0.5,"family":"sensitivity","grader":"rubric","split":"dev"},"t07":{"n":4,"pass":0.5,"family":"sensitivity","grader":"rubric","split":"dev"},"t06":{"n":4,"pass":0.5,"family":"sensitivity","grader":"rubric","split":"dev"},"t05":{"n":4,"pass":0.5,"family":"sensitivity","grader":"rubric","split":"dev"},"t11":{"n":4,"pass":0.5,"family":"sensitivity","grader":"rubric","split":"dev"},"t09":{"n":4,"pass":0.5,"family":"sensitivity","grader":"rubric","split":"dev"},"t12":{"n":4,"pass":0.5,"family":"sensitivity","grader":"rubric","split":"dev"},"add":{"n":1,"pass":1,"family":"aider","grader":"cmd","split":"dev"}},"bestOfNTasks":12,"extractedWith":"data/extract_v3.py"},"tiers":{"frontier":{"key":"frontier","kind":"openrouter","model":"anthropic/claude-sonnet-5","label":"Frontier · Claude Sonnet 5","price":{"in":2,"out":10},"run":{"n":1195,"model":"anthropic/claude-sonnet-5","firstAt":"2026-09-17T04:13:31.544Z","lastAt":"2026-09-17T04:26:45.417Z","truncated":18,"parseFailed":0,"costSource":"billed"}},"large":{"key":"large","kind":"workers-ai","model":"@cf/meta/llama-3.3-70b-instruct-fp8-fast","label":"Large open · Llama 3.3 70B","price":null,"run":{"n":1554,"model":"@cf/meta/llama-3.3-70b-instruct-fp8-fast","firstAt":"2026-09-17T04:13:23.150Z","lastAt":"2026-09-17T04:21:23.551Z","truncated":45,"parseFailed":0,"costSource":"neurons"}},"small":{"key":"small","kind":"workers-ai","model":"@cf/qwen/qwen3-30b-a3b-fp8","label":"Small open · Qwen3 30B-A3B","price":null,"run":{"n":1554,"model":"@cf/qwen/qwen3-30b-a3b-fp8","firstAt":"2026-09-17T04:13:15.226Z","lastAt":"2026-09-17T04:15:45.121Z","truncated":45,"parseFailed":0,"costSource":"neurons"}}},"cards":{"frontier":{"test":{"n":1195,"scored":1195,"parseFailures":0,"baseRate":0.5104602510460251,"accuracy":0.5790794979079498,"brier":0.3145393305439331,"logLoss":0.9091258986464389,"auc":0.6295053944234272,"ece":0.27754811715481154,"reliability":[{"lo":0,"hi":0.1,"n":77,"conf":0.045324675324675275,"acc":0.16883116883116883},{"lo":0.1,"hi":0.2,"n":102,"conf":0.12313725490196076,"acc":0.3431372549019608},{"lo":0.2,"hi":0.3,"n":11,"conf":0.21818181818181817,"acc":0.6363636363636364},{"lo":0.3,"hi":0.4,"n":19,"conf":0.3368421052631578,"acc":0.47368421052631576},{"lo":0.4,"hi":0.5,"n":1,"conf":0.4,"acc":0},{"lo":0.5,"hi":0.6,"n":2,"conf":0.55,"acc":0.5},{"lo":0.6,"hi":0.7,"n":55,"conf":0.6489090909090908,"acc":0.38181818181818183},{"lo":0.7,"hi":0.8,"n":116,"conf":0.7314655172413793,"acc":0.5258620689655172},{"lo":0.8,"hi":0.9,"n":442,"conf":0.8569909502262449,"acc":0.5135746606334841},{"lo":0.9,"hi":1,"n":370,"conf":0.9126756756756743,"acc":0.6378378378378379}],"latencyP50Ms":2202,"latencyP95Ms":2667.6,"usdPer1k":4.721464435146439,"brierCi":{"lo":0.295299753138075,"hi":0.3344168786610881,"iters":400},"byTask":{"ap_grade_school":{"n":321,"scored":321,"parseFailures":0,"baseRate":0.6448598130841121,"accuracy":0.6853582554517134,"brier":0.24168255451713389,"logLoss":0.710459502830096,"auc":0.671116196287821,"ece":0.17623052959501473,"latencyP50Ms":2201,"latencyP95Ms":2584,"usdPer1k":3.4103987538940816},"ap_dot_dsl":{"n":295,"scored":295,"parseFailures":0,"baseRate":0.2711864406779661,"accuracy":0.4033898305084746,"brier":0.44029084745762725,"logLoss":1.2306660694846836,"auc":0.8529651162790698,"ece":0.498847457627119,"latencyP50Ms":2193,"latencyP95Ms":2631.7000000000003,"usdPer1k":5.396176271186438},"ap_connect":{"n":57,"scored":57,"parseFailures":0,"baseRate":0.24561403508771928,"accuracy":0.631578947368421,"brier":0.24656315789473685,"logLoss":0.7083010835185292,"auc":0.7358803986710963,"ece":0.23596491228070177,"latencyP50Ms":2218,"latencyP95Ms":2627.5999999999995,"usdPer1k":6.699438596491229},"ap_pov":{"n":317,"scored":317,"parseFailures":0,"baseRate":0.6845425867507886,"accuracy":0.6309148264984227,"brier":0.262298738170347,"logLoss":0.7648148225924696,"auc":0.5987557603686636,"ece":0.1826813880126175,"latencyP50Ms":2207,"latencyP95Ms":2667.4,"usdPer1k":4.1663406940063075},"ap_beer_song":{"n":85,"scored":85,"parseFailures":0,"baseRate":0.4823529411764706,"accuracy":0.5764705882352941,"brier":0.336330588235294,"logLoss":0.9952843433545588,"auc":0.7281042128603105,"ece":0.3325882352941175,"latencyP50Ms":2215,"latencyP95Ms":2763.5999999999995,"usdPer1k":8.143552941176472},"ap_bottle_song":{"n":120,"scored":120,"parseFailures":0,"baseRate":0.425,"accuracy":0.5666666666666667,"brier":0.35514749999999984,"logLoss":1.0656902355712392,"auc":0.7014776925262859,"ece":0.36341666666666717,"latencyP50Ms":2203.5,"latencyP95Ms":2850.5499999999997,"usdPer1k":4.672833333333336}}},"all":{"n":1195,"scored":1195,"parseFailures":0,"baseRate":0.5104602510460251,"accuracy":0.5790794979079498,"brier":0.3145393305439331,"logLoss":0.9091258986464389,"auc":0.6295053944234272,"ece":0.27754811715481154,"reliability":[{"lo":0,"hi":0.1,"n":77,"conf":0.045324675324675275,"acc":0.16883116883116883},{"lo":0.1,"hi":0.2,"n":102,"conf":0.12313725490196076,"acc":0.3431372549019608},{"lo":0.2,"hi":0.3,"n":11,"conf":0.21818181818181817,"acc":0.6363636363636364},{"lo":0.3,"hi":0.4,"n":19,"conf":0.3368421052631578,"acc":0.47368421052631576},{"lo":0.4,"hi":0.5,"n":1,"conf":0.4,"acc":0},{"lo":0.5,"hi":0.6,"n":2,"conf":0.55,"acc":0.5},{"lo":0.6,"hi":0.7,"n":55,"conf":0.6489090909090908,"acc":0.38181818181818183},{"lo":0.7,"hi":0.8,"n":116,"conf":0.7314655172413793,"acc":0.5258620689655172},{"lo":0.8,"hi":0.9,"n":442,"conf":0.8569909502262449,"acc":0.5135746606334841},{"lo":0.9,"hi":1,"n":370,"conf":0.9126756756756743,"acc":0.6378378378378379}],"latencyP50Ms":2202,"latencyP95Ms":2667.6,"usdPer1k":4.721464435146439,"brierCi":{"lo":0.295299753138075,"hi":0.3344168786610881,"iters":400},"byTask":{"ap_grade_school":{"n":321,"scored":321,"parseFailures":0,"baseRate":0.6448598130841121,"accuracy":0.6853582554517134,"brier":0.24168255451713389,"logLoss":0.710459502830096,"auc":0.671116196287821,"ece":0.17623052959501473,"latencyP50Ms":2201,"latencyP95Ms":2584,"usdPer1k":3.4103987538940816},"ap_dot_dsl":{"n":295,"scored":295,"parseFailures":0,"baseRate":0.2711864406779661,"accuracy":0.4033898305084746,"brier":0.44029084745762725,"logLoss":1.2306660694846836,"auc":0.8529651162790698,"ece":0.498847457627119,"latencyP50Ms":2193,"latencyP95Ms":2631.7000000000003,"usdPer1k":5.396176271186438},"ap_connect":{"n":57,"scored":57,"parseFailures":0,"baseRate":0.24561403508771928,"accuracy":0.631578947368421,"brier":0.24656315789473685,"logLoss":0.7083010835185292,"auc":0.7358803986710963,"ece":0.23596491228070177,"latencyP50Ms":2218,"latencyP95Ms":2627.5999999999995,"usdPer1k":6.699438596491229},"ap_pov":{"n":317,"scored":317,"parseFailures":0,"baseRate":0.6845425867507886,"accuracy":0.6309148264984227,"brier":0.262298738170347,"logLoss":0.7648148225924696,"auc":0.5987557603686636,"ece":0.1826813880126175,"latencyP50Ms":2207,"latencyP95Ms":2667.4,"usdPer1k":4.1663406940063075},"ap_beer_song":{"n":85,"scored":85,"parseFailures":0,"baseRate":0.4823529411764706,"accuracy":0.5764705882352941,"brier":0.336330588235294,"logLoss":0.9952843433545588,"auc":0.7281042128603105,"ece":0.3325882352941175,"latencyP50Ms":2215,"latencyP95Ms":2763.5999999999995,"usdPer1k":8.143552941176472},"ap_bottle_song":{"n":120,"scored":120,"parseFailures":0,"baseRate":0.425,"accuracy":0.5666666666666667,"brier":0.35514749999999984,"logLoss":1.0656902355712392,"auc":0.7014776925262859,"ece":0.36341666666666717,"latencyP50Ms":2203.5,"latencyP95Ms":2850.5499999999997,"usdPer1k":4.672833333333336}}}},"large":{"test":{"n":1195,"scored":1195,"parseFailures":0,"baseRate":0.5104602510460251,"accuracy":0.5138075313807532,"brier":0.4476395815899536,"logLoss":2.5844875761880033,"auc":0.4534832562701415,"ece":0.4485941422594194,"reliability":[{"lo":0,"hi":0.1,"n":45,"conf":0,"acc":0.08888888888888889},{"lo":0.1,"hi":0.2,"n":0,"conf":null,"acc":null},{"lo":0.2,"hi":0.3,"n":69,"conf":0.19999999999999976,"acc":0.7391304347826086},{"lo":0.3,"hi":0.4,"n":0,"conf":null,"acc":null},{"lo":0.4,"hi":0.5,"n":0,"conf":null,"acc":null},{"lo":0.5,"hi":0.6,"n":0,"conf":null,"acc":null},{"lo":0.6,"hi":0.7,"n":10,"conf":0.5999999999999999,"acc":0.4},{"lo":0.7,"hi":0.8,"n":74,"conf":0.7000000000000008,"acc":0.8918918918918919},{"lo":0.8,"hi":0.9,"n":16,"conf":0.8000000000000002,"acc":0.8125},{"lo":0.9,"hi":1,"n":981,"conf":0.9688786952089767,"acc":0.48114169215086644}],"latencyP50Ms":741,"latencyP95Ms":2800.1999999999994,"usdPer1k":0.33032991093591624,"brierCi":{"lo":0.41969301882844656,"hi":0.46948953556484907,"iters":400},"byTask":{"ap_pov":{"n":317,"scored":317,"parseFailures":0,"baseRate":0.6845425867507886,"accuracy":0.555205047318612,"brier":0.38147003154574194,"logLoss":1.3869590223869999,"auc":0.30172811059907834,"ece":0.40757097791798025,"latencyP50Ms":693,"latencyP95Ms":3316.5999999999976,"usdPer1k":0.2853759281748101},"ap_grade_school":{"n":321,"scored":321,"parseFailures":0,"baseRate":0.6448598130841121,"accuracy":0.6199376947040498,"brier":0.3551744548286615,"logLoss":2.994093108992974,"auc":0.263348588863463,"ece":0.34922118380062367,"latencyP50Ms":654,"latencyP95Ms":3111,"usdPer1k":0.2194579327425853},"ap_connect":{"n":57,"scored":57,"parseFailures":0,"baseRate":0.24561403508771928,"accuracy":0.24561403508771928,"brier":0.677682456140351,"logLoss":2.3163110760650003,"auc":0.39368770764119604,"ece":0.6984210526315795,"latencyP50Ms":899,"latencyP95Ms":3042.5999999999995,"usdPer1k":0.5442892743830096},"ap_dot_dsl":{"n":295,"scored":295,"parseFailures":0,"baseRate":0.2711864406779661,"accuracy":0.45084745762711864,"brier":0.5114559322033896,"logLoss":2.204983754273828,"auc":0.9429360465116279,"ece":0.5384745762711862,"latencyP50Ms":789,"latencyP95Ms":2438.3000000000006,"usdPer1k":0.3742473503177449},"ap_beer_song":{"n":85,"scored":85,"parseFailures":0,"baseRate":0.4823529411764706,"accuracy":0.47058823529411764,"brier":0.5145611764705886,"logLoss":2.6066336694269268,"auc":0.47560975609756095,"ece":0.5156470588235291,"latencyP50Ms":1037,"latencyP95Ms":2151.6,"usdPer1k":0.6496296768188478},"ap_bottle_song":{"n":120,"scored":120,"parseFailures":0,"baseRate":0.425,"accuracy":0.43333333333333335,"brier":0.5562266666666671,"logLoss":5.6969079559454325,"auc":0.477124183006536,"ece":0.5573333333333325,"latencyP50Ms":718,"latencyP95Ms":1979,"usdPer1k":0.3099008201122283}}},"all":{"n":1554,"scored":1554,"parseFailures":0,"baseRate":0.4942084942084942,"accuracy":0.49356499356499356,"brier":0.470828635778633,"logLoss":2.7993353807101746,"auc":0.4450250808418151,"ece":0.47099742599743505,"reliability":[{"lo":0,"hi":0.1,"n":50,"conf":0,"acc":0.18},{"lo":0.1,"hi":0.2,"n":0,"conf":null,"acc":null},{"lo":0.2,"hi":0.3,"n":71,"conf":0.19999999999999973,"acc":0.7323943661971831},{"lo":0.3,"hi":0.4,"n":0,"conf":null,"acc":null},{"lo":0.4,"hi":0.5,"n":0,"conf":null,"acc":null},{"lo":0.5,"hi":0.6,"n":0,"conf":null,"acc":null},{"lo":0.6,"hi":0.7,"n":12,"conf":0.5999999999999999,"acc":0.3333333333333333},{"lo":0.7,"hi":0.8,"n":74,"conf":0.7000000000000008,"acc":0.8918918918918919},{"lo":0.8,"hi":0.9,"n":24,"conf":0.8000000000000003,"acc":0.8333333333333334},{"lo":0.9,"hi":1,"n":1323,"conf":0.970468631897214,"acc":0.46636432350718066}],"latencyP50Ms":749,"latencyP95Ms":2744.999999999998,"usdPer1k":0.33748924260648844,"brierCi":{"lo":0.44729543597168303,"hi":0.4935337982625468,"iters":400},"byTask":{"ap_pov":{"n":317,"scored":317,"parseFailures":0,"baseRate":0.6845425867507886,"accuracy":0.555205047318612,"brier":0.38147003154574194,"logLoss":1.3869590223869999,"auc":0.30172811059907834,"ece":0.40757097791798025,"latencyP50Ms":693,"latencyP95Ms":3316.5999999999976,"usdPer1k":0.2853759281748101},"ap_grade_school":{"n":321,"scored":321,"parseFailures":0,"baseRate":0.6448598130841121,"accuracy":0.6199376947040498,"brier":0.3551744548286615,"logLoss":2.994093108992974,"auc":0.263348588863463,"ece":0.34922118380062367,"latencyP50Ms":654,"latencyP95Ms":3111,"usdPer1k":0.2194579327425853},"tail_slice":{"n":16,"scored":16,"parseFailures":0,"baseRate":1,"accuracy":1,"brier":0.007524999999999996,"logLoss":0.08153297070674509,"auc":null,"ece":0.07749999999999979,"latencyP50Ms":547.5,"latencyP95Ms":891.25,"usdPer1k":0.14850885784626006},"ap_pig_latin":{"n":25,"scored":25,"parseFailures":0,"baseRate":0.72,"accuracy":0.72,"brier":0.26614800000000005,"logLoss":1.1200485278127146,"auc":0.4365079365079365,"ece":0.24919999999999942,"latencyP50Ms":824,"latencyP95Ms":1847.7999999999997,"usdPer1k":0.4685527016448976},"ap_book_store":{"n":19,"scored":19,"parseFailures":0,"baseRate":0.10526315789473684,"accuracy":0.10526315789473684,"brier":0.8689,"logLoss":3.954229524587913,"auc":0.3088235294117647,"ece":0.8784210526315789,"latencyP50Ms":982,"latencyP95Ms":1680.3,"usdPer1k":0.5710762591111032},"ap_go_counting":{"n":15,"scored":15,"parseFailures":0,"baseRate":0,"accuracy":0,"brier":0.9222,"logLoss":3.486001765661361,"auc":null,"ece":0.9599999999999999,"latencyP50Ms":1042,"latencyP95Ms":2273.1,"usdPer1k":0.6054232589721679},"ap_bowling":{"n":8,"scored":8,"parseFailures":0,"baseRate":0,"accuracy":0,"brier":0.879375,"logLoss":2.822445478414004,"auc":null,"ece":0.9375000000000001,"latencyP50Ms":1226.5,"latencyP95Ms":2474.6,"usdPer1k":0.8671842241287232},"ap_paasio":{"n":21,"scored":21,"parseFailures":0,"baseRate":0.09523809523809523,"accuracy":0.09523809523809523,"brier":0.8314857142857144,"logLoss":3.4573634439458143,"auc":0.47368421052631576,"ece":0.8609523809523806,"latencyP50Ms":1080,"latencyP95Ms":1974,"usdPer1k":0.49296062287830167},"ap_connect":{"n":57,"scored":57,"parseFailures":0,"baseRate":0.24561403508771928,"accuracy":0.24561403508771928,"brier":0.677682456140351,"logLoss":2.3163110760650003,"auc":0.39368770764119604,"ece":0.6984210526315795,"latencyP50Ms":899,"latencyP95Ms":3042.5999999999995,"usdPer1k":0.5442892743830096},"ap_dot_dsl":{"n":295,"scored":295,"parseFailures":0,"baseRate":0.2711864406779661,"accuracy":0.45084745762711864,"brier":0.5114559322033896,"logLoss":2.204983754273828,"auc":0.9429360465116279,"ece":0.5384745762711862,"latencyP50Ms":789,"latencyP95Ms":2438.3000000000006,"usdPer1k":0.3742473503177449},"drop_off_by_one":{"n":25,"scored":25,"parseFailures":0,"baseRate":1,"accuracy":0.84,"brier":0.162048,"logLoss":2.2363781136156096,"auc":null,"ece":0.1848000000000002,"latencyP50Ms":569,"latencyP95Ms":2128.199999999999,"usdPer1k":0.12692384979248045},"t04":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0.5,"brier":0.5025,"logLoss":6.934095657882341,"auc":0.25,"ece":0.475,"latencyP50Ms":525,"latencyP95Ms":1232.9999999999995,"usdPer1k":0.10786232829093934},"ap_beer_song":{"n":85,"scored":85,"parseFailures":0,"baseRate":0.4823529411764706,"accuracy":0.47058823529411764,"brier":0.5145611764705886,"logLoss":2.6066336694269268,"auc":0.47560975609756095,"ece":0.5156470588235291,"latencyP50Ms":1037,"latencyP95Ms":2151.6,"usdPer1k":0.6496296768188478},"ap_bottle_song":{"n":120,"scored":120,"parseFailures":0,"baseRate":0.425,"accuracy":0.43333333333333335,"brier":0.5562266666666671,"logLoss":5.6969079559454325,"auc":0.477124183006536,"ece":0.5573333333333325,"latencyP50Ms":718,"latencyP95Ms":1979,"usdPer1k":0.3099008201122283},"take_off_by_one":{"n":17,"scored":17,"parseFailures":0,"baseRate":1,"accuracy":1,"brier":0.000052941176470588326,"logLoss":0.005321236628559605,"auc":null,"ece":0.0052941176470587825,"latencyP50Ms":567,"latencyP95Ms":2172.399999999998,"usdPer1k":0.10647326172099396},"ap_hangman":{"n":34,"scored":34,"parseFailures":0,"baseRate":0.029411764705882353,"accuracy":0.029411764705882353,"brier":0.9352588235294119,"logLoss":7.62070513291625,"auc":0.12121212121212122,"ece":0.9505882352941174,"latencyP50Ms":831.5,"latencyP95Ms":4688,"usdPer1k":0.38647046094782217},"t03":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0.5,"brier":0.52,"logLoss":7.019327054624863,"auc":0,"ece":0.6,"latencyP50Ms":1006,"latencyP95Ms":2083.0999999999995,"usdPer1k":0.11240917158126831},"ap_phone_number":{"n":9,"scored":9,"parseFailures":0,"baseRate":0,"accuracy":0,"brier":0.9628555555555555,"logLoss":4.247517316558291,"auc":null,"ece":0.9811111111111112,"latencyP50Ms":709,"latencyP95Ms":15424.799999999992,"usdPer1k":0.5656588083902995},"t08":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0.5,"brier":0.50005,"logLoss":6.9127804468945095,"auc":0,"ece":0.495,"latencyP50Ms":573.5,"latencyP95Ms":601.45,"usdPer1k":0.12641635990142824},"t02":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0.5,"brier":0.52,"logLoss":7.019327054624863,"auc":0,"ece":0.6,"latencyP50Ms":2169.5,"latencyP95Ms":3339.3999999999996,"usdPer1k":0.10859569239616393},"ap_forth":{"n":15,"scored":15,"parseFailures":0,"baseRate":0,"accuracy":0,"brier":0.96458,"logLoss":4.283282603501271,"auc":null,"ece":0.982,"latencyP50Ms":1189,"latencyP95Ms":3455.799999999997,"usdPer1k":0.6611978070576985},"t01":{"n":5,"scored":5,"parseFailures":0,"baseRate":0.4,"accuracy":0.2,"brier":0.808,"logLoss":11.097037156617008,"auc":0,"ece":0.84,"latencyP50Ms":523,"latencyP95Ms":679.1999999999999,"usdPer1k":0.10472354869842528},"t10":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0.5,"brier":0.495025,"logLoss":4.605170685981152,"auc":0.75,"ece":0.49750000000000005,"latencyP50Ms":650.5,"latencyP95Ms":917.6999999999999,"usdPer1k":0.13976352787017823},"t07":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0.5,"brier":0.5,"logLoss":6.907755778968009,"auc":0.5,"ece":0.5,"latencyP50Ms":549.5,"latencyP95Ms":652.3499999999999,"usdPer1k":0.1150492660999298},"ap_list_ops":{"n":14,"scored":14,"parseFailures":0,"baseRate":0.07142857142857142,"accuracy":0.07142857142857142,"brier":0.9129428571428573,"logLoss":5.592710249828107,"auc":0.4230769230769231,"ece":0.9200000000000003,"latencyP50Ms":861,"latencyP95Ms":1319.8999999999999,"usdPer1k":0.31733116190774097},"ap_poker":{"n":10,"scored":10,"parseFailures":0,"baseRate":0.3,"accuracy":0.3,"brier":0.6480199999999999,"logLoss":2.434288162290878,"auc":0.35714285714285715,"ece":0.6579999999999999,"latencyP50Ms":830,"latencyP95Ms":1649.1999999999994,"usdPer1k":0.510154803276062},"ap_grep":{"n":13,"scored":13,"parseFailures":0,"baseRate":0.07692307692307693,"accuracy":0.07692307692307693,"brier":0.8571461538461538,"logLoss":3.2644486328286026,"auc":0.3333333333333333,"ece":0.8853846153846152,"latencyP50Ms":772,"latencyP95Ms":2455.399999999999,"usdPer1k":0.3501534976959229},"ap_affine_cipher":{"n":22,"scored":22,"parseFailures":0,"baseRate":0.5454545454545454,"accuracy":0.5454545454545454,"brier":0.44554999999999995,"logLoss":2.09828438883045,"auc":0.5416666666666666,"ece":0.44499999999999973,"latencyP50Ms":798,"latencyP95Ms":1718.5,"usdPer1k":0.5549191360473631},"t06":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0.5,"brier":0.502525,"logLoss":6.936607991845591,"auc":0,"ece":0.47250000000000003,"latencyP50Ms":1374,"latencyP95Ms":2874.2499999999995,"usdPer1k":0.11842272734642027},"ap_dominoes":{"n":12,"scored":12,"parseFailures":0,"baseRate":0.6666666666666666,"accuracy":0.5833333333333334,"brier":0.31854166666666667,"logLoss":1.8909797341720511,"auc":0.4375,"ece":0.3291666666666666,"latencyP50Ms":960.5,"latencyP95Ms":22496.799999999967,"usdPer1k":0.3765657812754313},"ap_food_chain":{"n":13,"scored":13,"parseFailures":0,"baseRate":0.5384615384615384,"accuracy":0.6153846153846154,"brier":0.38009230769230773,"logLoss":1.7937959101714749,"auc":0.5833333333333334,"ece":0.39076923076923087,"latencyP50Ms":1187,"latencyP95Ms":3674.1999999999953,"usdPer1k":0.4517409010667067},"take_nth_offset":{"n":15,"scored":15,"parseFailures":0,"baseRate":1,"accuracy":1,"brier":0.000673333333333333,"logLoss":0.007694923434521874,"auc":null,"ece":0.007333333333333303,"latencyP50Ms":549,"latencyP95Ms":1289.4999999999986,"usdPer1k":0.11997744935353599},"t05":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0.5,"brier":0.52,"logLoss":7.019327054624863,"auc":0,"ece":0.6,"latencyP50Ms":526.5,"latencyP95Ms":730.8999999999999,"usdPer1k":0.11006241273880005},"t11":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0.5,"brier":0.5049999999999999,"logLoss":6.9604355367966715,"auc":0,"ece":0.44999999999999996,"latencyP50Ms":570,"latencyP95Ms":915.5499999999998,"usdPer1k":0.12040280151367187},"sliding_window_drops_first":{"n":6,"scored":6,"parseFailures":0,"baseRate":0.8333333333333334,"accuracy":0.8333333333333334,"brier":0.14333333333333334,"logLoss":0.4715646118805296,"auc":0.5,"ece":0.06666666666666665,"latencyP50Ms":644.5,"latencyP95Ms":1150.5,"usdPer1k":0.15694862206776936},"t09":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0.5,"brier":0.5,"logLoss":6.907755778968009,"auc":0.5,"ece":0.5,"latencyP50Ms":729,"latencyP95Ms":987.9999999999999,"usdPer1k":0.11900941967964172},"t12":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0.5,"brier":0.5000249999999999,"logLoss":6.910268112931259,"auc":0.25,"ece":0.49750000000000005,"latencyP50Ms":602,"latencyP95Ms":1205.1999999999998,"usdPer1k":0.12766307520866393},"add":{"n":1,"scored":1,"parseFailures":0,"baseRate":1,"accuracy":1,"brier":0,"logLoss":0.000001000000500029089,"auc":null,"ece":0,"latencyP50Ms":519,"latencyP95Ms":519,"usdPer1k":0.0871815481185913}}}},"small":{"test":{"n":1195,"scored":1195,"parseFailures":0,"baseRate":0.5104602510460251,"accuracy":0.502092050209205,"brier":0.43181380753137927,"logLoss":3.3357140017663647,"auc":0.48525430853299706,"ece":0.42255230125522925,"reliability":[{"lo":0,"hi":0.1,"n":0,"conf":null,"acc":null},{"lo":0.1,"hi":0.2,"n":430,"conf":0.1000000000000008,"acc":0.5348837209302325},{"lo":0.2,"hi":0.3,"n":221,"conf":0.2000000000000004,"acc":0.497737556561086},{"lo":0.3,"hi":0.4,"n":27,"conf":0.29999999999999993,"acc":0.14814814814814814},{"lo":0.4,"hi":0.5,"n":0,"conf":null,"acc":null},{"lo":0.5,"hi":0.6,"n":0,"conf":null,"acc":null},{"lo":0.6,"hi":0.7,"n":0,"conf":null,"acc":null},{"lo":0.7,"hi":0.8,"n":0,"conf":null,"acc":null},{"lo":0.8,"hi":0.9,"n":0,"conf":null,"acc":null},{"lo":0.9,"hi":1,"n":517,"conf":0.994294003868471,"acc":0.5145067698259188}],"latencyP50Ms":422,"latencyP95Ms":619.3,"usdPer1k":0.05746076132453139,"brierCi":{"lo":0.40804503138075177,"hi":0.459465951882843,"iters":400},"byTask":{"ap_pov":{"n":317,"scored":317,"parseFailures":0,"baseRate":0.6845425867507886,"accuracy":0.2996845425867508,"brier":0.5852602523659309,"logLoss":3.3017116324898,"auc":0.27486175115207373,"ece":0.6121451104100949,"latencyP50Ms":411,"latencyP95Ms":597.2,"usdPer1k":0.04912645847188187},"ap_grade_school":{"n":321,"scored":321,"parseFailures":0,"baseRate":0.6448598130841121,"accuracy":0.5700934579439252,"brier":0.32705607476635584,"logLoss":1.558360654991591,"auc":0.698512585812357,"ece":0.33489096573208743,"latencyP50Ms":417,"latencyP95Ms":619,"usdPer1k":0.037658668955167125},"ap_connect":{"n":57,"scored":57,"parseFailures":0,"baseRate":0.24561403508771928,"accuracy":0.7368421052631579,"brier":0.27192982456140335,"logLoss":1.486808856585606,"auc":0.7259136212624585,"ece":0.3526315789473683,"latencyP50Ms":439,"latencyP95Ms":671.3999999999997,"usdPer1k":0.09422221757654556},"ap_dot_dsl":{"n":295,"scored":295,"parseFailures":0,"baseRate":0.2711864406779661,"accuracy":0.576271186440678,"brier":0.4266525423728821,"logLoss":5.668924729699635,"auc":0.6872093023255814,"ece":0.4344067796610169,"latencyP50Ms":438,"latencyP95Ms":623.6,"usdPer1k":0.06450401623774382},"ap_beer_song":{"n":85,"scored":85,"parseFailures":0,"baseRate":0.4823529411764706,"accuracy":0.5058823529411764,"brier":0.4483529411764705,"logLoss":4.004017734331005,"auc":0.4772727272727273,"ece":0.43176470588235294,"latencyP50Ms":416,"latencyP95Ms":576.2,"usdPer1k":0.1199735921298757},"ap_bottle_song":{"n":120,"scored":120,"parseFailures":0,"baseRate":0.425,"accuracy":0.5583333333333333,"brier":0.3836041666666666,"logLoss":2.848995557119637,"auc":0.4862176754759875,"ece":0.34375,"latencyP50Ms":428.5,"latencyP95Ms":611.2,"usdPer1k":0.053391526579856866}}},"all":{"n":1554,"scored":1554,"parseFailures":0,"baseRate":0.4942084942084942,"accuracy":0.5,"brier":0.4396315958815924,"logLoss":3.562741815401994,"auc":0.4780716576547922,"ece":0.4335585585585586,"reliability":[{"lo":0,"hi":0.1,"n":2,"conf":0,"acc":0.5},{"lo":0.1,"hi":0.2,"n":512,"conf":0.1000000000000009,"acc":0.541015625},{"lo":0.2,"hi":0.3,"n":274,"conf":0.20000000000000087,"acc":0.48175182481751827},{"lo":0.3,"hi":0.4,"n":49,"conf":0.30000000000000027,"acc":0.08163265306122448},{"lo":0.4,"hi":0.5,"n":0,"conf":null,"acc":null},{"lo":0.5,"hi":0.6,"n":0,"conf":null,"acc":null},{"lo":0.6,"hi":0.7,"n":0,"conf":null,"acc":null},{"lo":0.7,"hi":0.8,"n":0,"conf":null,"acc":null},{"lo":0.8,"hi":0.9,"n":0,"conf":null,"acc":null},{"lo":0.9,"hi":1,"n":717,"conf":0.9944909344490945,"acc":0.49372384937238495}],"latencyP50Ms":423,"latencyP95Ms":614.3499999999999,"usdPer1k":0.05871455572792917,"brierCi":{"lo":0.41865954794079413,"hi":0.46148045366794893,"iters":400},"byTask":{"ap_pov":{"n":317,"scored":317,"parseFailures":0,"baseRate":0.6845425867507886,"accuracy":0.2996845425867508,"brier":0.5852602523659309,"logLoss":3.3017116324898,"auc":0.27486175115207373,"ece":0.6121451104100949,"latencyP50Ms":411,"latencyP95Ms":597.2,"usdPer1k":0.04912645847188187},"ap_grade_school":{"n":321,"scored":321,"parseFailures":0,"baseRate":0.6448598130841121,"accuracy":0.5700934579439252,"brier":0.32705607476635584,"logLoss":1.558360654991591,"auc":0.698512585812357,"ece":0.33489096573208743,"latencyP50Ms":417,"latencyP95Ms":619,"usdPer1k":0.037658668955167125},"ap_pig_latin":{"n":25,"scored":25,"parseFailures":0,"baseRate":0.72,"accuracy":0.32,"brier":0.45730000000000026,"logLoss":1.1973581052310283,"auc":0.373015873015873,"ece":0.5339999999999999,"latencyP50Ms":417,"latencyP95Ms":575.3999999999999,"usdPer1k":0.08094642791748044},"ap_bowling":{"n":8,"scored":8,"parseFailures":0,"baseRate":0,"accuracy":1,"brier":0.053750000000000006,"logLoss":0.2457406092553273,"auc":null,"ece":0.2125,"latencyP50Ms":430.5,"latencyP95Ms":718.3999999999999,"usdPer1k":0.1532257042527199},"ap_go_counting":{"n":15,"scored":15,"parseFailures":0,"baseRate":0,"accuracy":0.3333333333333333,"brier":0.6664999999999999,"logLoss":7.131767772622829,"auc":null,"ece":0.7300000000000001,"latencyP50Ms":470,"latencyP95Ms":697.1999999999998,"usdPer1k":0.10483996540705362},"tail_slice":{"n":16,"scored":16,"parseFailures":0,"baseRate":1,"accuracy":1,"brier":0.00015625000000000027,"logLoss":0.0032067683996906907,"auc":null,"ece":0.0031250000000000444,"latencyP50Ms":389,"latencyP95Ms":710.75,"usdPer1k":0.025803593277931216},"ap_book_store":{"n":19,"scored":19,"parseFailures":0,"baseRate":0.10526315789473684,"accuracy":0.8947368421052632,"brier":0.10157894736842107,"logLoss":0.3576155687607852,"auc":0.5147058823529411,"ece":0.04736842105263156,"latencyP50Ms":471,"latencyP95Ms":705.2999999999979,"usdPer1k":0.1018502839740954},"ap_paasio":{"n":21,"scored":21,"parseFailures":0,"baseRate":0.09523809523809523,"accuracy":0.42857142857142855,"brier":0.5713095238095237,"logLoss":6.419167256169617,"auc":0.7631578947368421,"ece":0.6261904761904761,"latencyP50Ms":482,"latencyP95Ms":616,"usdPer1k":0.0853243663197472},"ap_connect":{"n":57,"scored":57,"parseFailures":0,"baseRate":0.24561403508771928,"accuracy":0.7368421052631579,"brier":0.27192982456140335,"logLoss":1.486808856585606,"auc":0.7259136212624585,"ece":0.3526315789473683,"latencyP50Ms":439,"latencyP95Ms":671.3999999999997,"usdPer1k":0.09422221757654556},"ap_dot_dsl":{"n":295,"scored":295,"parseFailures":0,"baseRate":0.2711864406779661,"accuracy":0.576271186440678,"brier":0.4266525423728821,"logLoss":5.668924729699635,"auc":0.6872093023255814,"ece":0.4344067796610169,"latencyP50Ms":438,"latencyP95Ms":623.6,"usdPer1k":0.06450401623774382},"drop_off_by_one":{"n":25,"scored":25,"parseFailures":0,"baseRate":1,"accuracy":0.68,"brier":0.2592000000000001,"logLoss":0.7368279097584346,"auc":null,"ece":0.28800000000000003,"latencyP50Ms":378,"latencyP95Ms":637.5999999999999,"usdPer1k":0.021723947129249573},"t04":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0,"brier":0.905,"logLoss":8.059047825464782,"auc":0,"ece":0.95,"latencyP50Ms":363,"latencyP95Ms":537.65,"usdPer1k":0.018298334568738934},"ap_beer_song":{"n":85,"scored":85,"parseFailures":0,"baseRate":0.4823529411764706,"accuracy":0.5058823529411764,"brier":0.4483529411764705,"logLoss":4.004017734331005,"auc":0.4772727272727273,"ece":0.43176470588235294,"latencyP50Ms":416,"latencyP95Ms":576.2,"usdPer1k":0.1199735921298757},"ap_bottle_song":{"n":120,"scored":120,"parseFailures":0,"baseRate":0.425,"accuracy":0.5583333333333333,"brier":0.3836041666666666,"logLoss":2.848995557119637,"auc":0.4862176754759875,"ece":0.34375,"latencyP50Ms":428.5,"latencyP95Ms":611.2,"usdPer1k":0.053391526579856866},"take_off_by_one":{"n":17,"scored":17,"parseFailures":0,"baseRate":1,"accuracy":1,"brier":0,"logLoss":0.000001000000500029089,"auc":null,"ece":0,"latencyP50Ms":426,"latencyP95Ms":508,"usdPer1k":0.01805744818378897},"ap_hangman":{"n":34,"scored":34,"parseFailures":0,"baseRate":0.029411764705882353,"accuracy":0.17647058823529413,"brier":0.8161764705882352,"logLoss":10.13045103440962,"auc":0.6363636363636364,"ece":0.8411764705882352,"latencyP50Ms":401,"latencyP95Ms":490.75,"usdPer1k":0.0667858391789829},"t03":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0,"brier":0.905,"logLoss":8.059047825464782,"auc":0,"ece":0.95,"latencyP50Ms":427,"latencyP95Ms":522.8,"usdPer1k":0.019086826145648956},"ap_phone_number":{"n":9,"scored":9,"parseFailures":0,"baseRate":0,"accuracy":0.7777777777777778,"brier":0.25777777777777783,"logLoss":3.24716918766742,"auc":null,"ece":0.37777777777777777,"latencyP50Ms":455,"latencyP95Ms":607.1999999999999,"usdPer1k":0.10280254030227662},"t08":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0,"brier":0.905,"logLoss":8.059047825464782,"auc":0,"ece":0.95,"latencyP50Ms":427,"latencyP95Ms":482.2,"usdPer1k":0.021515886902809138},"t02":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0,"brier":0.905,"logLoss":8.059047825464782,"auc":0,"ece":0.95,"latencyP50Ms":411.5,"latencyP95Ms":474.5,"usdPer1k":0.01842551055550575},"ap_forth":{"n":15,"scored":15,"parseFailures":0,"baseRate":0,"accuracy":0.5333333333333333,"brier":0.477,"logLoss":5.118908518836483,"auc":null,"ece":0.56,"latencyP50Ms":481,"latencyP95Ms":580,"usdPer1k":0.11468647267023722},"t01":{"n":5,"scored":5,"parseFailures":0,"baseRate":0.4,"accuracy":0,"brier":0.924,"logLoss":9.210340371958928,"auc":0,"ece":0.96,"latencyP50Ms":317,"latencyP95Ms":407.2,"usdPer1k":0.01775402109622955},"t10":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0.5,"brier":0.5,"logLoss":6.907755778968009,"auc":0.5,"ece":0.5,"latencyP50Ms":408.5,"latencyP95Ms":560.9499999999999,"usdPer1k":0.024135713279247285},"t07":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0.5,"brier":0.5,"logLoss":6.907755778968009,"auc":0.5,"ece":0.5,"latencyP50Ms":380,"latencyP95Ms":413.05,"usdPer1k":0.0195446597635746},"ap_list_ops":{"n":14,"scored":14,"parseFailures":0,"baseRate":0.07142857142857142,"accuracy":0.07142857142857142,"brier":0.9285714285714286,"logLoss":12.828688446654441,"auc":0.5,"ece":0.9285714285714286,"latencyP50Ms":407,"latencyP95Ms":494.15,"usdPer1k":0.05464159825869969},"ap_poker":{"n":10,"scored":10,"parseFailures":0,"baseRate":0.3,"accuracy":0.6,"brier":0.372,"logLoss":2.2869154920756714,"auc":0.5238095238095238,"ece":0.38999999999999996,"latencyP50Ms":464,"latencyP95Ms":717.1499999999999,"usdPer1k":0.09001724433898925},"ap_grep":{"n":13,"scored":13,"parseFailures":0,"baseRate":0.07692307692307693,"accuracy":0.5384615384615384,"brier":0.47788461538461535,"logLoss":5.658571316977693,"auc":0.7916666666666666,"ece":0.5576923076923077,"latencyP50Ms":424,"latencyP95Ms":506.4,"usdPer1k":0.060673724944774916},"ap_affine_cipher":{"n":22,"scored":22,"parseFailures":0,"baseRate":0.5454545454545454,"accuracy":0.5,"brier":0.4273863636363636,"logLoss":3.708564544343908,"auc":0.525,"ece":0.425,"latencyP50Ms":481.5,"latencyP95Ms":657.0999999999999,"usdPer1k":0.09726905083656309},"t06":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0.5,"brier":0.4525,"logLoss":4.02952441273264,"auc":0.625,"ece":0.475,"latencyP50Ms":397.5,"latencyP95Ms":501.65,"usdPer1k":0.02012966904044151},"ap_dominoes":{"n":12,"scored":12,"parseFailures":0,"baseRate":0.6666666666666666,"accuracy":0.3333333333333333,"brier":0.5383333333333334,"logLoss":1.5431727574967509,"auc":0.296875,"ece":0.5833333333333334,"latencyP50Ms":402.5,"latencyP95Ms":559.75,"usdPer1k":0.06486594390869142},"ap_food_chain":{"n":13,"scored":13,"parseFailures":0,"baseRate":0.5384615384615384,"accuracy":0.5384615384615384,"brier":0.4030769230769231,"logLoss":2.0263170077530424,"auc":0.38095238095238093,"ece":0.4615384615384615,"latencyP50Ms":426,"latencyP95Ms":543.9999999999999,"usdPer1k":0.07795169874338002},"take_nth_offset":{"n":15,"scored":15,"parseFailures":0,"baseRate":1,"accuracy":1,"brier":0,"logLoss":0.000001000000500029089,"auc":null,"ece":0,"latencyP50Ms":400,"latencyP95Ms":467.09999999999997,"usdPer1k":0.02064345981280009},"t05":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0,"brier":0.905,"logLoss":8.059047825464782,"auc":0,"ece":0.95,"latencyP50Ms":394.5,"latencyP95Ms":457.75,"usdPer1k":0.01867986252903938},"t11":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0,"brier":0.905,"logLoss":8.059047825464782,"auc":0,"ece":0.95,"latencyP50Ms":447,"latencyP95Ms":577.6999999999999,"usdPer1k":0.02047304433584213},"sliding_window_drops_first":{"n":6,"scored":6,"parseFailures":0,"baseRate":0.8333333333333334,"accuracy":0.16666666666666666,"brier":0.6766666666666667,"logLoss":1.9363809967713423,"auc":0.5,"ece":0.7333333333333334,"latencyP50Ms":413,"latencyP95Ms":553.5,"usdPer1k":0.02681064756711324},"t09":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0,"brier":0.9525,"logLoss":10.93727919170734,"auc":0,"ece":0.975,"latencyP50Ms":386.5,"latencyP95Ms":428.7,"usdPer1k":0.020231409698724744},"t12":{"n":4,"scored":4,"parseFailures":0,"baseRate":0.5,"accuracy":0,"brier":0.905,"logLoss":8.059047825464782,"auc":0,"ece":0.95,"latencyP50Ms":426.5,"latencyP95Ms":490.75,"usdPer1k":0.0217320858836174},"add":{"n":1,"scored":1,"parseFailures":0,"baseRate":1,"accuracy":1,"brier":0,"logLoss":0.000001000000500029089,"auc":null,"ece":0,"latencyP50Ms":326,"latencyP95Ms":326,"usdPer1k":0.014711971282958983}}}}},"cascades":{"small>frontier":{"order":["small","frontier"],"n":1195,"sweep":[{"tau":0,"accuracy":0.502092050209205,"brier":0.43181380753137927,"ece":0.42255230125522925,"usdPer1k":0.05746076132453139,"latencyP50Ms":422,"accepted":{"small":1195,"frontier":0}},{"tau":0.1,"accuracy":0.502092050209205,"brier":0.43181380753137927,"ece":0.42255230125522925,"usdPer1k":0.05746076132453139,"latencyP50Ms":422,"accepted":{"small":1195,"frontier":0}},{"tau":0.2,"accuracy":0.502092050209205,"brier":0.43181380753137927,"ece":0.42255230125522925,"usdPer1k":0.05746076132453139,"latencyP50Ms":422,"accepted":{"small":1195,"frontier":0}},{"tau":0.25,"accuracy":0.49623430962343096,"brier":0.43419430962342964,"ece":0.4248368200836813,"usdPer1k":0.2412247780609332,"latencyP50Ms":424,"accepted":{"small":1168,"frontier":27}},{"tau":0.3,"accuracy":0.49623430962343096,"brier":0.43419430962342964,"ece":0.4248368200836813,"usdPer1k":0.2412247780609332,"latencyP50Ms":424,"accepted":{"small":1168,"frontier":27}},{"tau":0.35,"accuracy":0.5221757322175732,"brier":0.4221763179916306,"ece":0.41058577405857666,"usdPer1k":1.136878334546288,"latencyP50Ms":443,"accepted":{"small":947,"frontier":248}},{"tau":0.4,"accuracy":0.5221757322175732,"brier":0.4221763179916306,"ece":0.41058577405857666,"usdPer1k":1.136878334546288,"latencyP50Ms":443,"accepted":{"small":947,"frontier":248}},{"tau":0.45,"accuracy":0.5514644351464435,"brier":0.3872567364016737,"ece":0.35951464435146274,"usdPer1k":2.9768565772241136,"latencyP50Ms":2525,"accepted":{"small":458,"frontier":737}},{"tau":0.5,"accuracy":0.5514644351464435,"brier":0.3872567364016737,"ece":0.35951464435146274,"usdPer1k":2.9768565772241136,"latencyP50Ms":2525,"accepted":{"small":458,"frontier":737}}]},"small>large>frontier":{"order":["small","large","frontier"],"n":1195,"sweep":[{"tau":0,"accuracy":0.502092050209205,"brier":0.43181380753137927,"ece":0.42255230125522925,"usdPer1k":0.05746076132453139,"latencyP50Ms":422,"accepted":{"small":1195,"large":0,"frontier":0}},{"tau":0.1,"accuracy":0.502092050209205,"brier":0.43181380753137927,"ece":0.42255230125522925,"usdPer1k":0.05746076132453139,"latencyP50Ms":422,"accepted":{"small":1195,"large":0,"frontier":0}},{"tau":0.2,"accuracy":0.502092050209205,"brier":0.43181380753137927,"ece":0.42255230125522925,"usdPer1k":0.05746076132453139,"latencyP50Ms":422,"accepted":{"small":1195,"large":0,"frontier":0}},{"tau":0.25,"accuracy":0.48619246861924686,"brier":0.4460900418410024,"ece":0.43762343096234246,"usdPer1k":0.0764001288119718,"latencyP50Ms":424,"accepted":{"small":1168,"large":26,"frontier":1}},{"tau":0.3,"accuracy":0.48619246861924686,"brier":0.4460900418410024,"ece":0.43762343096234246,"usdPer1k":0.0764001288119718,"latencyP50Ms":424,"accepted":{"small":1168,"large":26,"frontier":1}},{"tau":0.35,"accuracy":0.49372384937238495,"brier":0.46346786610878404,"ece":0.45739748953974957,"usdPer1k":0.31898318022374844,"latencyP50Ms":443,"accepted":{"small":947,"large":194,"frontier":54}},{"tau":0.4,"accuracy":0.49372384937238495,"brier":0.46346786610878404,"ece":0.45739748953974957,"usdPer1k":0.31898318022374844,"latencyP50Ms":443,"accepted":{"small":947,"large":194,"frontier":54}},{"tau":0.45,"accuracy":0.5523012552301255,"brier":0.4156232635983249,"ece":0.38940585774058556,"usdPer1k":1.892347489114887,"latencyP50Ms":1207,"accepted":{"small":458,"large":296,"frontier":441}},{"tau":0.5,"accuracy":0.5698744769874478,"brier":0.3872583263598326,"ece":0.3553221757322161,"usdPer1k":2.625773430537483,"latencyP50Ms":2929,"accepted":{"small":458,"large":138,"frontier":599}}]}},"bestof":{"frontier":{"tier":"frontier","draws":300,"seed":11,"curve":[{"n":1,"tasks":6,"picked":0.4666666666666667,"random":0.4666666666666667,"oracle":0.4666666666666667},{"n":2,"tasks":6,"picked":0.5488888888888889,"random":0.4572222222222222,"oracle":0.675},{"n":3,"tasks":6,"picked":0.5938888888888889,"random":0.44666666666666666,"oracle":0.795},{"n":4,"tasks":6,"picked":0.6388888888888888,"random":0.47555555555555556,"oracle":0.87},{"n":6,"tasks":6,"picked":0.6588888888888889,"random":0.47055555555555556,"oracle":0.9355555555555556},{"n":8,"tasks":6,"picked":0.635,"random":0.46444444444444444,"oracle":0.9683333333333334},{"n":12,"tasks":6,"picked":0.6605555555555556,"random":0.4627777777777778,"oracle":0.9966666666666667}],"perTaskAtN4":{"ap_pov":{"n":317,"picked":0.76,"random":0.6666666666666666,"oracle":0.9966666666666667},"ap_grade_school":{"n":321,"picked":0.84,"random":0.6533333333333333,"oracle":0.9866666666666667},"ap_connect":{"n":57,"picked":0.39666666666666667,"random":0.25333333333333335,"oracle":0.6733333333333333},"ap_dot_dsl":{"n":295,"picked":0.5366666666666666,"random":0.27,"oracle":0.7033333333333334},"ap_beer_song":{"n":85,"picked":0.7033333333333334,"random":0.5466666666666666,"oracle":0.94},"ap_bottle_song":{"n":120,"picked":0.5966666666666667,"random":0.4633333333333333,"oracle":0.92}}},"large":{"tier":"large","draws":300,"seed":11,"curve":[{"n":1,"tasks":12,"picked":0.4636111111111111,"random":0.4636111111111111,"oracle":0.4636111111111111},{"n":2,"tasks":12,"picked":0.4513888888888889,"random":0.4716666666666667,"oracle":0.6930555555555555},{"n":3,"tasks":12,"picked":0.4588888888888889,"random":0.48138888888888887,"oracle":0.8055555555555556},{"n":4,"tasks":12,"picked":0.4247222222222222,"random":0.47055555555555556,"oracle":0.8586111111111111},{"n":6,"tasks":12,"picked":0.42583333333333334,"random":0.4802777777777778,"oracle":0.9227777777777778},{"n":8,"tasks":12,"picked":0.4191666666666667,"random":0.47583333333333333,"oracle":0.9547222222222222},{"n":12,"tasks":11,"picked":0.44212121212121214,"random":0.48393939393939395,"oracle":0.9842424242424243}],"perTaskAtN4":{"ap_pig_latin":{"n":25,"picked":0.6433333333333333,"random":0.75,"oracle":1},"ap_pov":{"n":317,"picked":0.58,"random":0.66,"oracle":0.9866666666666667},"ap_grade_school":{"n":321,"picked":0.37,"random":0.6333333333333333,"oracle":0.9766666666666667},"ap_book_store":{"n":19,"picked":0.06666666666666667,"random":0.13666666666666666,"oracle":0.42},"ap_connect":{"n":57,"picked":0.12,"random":0.25,"oracle":0.6466666666666666},"ap_dot_dsl":{"n":295,"picked":0.6333333333333333,"random":0.22666666666666666,"oracle":0.6766666666666666},"ap_beer_song":{"n":85,"picked":0.5066666666666667,"random":0.47,"oracle":0.9433333333333334},"ap_bottle_song":{"n":120,"picked":0.41333333333333333,"random":0.44333333333333336,"oracle":0.8933333333333333},"ap_poker":{"n":10,"picked":0.11,"random":0.30333333333333334,"oracle":0.8233333333333334},"ap_affine_cipher":{"n":22,"picked":0.5866666666666667,"random":0.5433333333333333,"oracle":0.96},"ap_dominoes":{"n":12,"picked":0.53,"random":0.6833333333333333,"oracle":1},"ap_food_chain":{"n":13,"picked":0.5366666666666666,"random":0.5466666666666666,"oracle":0.9766666666666667}}},"small":{"tier":"small","draws":300,"seed":11,"curve":[{"n":1,"tasks":12,"picked":0.4636111111111111,"random":0.4636111111111111,"oracle":0.4636111111111111},{"n":2,"tasks":12,"picked":0.4633333333333333,"random":0.4716666666666667,"oracle":0.6930555555555555},{"n":3,"tasks":12,"picked":0.45666666666666667,"random":0.48138888888888887,"oracle":0.8055555555555556},{"n":4,"tasks":12,"picked":0.44472222222222224,"random":0.47055555555555556,"oracle":0.8586111111111111},{"n":6,"tasks":12,"picked":0.43694444444444447,"random":0.4802777777777778,"oracle":0.9227777777777778},{"n":8,"tasks":12,"picked":0.4286111111111111,"random":0.47583333333333333,"oracle":0.9547222222222222},{"n":12,"tasks":11,"picked":0.43393939393939396,"random":0.48393939393939395,"oracle":0.9842424242424243}],"perTaskAtN4":{"ap_pig_latin":{"n":25,"picked":0.58,"random":0.75,"oracle":1},"ap_pov":{"n":317,"picked":0.47,"random":0.66,"oracle":0.9866666666666667},"ap_grade_school":{"n":321,"picked":0.7666666666666667,"random":0.6333333333333333,"oracle":0.9766666666666667},"ap_book_store":{"n":19,"picked":0.10666666666666667,"random":0.13666666666666666,"oracle":0.42},"ap_connect":{"n":57,"picked":0.3466666666666667,"random":0.25,"oracle":0.6466666666666666},"ap_dot_dsl":{"n":295,"picked":0.3333333333333333,"random":0.22666666666666666,"oracle":0.6766666666666666},"ap_beer_song":{"n":85,"picked":0.5566666666666666,"random":0.47,"oracle":0.9433333333333334},"ap_bottle_song":{"n":120,"picked":0.4633333333333333,"random":0.44333333333333336,"oracle":0.8933333333333333},"ap_poker":{"n":10,"picked":0.23666666666666666,"random":0.30333333333333334,"oracle":0.8233333333333334},"ap_affine_cipher":{"n":22,"picked":0.55,"random":0.5433333333333333,"oracle":0.96},"ap_dominoes":{"n":12,"picked":0.42333333333333334,"random":0.6833333333333333,"oracle":1},"ap_food_chain":{"n":13,"picked":0.5033333333333333,"random":0.5466666666666666,"oracle":0.9766666666666667}}}}}