{ "a": { "run_id": "20261002-2344", "target": "beta", "runner": "claude-code", "model": "haiku", "label": "Haiku 4.5", "repeats": 1, "judge_model": "claude-code:claude-sonnet-4-6", "judge_models": [ "claude-code:claude-sonnet-4-6" ], "questions_file": "input/questions.yaml", "created_at": "2026-10-02T23:44:44.004480+00:00", "agent_id": "agent_OHVSJI9Gd6gwsDnFSL-Xl", "librechat": null, "counts": { "expected": 146, "completed": 146, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "eligible": 146, "unexpected": 0 }, "routed_to": {}, "consistency": { "flaky": 0, "with_repeats": 0, "mean_pass_variance": null, "median_latency_stdev": null } }, "b": { "run_id": "20261002-0306", "target": "beta", "runner": "claude-code", "model": "haiku", "label": "Haiku 4.5", "repeats": 1, "judge_model": "claude-code:claude-sonnet-4-6", "judge_models": [ "claude-code:claude-sonnet-4-6" ], "questions_file": "input/questions.yaml", "created_at": "2026-10-02T03:06:52.093052+00:00", "agent_id": "agent_OHVSJI9Gd6gwsDnFSL-Xl", "librechat": null, "counts": { "expected": 146, "completed": 146, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "eligible": 146, "unexpected": 0 }, "routed_to": {}, "consistency": { "flaky": 0, "with_repeats": 0, "mean_pass_variance": null, "median_latency_stdev": null } }, "n_questions": 146, "only_a": [], "only_b": [], "warnings": [], "mismatched_questions": [], "metrics": [ { "key": "recall", "label": "Recall: passes / eligible turns (failed or missing = not passed)", "fmt": "pct", "better": "higher", "a": 64.38356164383562, "b": 56.16438356164384, "delta": -8.219178082191782, "verdict": "worse" }, { "key": "precision", "label": "Precision: passes / attempted answers", "fmt": "pct", "better": "higher", "a": 65.27777777777777, "b": 56.94444444444444, "delta": -8.333333333333329, "verdict": "worse" }, { "key": "attempt_rate", "label": "Attempt rate: attempted / eligible turns", "fmt": "pct", "better": "higher", "a": 98.63013698630137, "b": 98.63013698630137, "delta": 0.0, "verdict": "same" }, { "key": "pass_rate", "label": "Pass rate over graded answers (report definition; failures left out)", "fmt": "pct", "better": "higher", "a": 64.38356164383562, "b": 56.16438356164384, "delta": -8.219178082191782, "verdict": "worse" }, { "key": "median_latency", "label": "Median latency, completed turns", "fmt": "secs", "better": "lower", "a": 24.272344208555296, "b": 24.876953729020897, "delta": 0.6046095204656012, "verdict": "worse" }, { "key": "p90_latency", "label": "p90 latency, completed turns", "fmt": "secs", "better": "lower", "a": 69.85623683407903, "b": 67.0273954999866, "delta": -2.8288413340924308, "verdict": "better" }, { "key": "median_latency_all", "label": "Median latency, completed + failed turns", "fmt": "secs", "better": "lower", "a": 24.272344208555296, "b": 24.876953729020897, "delta": 0.6046095204656012, "verdict": "worse" }, { "key": "p90_latency_all", "label": "p90 latency, completed + failed turns", "fmt": "secs", "better": "lower", "a": 69.85623683407903, "b": 67.0273954999866, "delta": -2.8288413340924308, "verdict": "better" }, { "key": "fast_share", "label": "Under 10s, of completed turns", "fmt": "pct", "better": "higher", "a": 8.904109589041095, "b": 11.643835616438356, "delta": 2.7397260273972606, "verdict": "better" }, { "key": "fast_share_all", "label": "Under 10s, of completed + failed turns", "fmt": "pct", "better": "higher", "a": 8.904109589041095, "b": 11.643835616438356, "delta": 2.7397260273972606, "verdict": "better" }, { "key": "mean_llm_calls", "label": "LLM calls / answer (mean)", "fmt": "num", "better": "lower", "a": 8.10958904109589, "b": 8.349315068493151, "delta": 0.23972602739726057, "verdict": "worse" }, { "key": "median_llm_calls", "label": "LLM calls / answer (median)", "fmt": "num", "better": "lower", "a": 6.0, "b": 6.0, "delta": 0.0, "verdict": "same" }, { "key": "mean_tool_rounds", "label": "Tool rounds / answer (mean)", "fmt": "num", "better": "lower", "a": 0, "b": 0, "delta": 0, "verdict": "same" }, { "key": "mean_handoffs", "label": "Successful handoffs / answer", "fmt": "num", "better": "lower", "a": 0, "b": 0, "delta": 0, "verdict": "same" }, { "key": "failed_handoffs", "label": "Failed handoffs (total)", "fmt": "int", "better": "lower", "a": 0, "b": 0, "delta": 0, "verdict": "same" }, { "key": "tool_error_rate", "label": "Tool error rate", "fmt": "pct", "better": "lower", "a": 4.6817849305047545, "b": 3.80693405846363, "delta": -0.8748508720411245, "verdict": "better" }, { "key": "cost_per_answer", "label": "Cost / answer", "fmt": "usd", "better": "lower", "a": 0.061345979452054826, "b": 0.05962066404109587, "delta": -0.0017253154109589594, "verdict": "better" } ], "by_category": [ { "key": "Study discovery", "label": "Study discovery", "a": { "expected": 11, "completed": 11, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 8, "attempted": 11, "eligible": 11, "unexpected": 0, "recall": 72.72727272727273, "precision": 72.72727272727273, "attempt_rate": 100.0, "pass_rate": 72.72727272727273, "median_latency": 13.041051874868572, "p90_latency": 22.504419499775395, "median_latency_all": 13.041051874868572, "p90_latency_all": 22.504419499775395, "fast_share": 27.272727272727273, "fast_share_all": 27.272727272727273, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.142857142857143, "cost_per_answer": 0.02599065909090909, "latency_stdev": 6.446603748844569, "pass_variance": 0.1983471074380165, "flaky": true, "routed_to": {} }, "b": { "expected": 11, "completed": 11, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 9, "attempted": 11, "eligible": 11, "unexpected": 0, "recall": 81.81818181818181, "precision": 81.81818181818181, "attempt_rate": 100.0, "pass_rate": 81.81818181818181, "median_latency": 14.017888374975882, "p90_latency": 21.1325674590189, "median_latency_all": 14.017888374975882, "p90_latency_all": 21.1325674590189, "fast_share": 36.36363636363637, "fast_share_all": 36.36363636363637, "mean_llm_calls": 4.363636363636363, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.02334350454545455, "latency_stdev": 7.04611590716731, "pass_variance": 0.14876033057851237, "flaky": true, "routed_to": {} }, "delta": 9.09090909090908 }, { "key": "Cohort & clinical counts", "label": "Cohort & clinical counts", "a": { "expected": 13, "completed": 13, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 12, "attempted": 13, "eligible": 13, "unexpected": 0, "recall": 92.3076923076923, "precision": 92.3076923076923, "attempt_rate": 100.0, "pass_rate": 92.3076923076923, "median_latency": 18.765358916018158, "p90_latency": 72.42611099989153, "median_latency_all": 18.765358916018158, "p90_latency_all": 72.42611099989153, "fast_share": 15.384615384615385, "fast_share_all": 15.384615384615385, "mean_llm_calls": 8.153846153846153, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.627118644067797, "cost_per_answer": 0.05971108076923077, "latency_stdev": 25.731744499564314, "pass_variance": 0.07100591715976327, "flaky": true, "routed_to": {} }, "b": { "expected": 13, "completed": 13, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 8, "attempted": 13, "eligible": 13, "unexpected": 0, "recall": 61.53846153846154, "precision": 61.53846153846154, "attempt_rate": 100.0, "pass_rate": 61.53846153846154, "median_latency": 15.479777917033061, "p90_latency": 61.935912249959074, "median_latency_all": 15.479777917033061, "p90_latency_all": 61.935912249959074, "fast_share": 15.384615384615385, "fast_share_all": 15.384615384615385, "mean_llm_calls": 6.6923076923076925, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 6.593406593406593, "cost_per_answer": 0.043662288461538454, "latency_stdev": 26.89205551444193, "pass_variance": 0.23668639053254437, "flaky": true, "routed_to": {} }, "delta": -30.769230769230766 }, { "key": "Alteration frequency", "label": "Alteration frequency", "a": { "expected": 36, "completed": 36, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 22, "attempted": 36, "eligible": 36, "unexpected": 0, "recall": 61.111111111111114, "precision": 61.111111111111114, "attempt_rate": 100.0, "pass_rate": 61.111111111111114, "median_latency": 21.578758395975456, "p90_latency": 63.4664667500183, "median_latency_all": 21.578758395975456, "p90_latency_all": 63.4664667500183, "fast_share": 2.7777777777777777, "fast_share_all": 2.7777777777777777, "mean_llm_calls": 7.083333333333333, "median_llm_calls": 6.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.761904761904762, "cost_per_answer": 0.06102105277777777, "latency_stdev": 18.860102758928928, "pass_variance": 0.23765432098765432, "flaky": true, "routed_to": {} }, "b": { "expected": 36, "completed": 36, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 22, "attempted": 35, "eligible": 36, "unexpected": 0, "recall": 61.111111111111114, "precision": 62.857142857142854, "attempt_rate": 97.22222222222223, "pass_rate": 61.111111111111114, "median_latency": 18.379556520550977, "p90_latency": 56.31131679099053, "median_latency_all": 18.379556520550977, "p90_latency_all": 56.31131679099053, "fast_share": 13.88888888888889, "fast_share_all": 13.88888888888889, "mean_llm_calls": 7.194444444444445, "median_llm_calls": 5.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 2.0588235294117645, "cost_per_answer": 0.052933223611111094, "latency_stdev": 21.05512589706973, "pass_variance": 0.23765432098765432, "flaky": true, "routed_to": {} }, "delta": 0.0 }, { "key": "Variants & hotspots", "label": "Variants & hotspots", "a": { "expected": 16, "completed": 16, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 10, "attempted": 16, "eligible": 16, "unexpected": 0, "recall": 62.5, "precision": 62.5, "attempt_rate": 100.0, "pass_rate": 62.5, "median_latency": 26.62233320856467, "p90_latency": 53.398273583035916, "median_latency_all": 26.62233320856467, "p90_latency_all": 53.398273583035916, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8.3125, "median_llm_calls": 6.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 1.36986301369863, "cost_per_answer": 0.068387909375, "latency_stdev": 16.197183746861, "pass_variance": 0.234375, "flaky": true, "routed_to": {} }, "b": { "expected": 16, "completed": 16, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 8, "attempted": 16, "eligible": 16, "unexpected": 0, "recall": 50.0, "precision": 50.0, "attempt_rate": 100.0, "pass_rate": 50.0, "median_latency": 26.57338381302543, "p90_latency": 64.53444266703445, "median_latency_all": 26.57338381302543, "p90_latency_all": 64.53444266703445, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7.9375, "median_llm_calls": 6.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 2.0, "cost_per_answer": 0.064516003125, "latency_stdev": 16.101235717245146, "pass_variance": 0.25, "flaky": true, "routed_to": {} }, "delta": -12.5 }, { "key": "Co-occurrence & exclusivity", "label": "Co-occurrence & exclusivity", "a": { "expected": 8, "completed": 8, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 3, "attempted": 8, "eligible": 8, "unexpected": 0, "recall": 37.5, "precision": 37.5, "attempt_rate": 100.0, "pass_rate": 37.5, "median_latency": 37.22692339553032, "p90_latency": 69.85623683407903, "median_latency_all": 37.22692339553032, "p90_latency_all": 69.85623683407903, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8.875, "median_llm_calls": 9.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.594936708860759, "cost_per_answer": 0.0714636625, "latency_stdev": 16.092318875783437, "pass_variance": 0.234375, "flaky": true, "routed_to": {} }, "b": { "expected": 8, "completed": 8, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 4, "attempted": 8, "eligible": 8, "unexpected": 0, "recall": 50.0, "precision": 50.0, "attempt_rate": 100.0, "pass_rate": 50.0, "median_latency": 30.194357083004434, "p90_latency": 85.37187933304813, "median_latency_all": 30.194357083004434, "p90_latency_all": 85.37187933304813, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10.125, "median_llm_calls": 7.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.070707070707071, "cost_per_answer": 0.08579495000000001, "latency_stdev": 21.865563805107897, "pass_variance": 0.25, "flaky": true, "routed_to": {} }, "delta": 12.5 }, { "key": "Expression & multi-omics", "label": "Expression & multi-omics", "a": { "expected": 20, "completed": 20, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 12, "attempted": 19, "eligible": 20, "unexpected": 0, "recall": 60.0, "precision": 63.1578947368421, "attempt_rate": 95.0, "pass_rate": 60.0, "median_latency": 38.378604479599744, "p90_latency": 135.45407866709866, "median_latency_all": 38.378604479599744, "p90_latency_all": 135.45407866709866, "fast_share": 5.0, "fast_share_all": 5.0, "mean_llm_calls": 12.65, "median_llm_calls": 9.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 6.164383561643835, "cost_per_answer": 0.08341284, "latency_stdev": 43.416988137032426, "pass_variance": 0.24, "flaky": true, "routed_to": {} }, "b": { "expected": 20, "completed": 20, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 9, "attempted": 20, "eligible": 20, "unexpected": 0, "recall": 45.0, "precision": 45.0, "attempt_rate": 100.0, "pass_rate": 45.0, "median_latency": 50.624065104464535, "p90_latency": 187.72462337487377, "median_latency_all": 50.624065104464535, "p90_latency_all": 187.72462337487377, "fast_share": 5.0, "fast_share_all": 5.0, "mean_llm_calls": 13.6, "median_llm_calls": 9.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.483870967741935, "cost_per_answer": 0.0908440525, "latency_stdev": 51.499656789567396, "pass_variance": 0.24750000000000003, "flaky": true, "routed_to": {} }, "delta": -15.0 }, { "key": "Survival & outcomes", "label": "Survival & outcomes", "a": { "expected": 14, "completed": 14, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 9, "attempted": 13, "eligible": 14, "unexpected": 0, "recall": 64.28571428571429, "precision": 69.23076923076923, "attempt_rate": 92.85714285714286, "pass_rate": 64.28571428571429, "median_latency": 57.00851737498306, "p90_latency": 87.80997254210524, "median_latency_all": 57.00851737498306, "p90_latency_all": 87.80997254210524, "fast_share": 7.142857142857143, "fast_share_all": 7.142857142857143, "mean_llm_calls": 11.285714285714286, "median_llm_calls": 11.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 3.910614525139665, "cost_per_answer": 0.08385257500000001, "latency_stdev": 40.88940098172613, "pass_variance": 0.22959183673469385, "flaky": true, "routed_to": {} }, "b": { "expected": 14, "completed": 14, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 9, "attempted": 14, "eligible": 14, "unexpected": 0, "recall": 64.28571428571429, "precision": 64.28571428571429, "attempt_rate": 100.0, "pass_rate": 64.28571428571429, "median_latency": 36.11101235449314, "p90_latency": 94.1095539580565, "median_latency_all": 36.11101235449314, "p90_latency_all": 94.1095539580565, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11.071428571428571, "median_llm_calls": 9.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.913978494623656, "cost_per_answer": 0.083765475, "latency_stdev": 31.116910052952218, "pass_variance": 0.22959183673469385, "flaky": true, "routed_to": {} }, "delta": 0.0 }, { "key": "Treatment", "label": "Treatment", "a": { "expected": 6, "completed": 6, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 4, "attempted": 6, "eligible": 6, "unexpected": 0, "recall": 66.66666666666667, "precision": 66.66666666666667, "attempt_rate": 100.0, "pass_rate": 66.66666666666667, "median_latency": 34.93653020798229, "p90_latency": 64.8242743331939, "median_latency_all": 34.93653020798229, "p90_latency_all": 64.8242743331939, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 9.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.761904761904762, "cost_per_answer": 0.06371355833333334, "latency_stdev": 19.70919666199592, "pass_variance": 0.22222222222222224, "flaky": true, "routed_to": {} }, "b": { "expected": 6, "completed": 6, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 3, "attempted": 6, "eligible": 6, "unexpected": 0, "recall": 50.0, "precision": 50.0, "attempt_rate": 100.0, "pass_rate": 50.0, "median_latency": 54.36040793749271, "p90_latency": 140.99881487491075, "median_latency_all": 54.36040793749271, "p90_latency_all": 140.99881487491075, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 12.666666666666666, "median_llm_calls": 12.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 3.669724770642202, "cost_per_answer": 0.08383544166666668, "latency_stdev": 45.82724799893884, "pass_variance": 0.25, "flaky": true, "routed_to": {} }, "delta": -16.66666666666667 }, { "key": "Patient & sample lookup", "label": "Patient & sample lookup", "a": { "expected": 10, "completed": 10, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 5, "attempted": 10, "eligible": 10, "unexpected": 0, "recall": 50.0, "precision": 50.0, "attempt_rate": 100.0, "pass_rate": 50.0, "median_latency": 19.64964264561422, "p90_latency": 71.9098742918577, "median_latency_all": 19.64964264561422, "p90_latency_all": 71.9098742918577, "fast_share": 10.0, "fast_share_all": 10.0, "mean_llm_calls": 7.2, "median_llm_calls": 5.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 1.25, "cost_per_answer": 0.049207259999999996, "latency_stdev": 18.994483839857118, "pass_variance": 0.25, "flaky": true, "routed_to": {} }, "b": { "expected": 10, "completed": 10, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 5, "attempted": 9, "eligible": 10, "unexpected": 0, "recall": 50.0, "precision": 55.55555555555556, "attempt_rate": 90.0, "pass_rate": 50.0, "median_latency": 23.05657387553947, "p90_latency": 73.8052942919312, "median_latency_all": 23.05657387553947, "p90_latency_all": 73.8052942919312, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6.9, "median_llm_calls": 7.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 1.2345679012345678, "cost_per_answer": 0.04558893000000001, "latency_stdev": 18.334842592572944, "pass_variance": 0.25, "flaky": true, "routed_to": {} }, "delta": 0.0 }, { "key": "Out of scope", "label": "Out of scope", "a": { "expected": 12, "completed": 12, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 9, "attempted": 12, "eligible": 12, "unexpected": 0, "recall": 75.0, "precision": 75.0, "attempt_rate": 100.0, "pass_rate": 75.0, "median_latency": 11.789435292012058, "p90_latency": 41.01030508312397, "median_latency_all": 11.789435292012058, "p90_latency_all": 41.01030508312397, "fast_share": 33.333333333333336, "fast_share_all": 33.333333333333336, "mean_llm_calls": 3.6666666666666665, "median_llm_calls": 3.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.026262595833333336, "latency_stdev": 16.99780570778875, "pass_variance": 0.1875, "flaky": true, "routed_to": {} }, "b": { "expected": 12, "completed": 12, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 5, "attempted": 12, "eligible": 12, "unexpected": 0, "recall": 41.666666666666664, "precision": 41.666666666666664, "attempt_rate": 100.0, "pass_rate": 41.666666666666664, "median_latency": 13.435745124996174, "p90_latency": 24.73623345803935, "median_latency_all": 13.435745124996174, "p90_latency_all": 24.73623345803935, "fast_share": 41.666666666666664, "fast_share_all": 41.666666666666664, "mean_llm_calls": 3.75, "median_llm_calls": 2.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.02562644166666667, "latency_stdev": 12.76670684216732, "pass_variance": 0.24305555555555552, "flaky": true, "routed_to": {} }, "delta": -33.333333333333336 } ], "by_track": [ { "key": "data", "label": "Data", "a": { "expected": 62, "completed": 62, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 42, "attempted": 62, "eligible": 62, "unexpected": 0, "recall": 67.74193548387096, "precision": 67.74193548387096, "attempt_rate": 100.0, "pass_rate": 67.74193548387096, "median_latency": 19.56730258348398, "p90_latency": 48.10182329197414, "median_latency_all": 19.56730258348398, "p90_latency_all": 48.10182329197414, "fast_share": 9.67741935483871, "fast_share_all": 9.67741935483871, "mean_llm_calls": 6.096774193548387, "median_llm_calls": 6.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.086538461538462, "cost_per_answer": 0.04682392258064517, "latency_stdev": 16.153954099084487, "pass_variance": 0.21852237252861603, "flaky": true, "routed_to": {} }, "b": { "expected": 62, "completed": 62, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 43, "attempted": 62, "eligible": 62, "unexpected": 0, "recall": 69.35483870967742, "precision": 69.35483870967742, "attempt_rate": 100.0, "pass_rate": 69.35483870967742, "median_latency": 15.509465688024648, "p90_latency": 42.91149975010194, "median_latency_all": 15.509465688024648, "p90_latency_all": 42.91149975010194, "fast_share": 17.741935483870968, "fast_share_all": 17.741935483870968, "mean_llm_calls": 6.32258064516129, "median_llm_calls": 5.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 1.9736842105263157, "cost_per_answer": 0.04092405725806451, "latency_stdev": 15.577690891196546, "pass_variance": 0.21253902185223722, "flaky": true, "routed_to": {} }, "delta": 1.6129032258064626 }, { "key": "navigation", "label": "Navigation", "a": { "expected": 26, "completed": 26, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 9, "attempted": 26, "eligible": 26, "unexpected": 0, "recall": 34.61538461538461, "precision": 34.61538461538461, "attempt_rate": 100.0, "pass_rate": 34.61538461538461, "median_latency": 27.124293396016583, "p90_latency": 76.9661652080249, "median_latency_all": 27.124293396016583, "p90_latency_all": 76.9661652080249, "fast_share": 3.8461538461538463, "fast_share_all": 3.8461538461538463, "mean_llm_calls": 9.961538461538462, "median_llm_calls": 8.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.713804713804714, "cost_per_answer": 0.06759328653846156, "latency_stdev": 30.774356981448538, "pass_variance": 0.22633136094674555, "flaky": true, "routed_to": {} }, "b": { "expected": 26, "completed": 26, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 8, "attempted": 24, "eligible": 26, "unexpected": 0, "recall": 30.76923076923077, "precision": 33.333333333333336, "attempt_rate": 92.3076923076923, "pass_rate": 30.76923076923077, "median_latency": 26.016859812487382, "p90_latency": 64.53444266703445, "median_latency_all": 26.016859812487382, "p90_latency_all": 64.53444266703445, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8.961538461538462, "median_llm_calls": 7.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 2.867383512544803, "cost_per_answer": 0.06090043076923076, "latency_stdev": 24.388886606686647, "pass_variance": 0.21301775147928995, "flaky": true, "routed_to": {} }, "delta": -3.846153846153843 }, { "key": "analysis", "label": "Analysis", "a": { "expected": 46, "completed": 46, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 34, "attempted": 44, "eligible": 46, "unexpected": 0, "recall": 73.91304347826087, "precision": 77.27272727272727, "attempt_rate": 95.65217391304348, "pass_rate": 73.91304347826087, "median_latency": 47.02010527101811, "p90_latency": 87.80997254210524, "median_latency_all": 47.02010527101811, "p90_latency_all": 87.80997254210524, "fast_share": 4.3478260869565215, "fast_share_all": 4.3478260869565215, "mean_llm_calls": 10.934782608695652, "median_llm_calls": 9.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.490848585690515, "cost_per_answer": 0.08654028695652173, "latency_stdev": 33.43098475313496, "pass_variance": 0.19281663516068057, "flaky": true, "routed_to": {} }, "b": { "expected": 46, "completed": 46, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 26, "attempted": 46, "eligible": 46, "unexpected": 0, "recall": 56.52173913043478, "precision": 56.52173913043478, "attempt_rate": 100.0, "pass_rate": 56.52173913043478, "median_latency": 44.95913464552723, "p90_latency": 107.05490254203323, "median_latency_all": 44.95913464552723, "p90_latency_all": 107.05490254203323, "fast_share": 2.1739130434782608, "fast_share_all": 2.1739130434782608, "mean_llm_calls": 11.934782608695652, "median_llm_calls": 9.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.660377358490566, "cost_per_answer": 0.09296515000000002, "latency_stdev": 40.69271153511713, "pass_variance": 0.2457466918714556, "flaky": true, "routed_to": {} }, "delta": -17.391304347826093 }, { "key": "out_of_scope", "label": "Out of scope", "a": { "expected": 12, "completed": 12, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 9, "attempted": 12, "eligible": 12, "unexpected": 0, "recall": 75.0, "precision": 75.0, "attempt_rate": 100.0, "pass_rate": 75.0, "median_latency": 11.789435292012058, "p90_latency": 41.01030508312397, "median_latency_all": 11.789435292012058, "p90_latency_all": 41.01030508312397, "fast_share": 33.333333333333336, "fast_share_all": 33.333333333333336, "mean_llm_calls": 3.6666666666666665, "median_llm_calls": 3.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.026262595833333336, "latency_stdev": 16.99780570778875, "pass_variance": 0.1875, "flaky": true, "routed_to": {} }, "b": { "expected": 12, "completed": 12, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 5, "attempted": 12, "eligible": 12, "unexpected": 0, "recall": 41.666666666666664, "precision": 41.666666666666664, "attempt_rate": 100.0, "pass_rate": 41.666666666666664, "median_latency": 13.435745124996174, "p90_latency": 24.73623345803935, "median_latency_all": 13.435745124996174, "p90_latency_all": 24.73623345803935, "fast_share": 41.666666666666664, "fast_share_all": 41.666666666666664, "mean_llm_calls": 3.75, "median_llm_calls": 2.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.02562644166666667, "latency_stdev": 12.76670684216732, "pass_variance": 0.24305555555555552, "flaky": true, "routed_to": {} }, "delta": -33.333333333333336 } ], "questions": [ { "id": 4, "question": "How many primary samples are in the MSK-CHORD Study?", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 7.583730249898508, "p90_latency": 7.583730249898508, "median_latency_all": 7.583730249898508, "p90_latency_all": 7.583730249898508, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.012793, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 5.3312141669448465, "p90_latency": 5.3312141669448465, "median_latency_all": 5.3312141669448465, "p90_latency_all": 5.3312141669448465, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.00646035, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -2.2525160829536617, "verdict": "worse" }, { "id": 9, "question": "\"What are the top 5 frequently altered genes in the Osteosarcoma study from TARGET for mutations, copy numbers and SVs combined?\"", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 42.89042945811525, "p90_latency": 42.89042945811525, "median_latency_all": 42.89042945811525, "p90_latency_all": 42.89042945811525, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 12, "median_llm_calls": 12, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 13.333333333333334, "cost_per_answer": 0.09769945000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 7.9653255000011995, "p90_latency": 7.9653255000011995, "median_latency_all": 7.9653255000011995, "p90_latency_all": 7.9653255000011995, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.009151649999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -34.92510395811405, "verdict": "worse" }, { "id": 14, "question": "Which cancer type has the highest average tumor mutational burden across all studies?", "category": "Cohort & clinical counts", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 13.369705957826227, "p90_latency": 13.369705957826227, "median_latency_all": 13.369705957826227, "p90_latency_all": 13.369705957826227, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.02319065, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 13.74793320801109, "p90_latency": 13.74793320801109, "median_latency_all": 13.74793320801109, "p90_latency_all": 13.74793320801109, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.02317375, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 0.3782272501848638, "verdict": "worse" }, { "id": 16, "question": "How many unique patients have both primary and metastatic samples in the MSK-CHORD Study?", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 10.330350041855127, "p90_latency": 10.330350041855127, "median_latency_all": 10.330350041855127, "p90_latency_all": 10.330350041855127, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.019672150000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 22.2327292090049, "p90_latency": 22.2327292090049, "median_latency_all": 22.2327292090049, "p90_latency_all": 22.2327292090049, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.037197600000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 11.902379167149775, "verdict": "worse" }, { "id": 18, "question": "What are the most frequently administered systemic therapy regimens for lung cancer patients in the MSK-CHORD Study?", "category": "Treatment", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 48.98766983416863, "p90_latency": 48.98766983416863, "median_latency_all": 48.98766983416863, "p90_latency_all": 48.98766983416863, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 9.090909090909092, "cost_per_answer": 0.07156935, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 67.0273954999866, "p90_latency": 67.0273954999866, "median_latency_all": 67.0273954999866, "p90_latency_all": 67.0273954999866, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 18, "median_llm_calls": 18, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.555555555555555, "cost_per_answer": 0.1020438, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 18.03972566581797, "verdict": "worse" }, { "id": 23, "question": "What is the correlation between tumor mutational burden and microsatellite instability status in colorectal cancer patients from the MSK-CHORD Study?", "category": "Cohort & clinical counts", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 72.42611099989153, "p90_latency": 72.42611099989153, "median_latency_all": 72.42611099989153, "p90_latency_all": 72.42611099989153, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 15, "median_llm_calls": 15, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.0, "cost_per_answer": 0.15679875, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 61.935912249959074, "p90_latency": 61.935912249959074, "median_latency_all": 61.935912249959074, "p90_latency_all": 61.935912249959074, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 13.333333333333334, "cost_per_answer": 0.09356975, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -10.49019874993246, "verdict": "worse" }, { "id": 28, "question": "\"What percentage of glioblastoma patients have alterations in RB pathway genes (CDKN2A, CDK4, RB1)?\"", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 21.99409024999477, "p90_latency": 21.99409024999477, "median_latency_all": 21.99409024999477, "p90_latency_all": 21.99409024999477, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.052071, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 30.10613175004255, "p90_latency": 30.10613175004255, "median_latency_all": 30.10613175004255, "p90_latency_all": 30.10613175004255, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.11111111111111, "cost_per_answer": 0.0651299, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 8.11204150004778, "verdict": "worse" }, { "id": 43, "question": "In the TCGA PanCancer Atlas endometrial cancer study, in samples with both KRAS and NRAS mutations, what is the distribution of variant allele frequencies suggesting clonal vs subclonal events?", "category": "Variants & hotspots", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 53.398273583035916, "p90_latency": 53.398273583035916, "median_latency_all": 53.398273583035916, "p90_latency_all": 53.398273583035916, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.142857142857143, "cost_per_answer": 0.09730155, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 64.55056124995463, "p90_latency": 64.55056124995463, "median_latency_all": 64.55056124995463, "p90_latency_all": 64.55056124995463, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 13, "median_llm_calls": 13, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.142857142857143, "cost_per_answer": 0.1109465, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 11.15228766691871, "verdict": "worse" }, { "id": 47, "question": "How does ERBB2 mRNA expression vary across different cancer types in TCGA Pan-Cancer Atlas studies?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 31.949136334005743, "p90_latency": 31.949136334005743, "median_latency_all": 31.949136334005743, "p90_latency_all": 31.949136334005743, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.052148349999999996, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 38.79325995803811, "p90_latency": 38.79325995803811, "median_latency_all": 38.79325995803811, "p90_latency_all": 38.79325995803811, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.057731199999999996, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 6.844123624032363, "verdict": "worse" }, { "id": 48, "question": "In the TCGA PanCancer Atlas breast cancer study, what is the concordance between ERBB2 copy number amplification, mRNA overexpression, and protein overexpression?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 53.7614676670637, "p90_latency": 53.7614676670637, "median_latency_all": 53.7614676670637, "p90_latency_all": 53.7614676670637, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06939125, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 58.1444364170311, "p90_latency": 58.1444364170311, "median_latency_all": 58.1444364170311, "p90_latency_all": 58.1444364170311, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06643735, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 4.382968749967404, "verdict": "worse" }, { "id": 62, "question": "In the TCGA Glioblastoma multiforme study compare the median patient age at diagnosis between patients with IDH1 R132H mutation and patients with wild-type IDH1.", "category": "Cohort & clinical counts", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 58.67160724988207, "p90_latency": 58.67160724988207, "median_latency_all": 58.67160724988207, "p90_latency_all": 58.67160724988207, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 16, "median_llm_calls": 16, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.526315789473685, "cost_per_answer": 0.1164971, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 32.09884433296975, "p90_latency": 32.09884433296975, "median_latency_all": 32.09884433296975, "p90_latency_all": 32.09884433296975, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10, "median_llm_calls": 10, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 9.090909090909092, "cost_per_answer": 0.06416015, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -26.572762916912325, "verdict": "worse" }, { "id": 64, "question": "In TCGA PanCancer Atlas, which recurrent hotspot mutations occur almost exclusively in one cancer type?", "category": "Variants & hotspots", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 35.05605341680348, "p90_latency": 35.05605341680348, "median_latency_all": 35.05605341680348, "p90_latency_all": 35.05605341680348, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.078151, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 26.403779041953385, "p90_latency": 26.403779041953385, "median_latency_all": 26.403779041953385, "p90_latency_all": 26.403779041953385, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 16.666666666666668, "cost_per_answer": 0.04211, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -8.652274374850094, "verdict": "worse" }, { "id": 66, "question": "In colorectal cancer do patients with microsatellite instability in the TCGA dataset have a similar prognosis as those in the MSK Gastroenterology 2020 study?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 65.65851199999452, "p90_latency": 65.65851199999452, "median_latency_all": 65.65851199999452, "p90_latency_all": 65.65851199999452, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 18, "median_llm_calls": 18, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 9.523809523809524, "cost_per_answer": 0.11043935, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 58.47562683396973, "p90_latency": 58.47562683396973, "median_latency_all": 58.47562683396973, "p90_latency_all": 58.47562683396973, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.2631578947368425, "cost_per_answer": 0.0817927, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -7.182885166024789, "verdict": "worse" }, { "id": 76, "question": "Are TP53 mutations clinically actionable?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 8.951246124925092, "p90_latency": 8.951246124925092, "median_latency_all": 8.951246124925092, "p90_latency_all": 8.951246124925092, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.00522015, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 6.456490500015207, "p90_latency": 6.456490500015207, "median_latency_all": 6.456490500015207, "p90_latency_all": 6.456490500015207, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.00515155, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -2.4947556249098852, "verdict": "worse" }, { "id": 78, "question": "what's the best way to put clinical data into cbioportal?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 11.728277791989967, "p90_latency": 11.728277791989967, "median_latency_all": 11.728277791989967, "p90_latency_all": 11.728277791989967, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.015385049999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 14.584715207922272, "p90_latency": 14.584715207922272, "median_latency_all": 14.584715207922272, "p90_latency_all": 14.584715207922272, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.01691265, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 2.856437415932305, "verdict": "worse" }, { "id": 79, "question": "Can you please generate a survival curve for colorectal cancer patients based on the expression levels of IMPDH2?", "category": "Survival & outcomes", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 16.555832584155723, "p90_latency": 16.555832584155723, "median_latency_all": 16.555832584155723, "p90_latency_all": 16.555832584155723, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.02980025, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 19.353691500029527, "p90_latency": 19.353691500029527, "median_latency_all": 19.353691500029527, "p90_latency_all": 19.353691500029527, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.027240800000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 2.797858915873803, "verdict": "worse" }, { "id": 81, "question": "tell me about IDH1 mutations in glioma", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 25.62373641692102, "p90_latency": 25.62373641692102, "median_latency_all": 25.62373641692102, "p90_latency_all": 25.62373641692102, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06466965, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 22.674227666924708, "p90_latency": 22.674227666924708, "median_latency_all": 22.674227666924708, "p90_latency_all": 22.674227666924708, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.05490255, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -2.949508749996312, "verdict": "worse" }, { "id": 83, "question": "Help me rank TCGA cancer studies based on median CD3 expression", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 26.15529808285646, "p90_latency": 26.15529808285646, "median_latency_all": 26.15529808285646, "p90_latency_all": 26.15529808285646, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0448133, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 33.31231087492779, "p90_latency": 33.31231087492779, "median_latency_all": 33.31231087492779, "p90_latency_all": 33.31231087492779, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 16.666666666666668, "cost_per_answer": 0.0545449, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 7.157012792071328, "verdict": "worse" }, { "id": 84, "question": "Can you analyze thyroid hormone gene expression by diagnosis in the Pediatric Brain Tumor Atlas (PBTA, Provisional) cohort?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 11.85059279203415, "p90_latency": 11.85059279203415, "median_latency_all": 11.85059279203415, "p90_latency_all": 11.85059279203415, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.01623445, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 12.286775042070076, "p90_latency": 12.286775042070076, "median_latency_all": 12.286775042070076, "p90_latency_all": 12.286775042070076, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.01917005, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 0.4361822500359267, "verdict": "worse" }, { "id": 91, "question": "show me a histogram of C228T mutations in the tert promoter across cancer types", "category": "Variants & hotspots", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 49.4799349589739, "p90_latency": 49.4799349589739, "median_latency_all": 49.4799349589739, "p90_latency_all": 49.4799349589739, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 14, "median_llm_calls": 14, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.10329920000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 64.53444266703445, "p90_latency": 64.53444266703445, "median_latency_all": 64.53444266703445, "p90_latency_all": 64.53444266703445, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 17, "median_llm_calls": 17, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.12876175, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 15.054507708060555, "verdict": "worse" }, { "id": 92, "question": "what is the most prevalent TP53 mutation in uterine cancer that is not a point mutation", "category": "Variants & hotspots", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 22.663052084157243, "p90_latency": 22.663052084157243, "median_latency_all": 22.663052084157243, "p90_latency_all": 22.663052084157243, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0545841, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 15.539153459016234, "p90_latency": 15.539153459016234, "median_latency_all": 15.539153459016234, "p90_latency_all": 15.539153459016234, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.04195925, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -7.12389862514101, "verdict": "worse" }, { "id": 94, "question": "There is a heavily discussed driver alteration in MAP2K1 at codon 105 that significantly alters mRNA stability. Please list the expression values for the tumors that have a nucleotide change at this position", "category": "Expression & multi-omics", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 7.033814042108133, "p90_latency": 7.033814042108133, "median_latency_all": 7.033814042108133, "p90_latency_all": 7.033814042108133, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.00520015, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 7.9488951249513775, "p90_latency": 7.9488951249513775, "median_latency_all": 7.9488951249513775, "p90_latency_all": 7.9488951249513775, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.00535155, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 0.9150810828432441, "verdict": "worse" }, { "id": 96, "question": "Is KRAS G12C more aggressive than G12D?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 9.19752262509428, "p90_latency": 9.19752262509428, "median_latency_all": 9.19752262509428, "p90_latency_all": 9.19752262509428, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.014113549999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 10.386601791949943, "p90_latency": 10.386601791949943, "median_latency_all": 10.386601791949943, "p90_latency_all": 10.386601791949943, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.02697495, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 1.189079166855663, "verdict": "worse" }, { "id": 108, "question": "Can you find a study that I may emulate that has a data_clinical_outcomes.txt file and associated meta file to interogate", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 13.215299624949694, "p90_latency": 13.215299624949694, "median_latency_all": 13.215299624949694, "p90_latency_all": 13.215299624949694, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0207063, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 24.73623345803935, "p90_latency": 24.73623345803935, "median_latency_all": 24.73623345803935, "p90_latency_all": 24.73623345803935, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.04452010000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 11.520933833089657, "verdict": "worse" }, { "id": 113, "question": "In lower grade glioma, are there genes which are overexpressed in any of the molecular subtypes?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 135.45407866709866, "p90_latency": 135.45407866709866, "median_latency_all": 135.45407866709866, "p90_latency_all": 135.45407866709866, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 25, "median_llm_calls": 25, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 3.7037037037037037, "cost_per_answer": 0.15791254999999998, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 51.38973012496717, "p90_latency": 51.38973012496717, "median_latency_all": 51.38973012496717, "p90_latency_all": 51.38973012496717, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06581295, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -84.06434854213148, "verdict": "worse" }, { "id": 118, "question": "In TCGA lower grade glioma, show me IDH1 mRNA expression by IDH1 mutation status.", "category": "Expression & multi-omics", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 16.101902124937624, "p90_latency": 16.101902124937624, "median_latency_all": 16.101902124937624, "p90_latency_all": 16.101902124937624, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.020727950000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 10.381406000000425, "p90_latency": 10.381406000000425, "median_latency_all": 10.381406000000425, "p90_latency_all": 10.381406000000425, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0155883, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -5.720496124937199, "verdict": "worse" }, { "id": 134, "question": "compare egfr mutations between lung and brain cancer", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 25.180180000141263, "p90_latency": 25.180180000141263, "median_latency_all": 25.180180000141263, "p90_latency_all": 25.180180000141263, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06233025, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 10.119474583072588, "p90_latency": 10.119474583072588, "median_latency_all": 10.119474583072588, "p90_latency_all": 10.119474583072588, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0260757, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -15.060705417068675, "verdict": "worse" }, { "id": 7, "question": "What are the top 5 most frequently copy number altered genes in the Osteosarcoma study from TARGET?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 17.492500875145197, "p90_latency": 17.492500875145197, "median_latency_all": 17.492500875145197, "p90_latency_all": 17.492500875145197, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.028817, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 11.7000103329774, "p90_latency": 11.7000103329774, "median_latency_all": 11.7000103329774, "p90_latency_all": 11.7000103329774, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.030246, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -5.792490542167798, "verdict": "better" }, { "id": 22, "question": "Do patients with PIK3CA mutations have different overall survival outcomes compared to PIK3CA wild-type patients in breast cancer from the MSK-CHORD Study?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 0, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": null, "attempt_rate": 0.0, "pass_rate": 0.0, "median_latency": 13.700212708907202, "p90_latency": 13.700212708907202, "median_latency_all": 13.700212708907202, "p90_latency_all": 13.700212708907202, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.03731635, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "–" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 37.09595637500752, "p90_latency": 37.09595637500752, "median_latency_all": 37.09595637500752, "p90_latency_all": 37.09595637500752, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 9.090909090909092, "cost_per_answer": 0.0606784, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 23.395743666100316, "verdict": "better" }, { "id": 29, "question": "\"Are mutations in CDKN2A, CDK4, and RB1 mutually exclusive in glioblastoma patients?\"", "category": "Co-occurrence & exclusivity", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 30.86128199985251, "p90_latency": 30.86128199985251, "median_latency_all": 30.86128199985251, "p90_latency_all": 30.86128199985251, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 12.5, "cost_per_answer": 0.08566125, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 85.37187933304813, "p90_latency": 85.37187933304813, "median_latency_all": 85.37187933304813, "p90_latency_all": 85.37187933304813, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 24, "median_llm_calls": 24, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.11111111111111, "cost_per_answer": 0.15727755, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 54.51059733319562, "verdict": "better" }, { "id": 46, "question": "Which cancer types show the highest frequency of BRAF V600E mutations across all TCGA Pan-Cancer Atlas studies?", "category": "Variants & hotspots", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 24.784658959135413, "p90_latency": 24.784658959135413, "median_latency_all": 24.784658959135413, "p90_latency_all": 24.784658959135413, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.05104275, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 42.3261051250156, "p90_latency": 42.3261051250156, "median_latency_all": 42.3261051250156, "p90_latency_all": 42.3261051250156, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 13, "median_llm_calls": 13, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.6923076923076925, "cost_per_answer": 0.09608960000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 17.54144616588019, "verdict": "better" }, { "id": 55, "question": "What is the median survival time in the Pediatric Neuroblastoma study from TARGET?", "category": "Survival & outcomes", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 20.15210979199037, "p90_latency": 20.15210979199037, "median_latency_all": 20.15210979199037, "p90_latency_all": 20.15210979199037, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0300212, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 21.78750179207418, "p90_latency": 21.78750179207418, "median_latency_all": 21.78750179207418, "p90_latency_all": 21.78750179207418, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 20.0, "cost_per_answer": 0.0329023, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 1.6353920000838116, "verdict": "better" }, { "id": 63, "question": "In the TCGA PanCancer Atlas breast cancer study, what is the correlation coefficient between EGFR expression levels and PIK3CA mutation status considering only patients who also have a homozygous deletion of PTEN?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 0, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": null, "attempt_rate": 0.0, "pass_rate": 0.0, "median_latency": 11.979628500062972, "p90_latency": 11.979628500062972, "median_latency_all": 11.979628500062972, "p90_latency_all": 11.979628500062972, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.017100900000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "–" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 49.8584000839619, "p90_latency": 49.8584000839619, "median_latency_all": 49.8584000839619, "p90_latency_all": 49.8584000839619, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10, "median_llm_calls": 10, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 9.090909090909092, "cost_per_answer": 0.0941583, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 37.878771583898924, "verdict": "better" }, { "id": 73, "question": "What is the frequency of TP53 mutations in lung cancer?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 21.16342654195614, "p90_latency": 21.16342654195614, "median_latency_all": 21.16342654195614, "p90_latency_all": 21.16342654195614, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.04677885, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 19.78001700004097, "p90_latency": 19.78001700004097, "median_latency_all": 19.78001700004097, "p90_latency_all": 19.78001700004097, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.048131349999999996, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -1.3834095419151708, "verdict": "better" }, { "id": 74, "question": "What are the most mutated genes in lung cancer?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 20.861500874860212, "p90_latency": 20.861500874860212, "median_latency_all": 20.861500874860212, "p90_latency_all": 20.861500874860212, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0425006, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 14.715893624932505, "p90_latency": 14.715893624932505, "median_latency_all": 14.715893624932505, "p90_latency_all": 14.715893624932505, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.043239349999999996, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -6.145607249927707, "verdict": "better" }, { "id": 102, "question": "calculate median expression of ceacam5, itgb6, muc2, tpgb and muc1 mRNA in pancreatic cancer", "category": "Expression & multi-omics", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 21.48747487505898, "p90_latency": 21.48747487505898, "median_latency_all": 21.48747487505898, "p90_latency_all": 21.48747487505898, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 20.0, "cost_per_answer": 0.02724935, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 34.02031762502156, "p90_latency": 34.02031762502156, "median_latency_all": 34.02031762502156, "p90_latency_all": 34.02031762502156, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 13, "median_llm_calls": 13, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06413725, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 12.532842749962583, "verdict": "better" }, { "id": 107, "question": "What kind of cancer are there in the database?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 12.068518833024427, "p90_latency": 12.068518833024427, "median_latency_all": 12.068518833024427, "p90_latency_all": 12.068518833024427, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.02089865, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 14.005991666927002, "p90_latency": 14.005991666927002, "median_latency_all": 14.005991666927002, "p90_latency_all": 14.005991666927002, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.01673, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 1.937472833902575, "verdict": "better" }, { "id": 109, "question": "I receive a report with the PIK3CA p.*1069Wext*3 mutation. Can you provide details of it", "category": "Variants & hotspots", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 11.86692070798017, "p90_latency": 11.86692070798017, "median_latency_all": 11.86692070798017, "p90_latency_all": 11.86692070798017, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.02684245, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 35.535070250043646, "p90_latency": 35.535070250043646, "median_latency_all": 35.535070250043646, "p90_latency_all": 35.535070250043646, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0678667, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 23.668149542063475, "verdict": "better" }, { "id": 114, "question": "Are there differences in chromosome arm 7p copy number between lower grade glioma molecular subtypes?", "category": "Expression & multi-omics", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 151.22187341703102, "p90_latency": 151.22187341703102, "median_latency_all": 151.22187341703102, "p90_latency_all": 151.22187341703102, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 37, "median_llm_calls": 37, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.5, "cost_per_answer": 0.25001515, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 112.96131345897447, "p90_latency": 112.96131345897447, "median_latency_all": 112.96131345897447, "p90_latency_all": 112.96131345897447, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 34, "median_llm_calls": 34, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.5, "cost_per_answer": 0.2086518, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -38.26055995805655, "verdict": "better" }, { "id": 117, "question": "What cancer types in MSK-IMPACT have mutations in TP53?", "category": "Alteration frequency", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 14.15882012504153, "p90_latency": 14.15882012504153, "median_latency_all": 14.15882012504153, "p90_latency_all": 14.15882012504153, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0376315, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 58.952596207964234, "p90_latency": 58.952596207964234, "median_latency_all": 58.952596207964234, "p90_latency_all": 58.952596207964234, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 25, "median_llm_calls": 25, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.0, "cost_per_answer": 0.146692, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 44.793776082922705, "verdict": "better" }, { "id": 119, "question": "In lower grade glioma, how do outcomes differ for IDH1 altered vs EGFR altered patients?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 34.425335166975856, "p90_latency": 34.425335166975856, "median_latency_all": 34.425335166975856, "p90_latency_all": 34.425335166975856, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 9.090909090909092, "cost_per_answer": 0.06666314999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 35.126068333978765, "p90_latency": 35.126068333978765, "median_latency_all": 35.126068333978765, "p90_latency_all": 35.126068333978765, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.05225575, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 0.7007331670029089, "verdict": "better" }, { "id": 137, "question": "In the TCGA Glioblastoma multiforme study (gbm_tcga_pan_can_atlas_2018), how many patients have an IDH1 R132H mutation and how many are IDH1 wild-type.", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 36.974406958092004, "p90_latency": 36.974406958092004, "median_latency_all": 36.974406958092004, "p90_latency_all": 36.974406958092004, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 9.090909090909092, "cost_per_answer": 0.07383630000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 18.444066833006218, "p90_latency": 18.444066833006218, "median_latency_all": 18.444066833006218, "p90_latency_all": 18.444066833006218, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 20.0, "cost_per_answer": 0.0435146, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -18.530340125085786, "verdict": "better" }, { "id": 1, "question": "How many studies are in cBioPortal?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 7.848430417012423, "p90_latency": 7.848430417012423, "median_latency_all": 7.848430417012423, "p90_latency_all": 7.848430417012423, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.038178050000000005, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 7.78322079195641, "p90_latency": 7.78322079195641, "median_latency_all": 7.78322079195641, "p90_latency_all": 7.78322079195641, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.03960775, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -0.06520962505601346, "verdict": "same" }, { "id": 2, "question": "How many glioblastoma studies are in cBioPortal?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 7.526269916910678, "p90_latency": 7.526269916910678, "median_latency_all": 7.526269916910678, "p90_latency_all": 7.526269916910678, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.009253200000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 7.295206790906377, "p90_latency": 7.295206790906377, "median_latency_all": 7.295206790906377, "p90_latency_all": 7.295206790906377, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.00960305, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -0.231063126004301, "verdict": "same" }, { "id": 3, "question": "How many patients and samples are in the MSK-CHORD Study?", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 5.422681208932772, "p90_latency": 5.422681208932772, "median_latency_all": 5.422681208932772, "p90_latency_all": 5.422681208932772, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.00672685, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 5.836140208994038, "p90_latency": 5.836140208994038, "median_latency_all": 5.836140208994038, "p90_latency_all": 5.836140208994038, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.006997950000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 0.4134590000612661, "verdict": "same" }, { "id": 5, "question": "What treatment did most patients receive in the MSK-CHORD Study?", "category": "Treatment", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 13.14624737505801, "p90_latency": 13.14624737505801, "median_latency_all": 13.14624737505801, "p90_latency_all": 13.14624737505801, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0172286, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 14.141702457913198, "p90_latency": 14.141702457913198, "median_latency_all": 14.141702457913198, "p90_latency_all": 14.141702457913198, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.016124450000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 0.9954550828551874, "verdict": "same" }, { "id": 6, "question": "What are the top 5 most frequently mutated genes in the Osteosarcoma study from TARGET?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 16.540853208163753, "p90_latency": 16.540853208163753, "median_latency_all": 16.540853208163753, "p90_latency_all": 16.540853208163753, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0380423, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 12.402354791993275, "p90_latency": 12.402354791993275, "median_latency_all": 12.402354791993275, "p90_latency_all": 12.402354791993275, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 25.0, "cost_per_answer": 0.0153319, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -4.138498416170478, "verdict": "same" }, { "id": 8, "question": "What are the top 5 most frequently altered genes in a structural variant in the Osteosarcoma study from TARGET?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 12.4711567920167, "p90_latency": 12.4711567920167, "median_latency_all": 12.4711567920167, "p90_latency_all": 12.4711567920167, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.03657385, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 11.744760875008069, "p90_latency": 11.744760875008069, "median_latency_all": 11.744760875008069, "p90_latency_all": 11.744760875008069, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0324432, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -0.7263959170086309, "verdict": "same" }, { "id": 10, "question": "What is the median age at diagnosis for osteosarcoma patients in the TARGET study?", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 18.765358916018158, "p90_latency": 18.765358916018158, "median_latency_all": 18.765358916018158, "p90_latency_all": 18.765358916018158, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.038510199999999994, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 11.660302874981426, "p90_latency": 11.660302874981426, "median_latency_all": 11.660302874981426, "p90_latency_all": 11.660302874981426, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0201303, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -7.1050560410367325, "verdict": "same" }, { "id": 11, "question": "What are the top 5 most frequently mutated genes in the MSK-CHORD Study?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 14.310084749944508, "p90_latency": 14.310084749944508, "median_latency_all": 14.310084749944508, "p90_latency_all": 14.310084749944508, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.03383215, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 8.72936145903077, "p90_latency": 8.72936145903077, "median_latency_all": 8.72936145903077, "p90_latency_all": 8.72936145903077, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.011284049999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -5.580723290913738, "verdict": "same" }, { "id": 12, "question": "What is the most common cancer type in the MSK-CHORD Study based on sample count?", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 17.452195457881317, "p90_latency": 17.452195457881317, "median_latency_all": 17.452195457881317, "p90_latency_all": 17.452195457881317, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 20.0, "cost_per_answer": 0.0204221, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 15.479777917033061, "p90_latency": 15.479777917033061, "median_latency_all": 15.479777917033061, "p90_latency_all": 15.479777917033061, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.022850150000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -1.9724175408482552, "verdict": "same" }, { "id": 13, "question": "What percentage of patients in the MSK-CHORD Study have at least one TP53 mutation?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 20.06395437498577, "p90_latency": 20.06395437498577, "median_latency_all": 20.06395437498577, "p90_latency_all": 20.06395437498577, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.049797350000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 56.31131679099053, "p90_latency": 56.31131679099053, "median_latency_all": 56.31131679099053, "p90_latency_all": 56.31131679099053, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 23, "median_llm_calls": 23, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.10163255, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 36.24736241600476, "verdict": "same" }, { "id": 15, "question": "How many total studies contain mutation data in the cBioPortal database?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 6.161679999902844, "p90_latency": 6.161679999902844, "median_latency_all": 6.161679999902844, "p90_latency_all": 6.161679999902844, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0062757, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 4.999964000075124, "p90_latency": 4.999964000075124, "median_latency_all": 4.999964000075124, "p90_latency_all": 4.999964000075124, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.00630305, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -1.1617159998277202, "verdict": "same" }, { "id": 17, "question": "What are the top 5 most common primary diagnosis sites in the MSK-CHORD Study?", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 12.655637375079095, "p90_latency": 12.655637375079095, "median_latency_all": 12.655637375079095, "p90_latency_all": 12.655637375079095, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.02449265, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 14.65447020798456, "p90_latency": 14.65447020798456, "median_latency_all": 14.65447020798456, "p90_latency_all": 14.65447020798456, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0288607, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 1.9988328329054639, "verdict": "same" }, { "id": 19, "question": "What are the top 10 most frequently mutated genes across all cancer types in the MSK-CHORD Study?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 19.208780833054334, "p90_latency": 19.208780833054334, "median_latency_all": 19.208780833054334, "p90_latency_all": 19.208780833054334, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.04538145, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 16.979096041060984, "p90_latency": 16.979096041060984, "median_latency_all": 16.979096041060984, "p90_latency_all": 16.979096041060984, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.04053275, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -2.22968479199335, "verdict": "same" }, { "id": 20, "question": "What percentage of colorectal cancer samples have KRAS mutations in the MSK-CHORD Study?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 18.193725917022675, "p90_latency": 18.193725917022675, "median_latency_all": 18.193725917022675, "p90_latency_all": 18.193725917022675, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0465713, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 57.576570208068006, "p90_latency": 57.576570208068006, "median_latency_all": 57.576570208068006, "p90_latency_all": 57.576570208068006, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 24, "median_llm_calls": 24, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.11545645000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 39.38284429104533, "verdict": "same" }, { "id": 21, "question": "What are the most commonly co-occurring mutation pairs in breast cancer samples from the MSK-CHORD Study?", "category": "Co-occurrence & exclusivity", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 25.16546079190448, "p90_latency": 25.16546079190448, "median_latency_all": 25.16546079190448, "p90_latency_all": 25.16546079190448, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.055608, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 24.5089713340858, "p90_latency": 24.5089713340858, "median_latency_all": 24.5089713340858, "p90_latency_all": 24.5089713340858, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0494849, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -0.6564894578186795, "verdict": "same" }, { "id": 24, "question": "Which genomic alterations are associated with immunotherapy response in melanoma patients from the MSK-CHORD Study?", "category": "Treatment", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 15.532433375017717, "p90_latency": 15.532433375017717, "median_latency_all": 15.532433375017717, "p90_latency_all": 15.532433375017717, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0559131, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 140.99881487491075, "p90_latency": 140.99881487491075, "median_latency_all": 140.99881487491075, "p90_latency_all": 140.99881487491075, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 24, "median_llm_calls": 24, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.18496215000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 125.46638149989303, "verdict": "same" }, { "id": 25, "question": "How does the mutation landscape differ between primary and metastatic samples from the same patients in the MSK-CHORD Study?", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 50.830477457959205, "p90_latency": 50.830477457959205, "median_latency_all": 50.830477457959205, "p90_latency_all": 50.830477457959205, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.142857142857143, "cost_per_answer": 0.0905546, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 40.92044995899778, "p90_latency": 40.92044995899778, "median_latency_all": 40.92044995899778, "p90_latency_all": 40.92044995899778, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.07343039999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -9.910027498961426, "verdict": "same" }, { "id": 26, "question": "What percentage of genomic events in the MSK-CHORD Study occur in genes that are off-panel (not covered by the sequencing panel used)?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 19.263633749913424, "p90_latency": 19.263633749913424, "median_latency_all": 19.263633749913424, "p90_latency_all": 19.263633749913424, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.034050199999999996, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 15.154780915938318, "p90_latency": 15.154780915938318, "median_latency_all": 15.154780915938318, "p90_latency_all": 15.154780915938318, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.02793965, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -4.1088528339751065, "verdict": "same" }, { "id": 27, "question": "Is there a correlation between ERBB2 gene amplification and ERBB2 protein expression levels in breast cancer samples from the MSK-CHORD Study?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 62.31598745798692, "p90_latency": 62.31598745798692, "median_latency_all": 62.31598745798692, "p90_latency_all": 62.31598745798692, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 15, "median_llm_calls": 15, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.08274905, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 18.720580624998547, "p90_latency": 18.720580624998547, "median_latency_all": 18.720580624998547, "p90_latency_all": 18.720580624998547, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0300687, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -43.595406832988374, "verdict": "same" }, { "id": 30, "question": "In the TCGA PanCancer Atlas glioblastoma study, is CDK4 mRNA expression significantly higher in samples with CDK4 amplification compared to diploid samples?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 51.868061708053574, "p90_latency": 51.868061708053574, "median_latency_all": 51.868061708053574, "p90_latency_all": 51.868061708053574, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.6923076923076925, "cost_per_answer": 0.0766147, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 34.25058504194021, "p90_latency": 34.25058504194021, "median_latency_all": 34.25058504194021, "p90_latency_all": 34.25058504194021, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 12.5, "cost_per_answer": 0.04302985, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -17.61747666611336, "verdict": "same" }, { "id": 31, "question": "Which cancer types have the highest frequency of EGFR mutations across all TCGA Pan-Cancer Atlas studies?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 20.473883124999702, "p90_latency": 20.473883124999702, "median_latency_all": 20.473883124999702, "p90_latency_all": 20.473883124999702, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.05443925, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 11.164725209004246, "p90_latency": 11.164725209004246, "median_latency_all": 11.164725209004246, "p90_latency_all": 11.164725209004246, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.009168850000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -9.309157915995456, "verdict": "same" }, { "id": 32, "question": "What are the most frequent EGFR mutation variants in lung adenocarcinoma and what percentage are known hotspot mutations?", "category": "Variants & hotspots", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 22.03402654104866, "p90_latency": 22.03402654104866, "median_latency_all": 22.03402654104866, "p90_latency_all": 22.03402654104866, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0471928, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 25.017674000002444, "p90_latency": 25.017674000002444, "median_latency_all": 25.017674000002444, "p90_latency_all": 25.017674000002444, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0459125, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 2.983647458953783, "verdict": "same" }, { "id": 33, "question": "What is the correlation coefficient between EGFR copy number and EGFR mRNA expression in ovarian cancer?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 15.650371582945809, "p90_latency": 15.650371582945809, "median_latency_all": 15.650371582945809, "p90_latency_all": 15.650371582945809, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.024181099999999997, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 20.830639375024475, "p90_latency": 20.830639375024475, "median_latency_all": 20.830639375024475, "p90_latency_all": 20.830639375024475, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.03389535, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 5.180267792078666, "verdict": "same" }, { "id": 34, "question": "In the TCGA PanCancer Atlas ovarian cancer study, what is the correlation between EGFR mRNA expression and EGFR protein (RPPA) levels?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 31.887023292016238, "p90_latency": 31.887023292016238, "median_latency_all": 31.887023292016238, "p90_latency_all": 31.887023292016238, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.054240199999999995, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 17.4726960840635, "p90_latency": 17.4726960840635, "median_latency_all": 17.4726960840635, "p90_latency_all": 17.4726960840635, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.027269900000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -14.414327207952738, "verdict": "same" }, { "id": 35, "question": "In the TCGA PanCancer Atlas ovarian cancer study, do samples with TP53 truncating mutations have significantly lower TP53 mRNA expression compared to wild-type samples?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 44.808072625193745, "p90_latency": 44.808072625193745, "median_latency_all": 44.808072625193745, "p90_latency_all": 44.808072625193745, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 13, "median_llm_calls": 13, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 6.666666666666667, "cost_per_answer": 0.095161, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 52.01991516700946, "p90_latency": 52.01991516700946, "median_latency_all": 52.01991516700946, "p90_latency_all": 52.01991516700946, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 12, "median_llm_calls": 12, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.142857142857143, "cost_per_answer": 0.0784426, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 7.211842541815713, "verdict": "same" }, { "id": 36, "question": "Is BRCA1 promoter methylation associated with decreased BRCA1 mRNA expression in ovarian cancer?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 60.33826941694133, "p90_latency": 60.33826941694133, "median_latency_all": 60.33826941694133, "p90_latency_all": 60.33826941694133, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 16, "median_llm_calls": 16, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.11111111111111, "cost_per_answer": 0.09761565, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 72.72605225001462, "p90_latency": 72.72605225001462, "median_latency_all": 72.72605225001462, "p90_latency_all": 72.72605225001462, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 17, "median_llm_calls": 17, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.526315789473685, "cost_per_answer": 0.09821405, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 12.387782833073288, "verdict": "same" }, { "id": 37, "question": "Do ovarian cancer patients with BRCA1 or BRCA2 alterations have significantly different overall survival compared to wild-type patients?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 67.43744737491943, "p90_latency": 67.43744737491943, "median_latency_all": 67.43744737491943, "p90_latency_all": 67.43744737491943, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0912274, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 33.48613437497988, "p90_latency": 33.48613437497988, "median_latency_all": 33.48613437497988, "p90_latency_all": 33.48613437497988, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06383219999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -33.951312999939546, "verdict": "same" }, { "id": 38, "question": "Do lung adenocarcinoma patients with high EGFR mRNA expression (top quartile) have different survival outcomes than those with low expression?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 168.4785255000461, "p90_latency": 168.4785255000461, "median_latency_all": 168.4785255000461, "p90_latency_all": 168.4785255000461, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 29, "median_llm_calls": 29, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 3.0303030303030303, "cost_per_answer": 0.2457073, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 94.1095539580565, "p90_latency": 94.1095539580565, "median_latency_all": 94.1095539580565, "p90_latency_all": 94.1095539580565, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 21, "median_llm_calls": 21, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 8.695652173913043, "cost_per_answer": 0.17623025, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -74.3689715419896, "verdict": "same" }, { "id": 39, "question": "\"What are the survival differences between EGFR-mutated, EGFR-amplified, and EGFR wild-type lung adenocarcinoma patients?\"", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 68.40178983309306, "p90_latency": 68.40178983309306, "median_latency_all": 68.40178983309306, "p90_latency_all": 68.40178983309306, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0641583, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 49.50125508301426, "p90_latency": 49.50125508301426, "median_latency_all": 49.50125508301426, "p90_latency_all": 49.50125508301426, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.6923076923076925, "cost_per_answer": 0.07623814999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -18.900534750078805, "verdict": "same" }, { "id": 40, "question": "What are the most frequently altered genes in KRAS wild-type lung adenocarcinoma patients?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 70.93525837501511, "p90_latency": 70.93525837501511, "median_latency_all": 70.93525837501511, "p90_latency_all": 70.93525837501511, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 16, "median_llm_calls": 16, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 15.0, "cost_per_answer": 0.144188, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 43.638926958083175, "p90_latency": 43.638926958083175, "median_latency_all": 43.638926958083175, "p90_latency_all": 43.638926958083175, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.11111111111111, "cost_per_answer": 0.0757822, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -27.296331416931935, "verdict": "same" }, { "id": 41, "question": "How does PTEN alteration (mutations or homozygous deletions) affect pAKT protein levels in lung squamous cell carcinoma?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 120.24620691710152, "p90_latency": 120.24620691710152, "median_latency_all": 120.24620691710152, "p90_latency_all": 120.24620691710152, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 23, "median_llm_calls": 23, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 8.333333333333334, "cost_per_answer": 0.15692665000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 191.92802570899948, "p90_latency": 191.92802570899948, "median_latency_all": 191.92802570899948, "p90_latency_all": 191.92802570899948, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 35, "median_llm_calls": 35, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 2.6315789473684212, "cost_per_answer": 0.28418645000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 71.68181879189797, "verdict": "same" }, { "id": 42, "question": "In the TCGA PanCancer Atlas endometrial cancer study, what percentage of patients have co-occurring oncogenic mutations in both KRAS and NRAS?", "category": "Co-occurrence & exclusivity", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 38.89896187488921, "p90_latency": 38.89896187488921, "median_latency_all": 38.89896187488921, "p90_latency_all": 38.89896187488921, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.070824, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 59.45566337497439, "p90_latency": 59.45566337497439, "median_latency_all": 59.45566337497439, "p90_latency_all": 59.45566337497439, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 13, "median_llm_calls": 13, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 13.333333333333334, "cost_per_answer": 0.11191445000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 20.556701500085182, "verdict": "same" }, { "id": 44, "question": "What percentage of endometrial cancer samples have hypermutation (>5000 mutations) and how does this correlate with histological subtype?", "category": "Cohort & clinical counts", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 83.76366850011982, "p90_latency": 83.76366850011982, "median_latency_all": 83.76366850011982, "p90_latency_all": 83.76366850011982, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 19, "median_llm_calls": 19, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 8.695652173913043, "cost_per_answer": 0.20169685, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 100.60131299996283, "p90_latency": 100.60131299996283, "median_latency_all": 100.60131299996283, "p90_latency_all": 100.60131299996283, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 17, "median_llm_calls": 17, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.0, "cost_per_answer": 0.1735736, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 16.83764449984301, "verdict": "same" }, { "id": 45, "question": "What are the most frequently mutated genes in copy-number high subtype endometrial cancers compared to other subtypes?", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 63.4664667500183, "p90_latency": 63.4664667500183, "median_latency_all": 63.4664667500183, "p90_latency_all": 63.4664667500183, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 12, "median_llm_calls": 12, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 14.285714285714286, "cost_per_answer": 0.10902614999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 107.05490254203323, "p90_latency": 107.05490254203323, "median_latency_all": 107.05490254203323, "p90_latency_all": 107.05490254203323, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 21, "median_llm_calls": 21, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.0, "cost_per_answer": 0.17794135, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 43.588435792014934, "verdict": "same" }, { "id": 49, "question": "Which cancer types have the highest aneuploidy scores and how does this correlate with mutation burden across TCGA Pan-Cancer studies?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 31.90346320811659, "p90_latency": 31.90346320811659, "median_latency_all": 31.90346320811659, "p90_latency_all": 31.90346320811659, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.060433150000000005, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 56.98122958303429, "p90_latency": 56.98122958303429, "median_latency_all": 56.98122958303429, "p90_latency_all": 56.98122958303429, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 12, "median_llm_calls": 12, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 12.5, "cost_per_answer": 0.11309435000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 25.0777663749177, "verdict": "same" }, { "id": 50, "question": "\"Are mutations in DNA repair pathway genes (BRCA1, BRCA2, ATM, CHEK2) enriched in specific cancer types across TCGA Pan-Cancer Atlas?\"", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 29.346670584054664, "p90_latency": 29.346670584054664, "median_latency_all": 29.346670584054664, "p90_latency_all": 29.346670584054664, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06871985, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 42.233847457915545, "p90_latency": 42.233847457915545, "median_latency_all": 42.233847457915545, "p90_latency_all": 42.233847457915545, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.08564655, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 12.88717687386088, "verdict": "same" }, { "id": 51, "question": "What fraction of patients were older than five when diagnosed according to the Pediatric Neuroblastoma study from TARGET?", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 26.398481416050345, "p90_latency": 26.398481416050345, "median_latency_all": 26.398481416050345, "p90_latency_all": 26.398481416050345, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 14.285714285714286, "cost_per_answer": 0.038796800000000006, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 11.058286332990974, "p90_latency": 11.058286332990974, "median_latency_all": 11.058286332990974, "p90_latency_all": 11.058286332990974, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0216811, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -15.34019508305937, "verdict": "same" }, { "id": 52, "question": "What is the most frequent mutation in the TP53 gene in the TCGA breast cancer study?", "category": "Variants & hotspots", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 17.664326542057097, "p90_latency": 17.664326542057097, "median_latency_all": 17.664326542057097, "p90_latency_all": 17.664326542057097, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.04088855, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 15.595695750089362, "p90_latency": 15.595695750089362, "median_latency_all": 15.595695750089362, "p90_latency_all": 15.595695750089362, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.041515800000000005, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -2.0686307919677347, "verdict": "same" }, { "id": 53, "question": "How many patients have an EGFR amplification in the TCGA Lung Adenocarcinoma study?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 20.983555666869506, "p90_latency": 20.983555666869506, "median_latency_all": 20.983555666869506, "p90_latency_all": 20.983555666869506, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.027265650000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 6.273215082939714, "p90_latency": 6.273215082939714, "median_latency_all": 6.273215082939714, "p90_latency_all": 6.273215082939714, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.01246855, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -14.710340583929792, "verdict": "same" }, { "id": 54, "question": "Which KRAS mutations are most common in colorectal cancer?", "category": "Variants & hotspots", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 16.34329150011763, "p90_latency": 16.34329150011763, "median_latency_all": 16.34329150011763, "p90_latency_all": 16.34329150011763, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.045129350000000006, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 15.316389333922416, "p90_latency": 15.316389333922416, "median_latency_all": 15.316389333922416, "p90_latency_all": 15.316389333922416, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.044248, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -1.0269021661952138, "verdict": "same" }, { "id": 56, "question": "In the TCGA glioblastoma study (Cell 2013), how does methylation of the MGMT gene promoter affect the prognosis and treatment response in patients with glioblastoma?", "category": "Treatment", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 35.79396112496033, "p90_latency": 35.79396112496033, "median_latency_all": 35.79396112496033, "p90_latency_all": 35.79396112496033, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.142857142857143, "cost_per_answer": 0.0644286, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 70.39062387496233, "p90_latency": 70.39062387496233, "median_latency_all": 70.39062387496233, "p90_latency_all": 70.39062387496233, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 15, "median_llm_calls": 15, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.11111111111111, "cost_per_answer": 0.0949121, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 34.596662750002, "verdict": "same" }, { "id": 57, "question": "For patients with the EML4-ALK fusion gene in lung cancer how do the different fusion variants affect their long-term quality of life and risk of developing a secondary cancer?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 8.808347082929686, "p90_latency": 8.808347082929686, "median_latency_all": 8.808347082929686, "p90_latency_all": 8.808347082929686, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.00594265, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 7.93187608406879, "p90_latency": 7.93187608406879, "median_latency_all": 7.93187608406879, "p90_latency_all": 7.93187608406879, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.0056378, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -0.8764709988608956, "verdict": "same" }, { "id": 58, "question": "In the “Pan-cancer analysis of whole genomes” study what is the survival difference and the corresponding statistical significance between patients with mutations in both TP53 and KRAS versus patients with only a KRAS mutation?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 68.68970191595145, "p90_latency": 68.68970191595145, "median_latency_all": 68.68970191595145, "p90_latency_all": 68.68970191595145, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 14, "median_llm_calls": 14, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 20.0, "cost_per_answer": 0.10772785, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 30.11425483401399, "p90_latency": 30.11425483401399, "median_latency_all": 30.11425483401399, "p90_latency_all": 30.11425483401399, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.11111111111111, "cost_per_answer": 0.0571867, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -38.575447081937455, "verdict": "same" }, { "id": 59, "question": "Are there studies that were not processed using polyA enrichment in order to explore lncRNA-related questions?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 30.241244417149574, "p90_latency": 30.241244417149574, "median_latency_all": 30.241244417149574, "p90_latency_all": 30.241244417149574, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.08712660000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 51.285997375031, "p90_latency": 51.285997375031, "median_latency_all": 51.285997375031, "p90_latency_all": 51.285997375031, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 13, "median_llm_calls": 13, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1001369, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 21.04475295788143, "verdict": "same" }, { "id": 60, "question": "In the Breast Invasive Carcinoma TCGA study what are the top 5 down-regulated genes in TP53 mutated samples compared to non-mutated ones?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 70.00527529092506, "p90_latency": 70.00527529092506, "median_latency_all": 70.00527529092506, "p90_latency_all": 70.00527529092506, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 15, "median_llm_calls": 15, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 23.529411764705884, "cost_per_answer": 0.09548670000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 52.146174082998186, "p90_latency": 52.146174082998186, "median_latency_all": 52.146174082998186, "p90_latency_all": 52.146174082998186, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 13, "median_llm_calls": 13, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0734524, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -17.85910120792687, "verdict": "same" }, { "id": 61, "question": "In the Pediatric Neuroblastoma study from TARGET what is the survival difference and the corresponding statistical significance between patients who were older than four when diagnosed and the younger ones?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 44.8321332919877, "p90_latency": 44.8321332919877, "median_latency_all": 44.8321332919877, "p90_latency_all": 44.8321332919877, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 12, "median_llm_calls": 12, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.07235865, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 43.22056537505705, "p90_latency": 43.22056537505705, "median_latency_all": 43.22056537505705, "p90_latency_all": 43.22056537505705, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 8.333333333333334, "cost_per_answer": 0.0747771, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -1.6115679169306532, "verdict": "same" }, { "id": 65, "question": "In the TCGA PanCancer Atlas breast cancer study, what is the average tumor mutational burden of patients who have at least two of the five most frequently mutated genes and how does this compare to the average TMB of all patients?", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 33.73405033396557, "p90_latency": 33.73405033396557, "median_latency_all": 33.73405033396557, "p90_latency_all": 33.73405033396557, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0774551, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 42.08646308304742, "p90_latency": 42.08646308304742, "median_latency_all": 42.08646308304742, "p90_latency_all": 42.08646308304742, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10, "median_llm_calls": 10, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 15.384615384615385, "cost_per_answer": 0.07304725, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 8.35241274908185, "verdict": "same" }, { "id": 67, "question": "In the TCGA Breast Cancer study if a patient has both a PIK3CA mutation and a PTEN deletion what is their probability of having a luminal A subtype? Is this probability different from a patient with only a PIK3CA mutation?", "category": "Co-occurrence & exclusivity", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 69.85623683407903, "p90_latency": 69.85623683407903, "median_latency_all": 69.85623683407903, "p90_latency_all": 69.85623683407903, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.0, "cost_per_answer": 0.0577034, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 52.733513083076105, "p90_latency": 52.733513083076105, "median_latency_all": 52.733513083076105, "p90_latency_all": 52.733513083076105, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 14, "median_llm_calls": 14, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.882352941176471, "cost_per_answer": 0.1365357, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -17.122723751002923, "verdict": "same" }, { "id": 68, "question": "What are the key genomic differences between lung adenocarcinomas and squamous cell carcinomas identified in the Pan-Lung Cancer TCGA study?", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 40.410732083022594, "p90_latency": 40.410732083022594, "median_latency_all": 40.410732083022594, "p90_latency_all": 40.410732083022594, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.07793425, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 40.98416033398826, "p90_latency": 40.98416033398826, "median_latency_all": 40.98416033398826, "p90_latency_all": 40.98416033398826, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.07632985, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 0.5734282509656623, "verdict": "same" }, { "id": 69, "question": "Which patients have a TP53 G199V mutation? Which are somatic vs germline?", "category": "Patient & sample lookup", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 16.19073629193008, "p90_latency": 16.19073629193008, "median_latency_all": 16.19073629193008, "p90_latency_all": 16.19073629193008, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.061483800000000005, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 14.908294833032414, "p90_latency": 14.908294833032414, "median_latency_all": 14.908294833032414, "p90_latency_all": 14.908294833032414, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0489917, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -1.2824414588976651, "verdict": "same" }, { "id": 70, "question": "In MSS colorectal cancer, what is the frequency of BRAF oncogenic mutations (as defined by OncoKB)? Use the 2017 MSK study for this. Can you give me a table of all these mutations, with frequency, count and denominator?", "category": "Variants & hotspots", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 48.10182329197414, "p90_latency": 48.10182329197414, "median_latency_all": 48.10182329197414, "p90_latency_all": 48.10182329197414, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 13, "median_llm_calls": 13, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0958703, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 29.23516454093624, "p90_latency": 29.23516454093624, "median_latency_all": 29.23516454093624, "p90_latency_all": 29.23516454093624, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06152515, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -18.866658751037903, "verdict": "same" }, { "id": 71, "question": "What is the current and future support for storing and analyzing germline variants in cBioPortal, compared to other alternatives?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 20.588989624986425, "p90_latency": 20.588989624986425, "median_latency_all": 20.588989624986425, "p90_latency_all": 20.588989624986425, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0205159, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 16.263103874982335, "p90_latency": 16.263103874982335, "median_latency_all": 16.263103874982335, "p90_latency_all": 16.263103874982335, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.03841135, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -4.32588575000409, "verdict": "same" }, { "id": 72, "question": "Is there any study with a polygenic risk score?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 19.757701416965574, "p90_latency": 19.757701416965574, "median_latency_all": 19.757701416965574, "p90_latency_all": 19.757701416965574, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0304103, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 28.93217512499541, "p90_latency": 28.93217512499541, "median_latency_all": 28.93217512499541, "p90_latency_all": 28.93217512499541, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 13, "median_llm_calls": 13, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.04473485000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 9.174473708029836, "verdict": "same" }, { "id": 75, "question": "How many patients have a shallow deletion for SMARCA4 in the POG study?", "category": "Patient & sample lookup", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 23.30673862504773, "p90_latency": 23.30673862504773, "median_latency_all": 23.30673862504773, "p90_latency_all": 23.30673862504773, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10, "median_llm_calls": 10, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0455488, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 25.251175459008664, "p90_latency": 25.251175459008664, "median_latency_all": 25.251175459008664, "p90_latency_all": 25.251175459008664, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 12, "median_llm_calls": 12, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0475435, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 1.9444368339609355, "verdict": "same" }, { "id": 77, "question": "can you show me the minerva viewer for the ohsu htan sample", "category": "Patient & sample lookup", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 23.082301249960437, "p90_latency": 23.082301249960437, "median_latency_all": 23.082301249960437, "p90_latency_all": 23.082301249960437, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0350737, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 0, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": null, "attempt_rate": 0.0, "pass_rate": 0.0, "median_latency": 28.775507332989946, "p90_latency": 28.775507332989946, "median_latency_all": 28.775507332989946, "p90_latency_all": 28.775507332989946, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.04855915, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "–" }, "delta": 0.0, "latency_delta": 5.693206083029509, "verdict": "same" }, { "id": 80, "question": "Is there a cohort of NSCLC patient samples that have Kras mutations, wild-type p53, and high expression levels of c-Myc?", "category": "Patient & sample lookup", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 71.9098742918577, "p90_latency": 71.9098742918577, "median_latency_all": 71.9098742918577, "p90_latency_all": 71.9098742918577, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 14, "median_llm_calls": 14, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1324614, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 73.8052942919312, "p90_latency": 73.8052942919312, "median_latency_all": 73.8052942919312, "p90_latency_all": 73.8052942919312, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0841128, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 1.8954200000735, "verdict": "same" }, { "id": 82, "question": "How many GLASS patients developed hypermutation after TMZ treatment?", "category": "Treatment", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 64.8242743331939, "p90_latency": 64.8242743331939, "median_latency_all": 64.8242743331939, "p90_latency_all": 64.8242743331939, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.142857142857143, "cost_per_answer": 0.12086939999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 23.320388874970376, "p90_latency": 23.320388874970376, "median_latency_all": 23.320388874970376, "p90_latency_all": 23.320388874970376, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.045617, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -41.50388545822352, "verdict": "same" }, { "id": 85, "question": "generate a oncoprint of kras, stk11, keap1, tp53 alterations stratified by smoking status in lung adenocarcinoma from the mskcc clincogenomic sequencing cohort. Please only select oncogenic somatic alterations and remove samples with unknown smoking status", "category": "Alteration frequency", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 31.18790224986151, "p90_latency": 31.18790224986151, "median_latency_all": 31.18790224986151, "p90_latency_all": 31.18790224986151, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.0, "cost_per_answer": 0.044734699999999995, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 0, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": null, "attempt_rate": 0.0, "pass_rate": 0.0, "median_latency": 31.363461459055543, "p90_latency": 31.363461459055543, "median_latency_all": 31.363461459055543, "p90_latency_all": 31.363461459055543, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.07230875, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "–" }, "delta": 0.0, "latency_delta": 0.17555920919403434, "verdict": "same" }, { "id": 86, "question": "what are the most common events in her2- breast cancer?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 42.75667049991898, "p90_latency": 42.75667049991898, "median_latency_all": 42.75667049991898, "p90_latency_all": 42.75667049991898, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 13, "median_llm_calls": 13, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 14.285714285714286, "cost_per_answer": 0.0812734, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 42.91149975010194, "p90_latency": 42.91149975010194, "median_latency_all": 42.91149975010194, "p90_latency_all": 42.91149975010194, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 15, "median_llm_calls": 15, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0798579, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 0.15482925018295646, "verdict": "same" }, { "id": 87, "question": "Show me cases where PALB2 or ATM have germline mutations", "category": "Patient & sample lookup", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 13.14197887503542, "p90_latency": 13.14197887503542, "median_latency_all": 13.14197887503542, "p90_latency_all": 13.14197887503542, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.01801615, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 14.485865458031185, "p90_latency": 14.485865458031185, "median_latency_all": 14.485865458031185, "p90_latency_all": 14.485865458031185, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.022267349999999998, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 1.343886582995765, "verdict": "same" }, { "id": 88, "question": "what do idh1 mutations do?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 7.043088084086776, "p90_latency": 7.043088084086776, "median_latency_all": 7.043088084086776, "p90_latency_all": 7.043088084086776, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.00494765, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 6.352337707998231, "p90_latency": 6.352337707998231, "median_latency_all": 6.352337707998231, "p90_latency_all": 6.352337707998231, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.00503905, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -0.6907503760885447, "verdict": "same" }, { "id": 89, "question": "is there any imaging data?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 15.900410834001377, "p90_latency": 15.900410834001377, "median_latency_all": 15.900410834001377, "p90_latency_all": 15.900410834001377, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.01993445, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 21.1325674590189, "p90_latency": 21.1325674590189, "median_latency_all": 21.1325674590189, "p90_latency_all": 21.1325674590189, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.03083875, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 5.232156625017524, "verdict": "same" }, { "id": 90, "question": "show me the distribution of mutations in the tert promoter across cancer types", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 57.22530754096806, "p90_latency": 57.22530754096806, "median_latency_all": 57.22530754096806, "p90_latency_all": 57.22530754096806, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 15, "median_llm_calls": 15, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1084483, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 29.36711400002241, "p90_latency": 29.36711400002241, "median_latency_all": 29.36711400002241, "p90_latency_all": 29.36711400002241, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0534522, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -27.85819354094565, "verdict": "same" }, { "id": 93, "question": "Find patients IDs and samples in colorectal cancer that harbor the V600V alteration in BRAF", "category": "Patient & sample lookup", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 19.92238154122606, "p90_latency": 19.92238154122606, "median_latency_all": 19.92238154122606, "p90_latency_all": 19.92238154122606, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.046933550000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 12.470446708030067, "p90_latency": 12.470446708030067, "median_latency_all": 12.470446708030067, "p90_latency_all": 12.470446708030067, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.033882050000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -7.451934833195992, "verdict": "same" }, { "id": 95, "question": "download MSK-CHORD study on Non-Small Cell Lung Cancer dataset", "category": "Study discovery", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 21.902160374913365, "p90_latency": 21.902160374913365, "median_latency_all": 21.902160374913365, "p90_latency_all": 21.902160374913365, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 28.571428571428573, "cost_per_answer": 0.04900135, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 14.017888374975882, "p90_latency": 14.017888374975882, "median_latency_all": 14.017888374975882, "p90_latency_all": 14.017888374975882, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0178454, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -7.884271999937482, "verdict": "same" }, { "id": 97, "question": "give me a contingency table with the number of lung cancer patients with EGFR and/or KRAS alterations", "category": "Co-occurrence & exclusivity", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 38.4300362910144, "p90_latency": 38.4300362910144, "median_latency_all": 38.4300362910144, "p90_latency_all": 38.4300362910144, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.11111111111111, "cost_per_answer": 0.0865662, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 26.25150354206562, "p90_latency": 26.25150354206562, "median_latency_all": 26.25150354206562, "p90_latency_all": 26.25150354206562, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0620899, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -12.178532748948783, "verdict": "same" }, { "id": 98, "question": "Can you think of any flaws in the methodology used in the MSK-CHORD paper (Jee et al., Nature 2024)?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 7.568825166905299, "p90_latency": 7.568825166905299, "median_latency_all": 7.568825166905299, "p90_latency_all": 7.568825166905299, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.005165149999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 7.079789459006861, "p90_latency": 7.079789459006861, "median_latency_all": 7.079789459006861, "p90_latency_all": 7.079789459006861, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.00501155, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -0.489035707898438, "verdict": "same" }, { "id": 99, "question": "Can you help me write a production-ready Python script using Bokeh to build an interactive clinicogenomic dashboard to analyzing the MSK-CHORD dataset? I have some specific requirements I can give you", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 10.323846250073984, "p90_latency": 10.323846250073984, "median_latency_all": 10.323846250073984, "p90_latency_all": 10.323846250073984, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.01422575, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 6.855219249962829, "p90_latency": 6.855219249962829, "median_latency_all": 6.855219249962829, "p90_latency_all": 6.855219249962829, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.0054778000000000006, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -3.468627000111155, "verdict": "same" }, { "id": 100, "question": "How many samples are there that have any of these mutations in SEPHS1: p.Arg371Trp, p.Arg371Gln, p.Arg371Gly?", "category": "Patient & sample lookup", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 19.376903750002384, "p90_latency": 19.376903750002384, "median_latency_all": 19.376903750002384, "p90_latency_all": 19.376903750002384, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.04942625, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 20.861972292070277, "p90_latency": 20.861972292070277, "median_latency_all": 20.861972292070277, "p90_latency_all": 20.861972292070277, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.032396850000000005, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 1.4850685420678928, "verdict": "same" }, { "id": 101, "question": "List the top 20 mutated genes in study nbl_msk_2023.", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 11.436203832970932, "p90_latency": 11.436203832970932, "median_latency_all": 11.436203832970932, "p90_latency_all": 11.436203832970932, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.03245245, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 8.24736854096409, "p90_latency": 8.24736854096409, "median_latency_all": 8.24736854096409, "p90_latency_all": 8.24736854096409, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.00977605, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -3.188835292006843, "verdict": "same" }, { "id": 103, "question": "in salivary cancer (adenoid cystic carcinoma), what are the expected drivers ? Classify them by actionability. What about BCOR mutations, either somatic or germline ?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 70.65251783304848, "p90_latency": 70.65251783304848, "median_latency_all": 70.65251783304848, "p90_latency_all": 70.65251783304848, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.142857142857143, "cost_per_answer": 0.09077394999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 40.95905345806386, "p90_latency": 40.95905345806386, "median_latency_all": 40.95905345806386, "p90_latency_all": 40.95905345806386, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0615628, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -29.693464374984615, "verdict": "same" }, { "id": 104, "question": "write me python code that can query the timeline files for msk-chord", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 41.01030508312397, "p90_latency": 41.01030508312397, "median_latency_all": 41.01030508312397, "p90_latency_all": 41.01030508312397, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.03693245, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 22.02475137496367, "p90_latency": 22.02475137496367, "median_latency_all": 22.02475137496367, "p90_latency_all": 22.02475137496367, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0319798, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -18.985553708160296, "verdict": "same" }, { "id": 105, "question": "list the portal studies for pediatric cancers that were published in the last 5 years", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 11.134026957908645, "p90_latency": 11.134026957908645, "median_latency_all": 11.134026957908645, "p90_latency_all": 11.134026957908645, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.01474205, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 9.392431542044505, "p90_latency": 9.392431542044505, "median_latency_all": 9.392431542044505, "p90_latency_all": 9.392431542044505, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0142672, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -1.7415954158641398, "verdict": "same" }, { "id": 106, "question": "Which studies have RNA expression for renal cancer?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 23.76002945797518, "p90_latency": 23.76002945797518, "median_latency_all": 23.76002945797518, "p90_latency_all": 23.76002945797518, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 16.666666666666668, "cost_per_answer": 0.0424987, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 19.973856458091177, "p90_latency": 19.973856458091177, "median_latency_all": 19.973856458091177, "p90_latency_all": 19.973856458091177, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.03241225, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -3.786172999884002, "verdict": "same" }, { "id": 110, "question": "Can you explore the difference in mutation frequency between left-sided and right-sided CRC?", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 54.93557750014588, "p90_latency": 54.93557750014588, "median_latency_all": 54.93557750014588, "p90_latency_all": 54.93557750014588, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 12, "median_llm_calls": 12, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.142857142857143, "cost_per_answer": 0.11829630000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 46.69770391599741, "p90_latency": 46.69770391599741, "median_latency_all": 46.69770391599741, "p90_latency_all": 46.69770391599741, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.08538014999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -8.237873584148474, "verdict": "same" }, { "id": 111, "question": "can you show me a study with longitudinal data and a patient that has multiple samples over time?", "category": "Cohort & clinical counts", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 25.899991791928187, "p90_latency": 25.899991791928187, "median_latency_all": 25.899991791928187, "p90_latency_all": 25.899991791928187, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10, "median_llm_calls": 10, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.04281065, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 17.968103249906562, "p90_latency": 17.968103249906562, "median_latency_all": 17.968103249906562, "p90_latency_all": 17.968103249906562, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.02543975, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -7.931888542021625, "verdict": "same" }, { "id": 112, "question": "Which cBioPortal studies include lung adenocarcinoma samples with mutation and copy-number data?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 22.504419499775395, "p90_latency": 22.504419499775395, "median_latency_all": 22.504419499775395, "p90_latency_all": 22.504419499775395, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.03650975, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 14.757475542020984, "p90_latency": 14.757475542020984, "median_latency_all": 14.757475542020984, "p90_latency_all": 14.757475542020984, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0208144, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -7.746943957754411, "verdict": "same" }, { "id": 115, "question": "Are there DNA methylation differences between lower grade glioma molecular subtypes?", "category": "Expression & multi-omics", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 83.77965245884843, "p90_latency": 83.77965245884843, "median_latency_all": 83.77965245884843, "p90_latency_all": 83.77965245884843, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 23, "median_llm_calls": 23, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.5, "cost_per_answer": 0.1467429, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 85.49320962501224, "p90_latency": 85.49320962501224, "median_latency_all": 85.49320962501224, "p90_latency_all": 85.49320962501224, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 23, "median_llm_calls": 23, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.714285714285714, "cost_per_answer": 0.11772764999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 1.7135571661638096, "verdict": "same" }, { "id": 116, "question": "What are the frequencies of different KRAS mutations in TCGA PanCan Lung Adenocarcinoma?", "category": "Variants & hotspots", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 16.354385500075296, "p90_latency": 16.354385500075296, "median_latency_all": 16.354385500075296, "p90_latency_all": 16.354385500075296, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.04019995, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 16.622070208075456, "p90_latency": 16.622070208075456, "median_latency_all": 16.622070208075456, "p90_latency_all": 16.622070208075456, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.040970099999999995, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 0.26768470800016075, "verdict": "same" }, { "id": 120, "question": "is there a relationship between cic mutation and 19q del in lgg?", "category": "Co-occurrence & exclusivity", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 61.06280433316715, "p90_latency": 61.06280433316715, "median_latency_all": 61.06280433316715, "p90_latency_all": 61.06280433316715, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 15, "median_llm_calls": 15, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 13.333333333333334, "cost_per_answer": 0.09010710000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 32.427313333027996, "p90_latency": 32.427313333027996, "median_latency_all": 32.427313333027996, "p90_latency_all": 32.427313333027996, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06398345, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -28.635491000139154, "verdict": "same" }, { "id": 121, "question": "In TCGA lower grade glioma, show me samples with EGFR gains.", "category": "Alteration frequency", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 23.610738791991025, "p90_latency": 23.610738791991025, "median_latency_all": 23.610738791991025, "p90_latency_all": 23.610738791991025, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.052467400000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 23.039730667020194, "p90_latency": 23.039730667020194, "median_latency_all": 23.039730667020194, "p90_latency_all": 23.039730667020194, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.046173900000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -0.571008124970831, "verdict": "same" }, { "id": 122, "question": "In TCGA lower grade glioma, filter to samples that are both IDH1 and TP53 mutant and show me the summary page.", "category": "Patient & sample lookup", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 9.650154917035252, "p90_latency": 9.650154917035252, "median_latency_all": 9.650154917035252, "p90_latency_all": 9.650154917035252, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0147868, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 11.26802941702772, "p90_latency": 11.26802941702772, "median_latency_all": 11.26802941702772, "p90_latency_all": 11.26802941702772, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0153439, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 1.6178744999924675, "verdict": "same" }, { "id": 123, "question": "In TCGA lower grade glioma, show me samples that are TP53 mutant or EGFR amplified.", "category": "Patient & sample lookup", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 14.320340208010748, "p90_latency": 14.320340208010748, "median_latency_all": 14.320340208010748, "p90_latency_all": 14.320340208010748, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.017986150000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 33.062957166926935, "p90_latency": 33.062957166926935, "median_latency_all": 33.062957166926935, "p90_latency_all": 33.062957166926935, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.11111111111111, "cost_per_answer": 0.07224019999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 18.742616958916187, "verdict": "same" }, { "id": 124, "question": "show me EGFR expression across cancer types", "category": "Expression & multi-omics", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 28.34859500010498, "p90_latency": 28.34859500010498, "median_latency_all": 28.34859500010498, "p90_latency_all": 28.34859500010498, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.049737449999999996, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 18.327570249908604, "p90_latency": 18.327570249908604, "median_latency_all": 18.327570249908604, "p90_latency_all": 18.327570249908604, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.03589525, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -10.021024750196375, "verdict": "same" }, { "id": 125, "question": "show me point mutations in EGFR in lung cancer", "category": "Variants & hotspots", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 28.460007457993925, "p90_latency": 28.460007457993925, "median_latency_all": 28.460007457993925, "p90_latency_all": 28.460007457993925, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06797410000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 13.31783399998676, "p90_latency": 13.31783399998676, "median_latency_all": 13.31783399998676, "p90_latency_all": 13.31783399998676, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.03670895, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -15.142173458007164, "verdict": "same" }, { "id": 126, "question": "are there cdkn2a het losses in gbm tcga study?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 19.251771125011146, "p90_latency": 19.251771125011146, "median_latency_all": 19.251771125011146, "p90_latency_all": 19.251771125011146, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 14.285714285714286, "cost_per_answer": 0.0336546, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 13.439961791038513, "p90_latency": 13.439961791038513, "median_latency_all": 13.439961791038513, "p90_latency_all": 13.439961791038513, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.01977275, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -5.811809333972633, "verdict": "same" }, { "id": 127, "question": "show me gbm with mgmt hypermethylation", "category": "Patient & sample lookup", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 45.46448408300057, "p90_latency": 45.46448408300057, "median_latency_all": 45.46448408300057, "p90_latency_all": 45.46448408300057, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 16, "median_llm_calls": 16, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.2631578947368425, "cost_per_answer": 0.070356, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 26.20411729195621, "p90_latency": 26.20411729195621, "median_latency_all": 26.20411729195621, "p90_latency_all": 26.20411729195621, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0505518, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -19.260366791044362, "verdict": "same" }, { "id": 128, "question": "Is there a lower grade glioma study with race data?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 13.041051874868572, "p90_latency": 13.041051874868572, "median_latency_all": 13.041051874868572, "p90_latency_all": 13.041051874868572, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.018195050000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 15.311370582901873, "p90_latency": 15.311370582901873, "median_latency_all": 15.311370582901873, "p90_latency_all": 15.311370582901873, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.02362185, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 2.270318708033301, "verdict": "same" }, { "id": 129, "question": "compare atrx mutant vs cic mutant lgg - are there different outcomes?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 87.80997254210524, "p90_latency": 87.80997254210524, "median_latency_all": 87.80997254210524, "p90_latency_all": 87.80997254210524, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 14, "median_llm_calls": 14, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.12285389999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 67.72154749999754, "p90_latency": 67.72154749999754, "median_latency_all": 67.72154749999754, "p90_latency_all": 67.72154749999754, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.6923076923076925, "cost_per_answer": 0.0998866, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -20.0884250421077, "verdict": "same" }, { "id": 130, "question": "are there different outcomes for idh1 mutant vs egfr amp in lgg?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 64.8089574999176, "p90_latency": 64.8089574999176, "median_latency_all": 64.8089574999176, "p90_latency_all": 64.8089574999176, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 15, "median_llm_calls": 15, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.10265675, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 33.3106213749852, "p90_latency": 33.3106213749852, "median_latency_all": 33.3106213749852, "p90_latency_all": 33.3106213749852, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0875905, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -31.498336124932393, "verdict": "same" }, { "id": 131, "question": "what other genes are altered in kras mutant crc or luad", "category": "Co-occurrence & exclusivity", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 27.279392999829724, "p90_latency": 27.279392999829724, "median_latency_all": 27.279392999829724, "p90_latency_all": 27.279392999829724, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.054405800000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 27.96140083298087, "p90_latency": 27.96140083298087, "median_latency_all": 27.96140083298087, "p90_latency_all": 27.96140083298087, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0523109, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 0.6820078331511468, "verdict": "same" }, { "id": 132, "question": "Create an OncoPrint with a merged track for the EGFR family genes (EGFR, ERBB2, ERBB3, ERBB4) across TCGA PanCancer Atlas studies.", "category": "Alteration frequency", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 25.264511584071442, "p90_latency": 25.264511584071442, "median_latency_all": 25.264511584071442, "p90_latency_all": 25.264511584071442, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.036169150000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 14.971150250174105, "p90_latency": 14.971150250174105, "median_latency_all": 14.971150250174105, "p90_latency_all": 14.971150250174105, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.01865985, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -10.293361333897337, "verdict": "same" }, { "id": 133, "question": "What are the EGFR mutation frequencies across cancer types in the MSK-IMPACT 50K study?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 17.166514915879816, "p90_latency": 17.166514915879816, "median_latency_all": 17.166514915879816, "p90_latency_all": 17.166514915879816, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0411, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 15.173811500193551, "p90_latency": 15.173811500193551, "median_latency_all": 15.173811500193551, "p90_latency_all": 15.173811500193551, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.035426150000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -1.9927034156862646, "verdict": "same" }, { "id": 135, "question": "How does overall survival compare between prostate cancer patients where TMPRSS2 is acting as the upstream donor to an ERG fusion event and patients without this fusion?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 49.20807725004852, "p90_latency": 49.20807725004852, "median_latency_all": 49.20807725004852, "p90_latency_all": 49.20807725004852, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 13, "median_llm_calls": 13, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.07889205, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 125.43349083396606, "p90_latency": 125.43349083396606, "median_latency_all": 125.43349083396606, "p90_latency_all": 125.43349083396606, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 44, "median_llm_calls": 44, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.444444444444445, "cost_per_answer": 0.25513025, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 76.22541358391754, "verdict": "same" }, { "id": 136, "question": "In the TCGA Breast Cancer study, do TP53 mutations and high MYC expression co-occur or are they mutually exclusive?", "category": "Co-occurrence & exclusivity", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 36.02381050004624, "p90_latency": 36.02381050004624, "median_latency_all": 36.02381050004624, "p90_latency_all": 36.02381050004624, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.0, "cost_per_answer": 0.07083355000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 27.75049216602929, "p90_latency": 27.75049216602929, "median_latency_all": 27.75049216602929, "p90_latency_all": 27.75049216602929, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 12.5, "cost_per_answer": 0.05276275, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -8.273318334016949, "verdict": "same" }, { "id": 138, "question": "show me all KRAS mutations in colorectal cancer that are not at position 12", "category": "Variants & hotspots", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 29.804507832974195, "p90_latency": 29.804507832974195, "median_latency_all": 29.804507832974195, "p90_latency_all": 29.804507832974195, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.07184825, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 25.829602333018556, "p90_latency": 25.829602333018556, "median_latency_all": 25.829602333018556, "p90_latency_all": 25.829602333018556, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0689636, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -3.9749054999556392, "verdict": "same" }, { "id": 139, "question": "show me cholangio with idh1 mutations other than r132", "category": "Variants & hotspots", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 51.75310091697611, "p90_latency": 51.75310091697611, "median_latency_all": 51.75310091697611, "p90_latency_all": 51.75310091697611, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 17, "median_llm_calls": 17, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1289756, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 26.742988584097475, "p90_latency": 26.742988584097475, "median_latency_all": 26.742988584097475, "p90_latency_all": 26.742988584097475, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06907195, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -25.010112332878634, "verdict": "same" }, { "id": 140, "question": "what's the frequency of different idh1 mutations in glioma vs cholangio vs chndrosarcoma? Give me a link to compare the frequency of the specific IDH1 mutations in those cancer types.", "category": "Variants & hotspots", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 62.84515754203312, "p90_latency": 62.84515754203312, "median_latency_all": 62.84515754203312, "p90_latency_all": 62.84515754203312, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 12, "median_llm_calls": 12, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.6923076923076925, "cost_per_answer": 0.10300445000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 42.9717679170426, "p90_latency": 42.9717679170426, "median_latency_all": 42.9717679170426, "p90_latency_all": 42.9717679170426, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10, "median_llm_calls": 10, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.08260505, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -19.873389624990523, "verdict": "same" }, { "id": 141, "question": "is there a relatinoship between mgmt methylation and idh1 mutation in glioma?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 111.15956991701387, "p90_latency": 111.15956991701387, "median_latency_all": 111.15956991701387, "p90_latency_all": 111.15956991701387, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 29, "median_llm_calls": 29, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.16655835, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 187.72462337487377, "p90_latency": 187.72462337487377, "median_latency_all": 187.72462337487377, "p90_latency_all": 187.72462337487377, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 47, "median_llm_calls": 47, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 3.9215686274509802, "cost_per_answer": 0.2792596, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 76.5650534578599, "verdict": "same" }, { "id": 142, "question": "Give me an OncoPrint for RTK genes in lung cancer, limited to driver events.", "category": "Alteration frequency", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 12.250319665996358, "p90_latency": 12.250319665996358, "median_latency_all": 12.250319665996358, "p90_latency_all": 12.250319665996358, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.019287150000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 12.55894554196857, "p90_latency": 12.55894554196857, "median_latency_all": 12.55894554196857, "p90_latency_all": 12.55894554196857, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.036539800000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 0.30862587597221136, "verdict": "same" }, { "id": 143, "question": "show me P135L mutations in $p14^{ARF}$", "category": "Variants & hotspots", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 17.65531670791097, "p90_latency": 17.65531670791097, "median_latency_all": 17.65531670791097, "p90_latency_all": 17.65531670791097, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.04190215, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 32.01869891700335, "p90_latency": 32.01869891700335, "median_latency_all": 32.01869891700335, "p90_latency_all": 32.01869891700335, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 13, "median_llm_calls": 13, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.053001150000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 14.363382209092379, "verdict": "same" }, { "id": 144, "question": "are lung carcinosarcomas associated with BRIP1 mutations?", "category": "Alteration frequency", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 76.9661652080249, "p90_latency": 76.9661652080249, "median_latency_all": 76.9661652080249, "p90_latency_all": 76.9661652080249, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 18, "median_llm_calls": 18, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.761904761904762, "cost_per_answer": 0.1405141, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 16.331552208168432, "p90_latency": 16.331552208168432, "median_latency_all": 16.331552208168432, "p90_latency_all": 16.331552208168432, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.02453375, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -60.63461299985647, "verdict": "same" }, { "id": 145, "question": "Which genes are enriched for mutations between NSCLC vs squamous cell carcinoma?", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 7.7899905000813305, "p90_latency": 7.7899905000813305, "median_latency_all": 7.7899905000813305, "p90_latency_all": 7.7899905000813305, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.031256650000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 8.39595391601324, "p90_latency": 8.39595391601324, "median_latency_all": 8.39595391601324, "p90_latency_all": 8.39595391601324, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.010147950000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 0.6059634159319103, "verdict": "same" }, { "id": 146, "question": "In the MSK-CHORD study, is it possible to see which patients received radiation therapy?", "category": "Treatment", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 34.07909929100424, "p90_latency": 34.07909929100424, "median_latency_all": 34.07909929100424, "p90_latency_all": 34.07909929100424, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10, "median_llm_calls": 10, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0522723, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 41.69342037499882, "p90_latency": 41.69342037499882, "median_latency_all": 41.69342037499882, "p90_latency_all": 41.69342037499882, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.05935315, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 7.614321083994582, "verdict": "same" } ], "question_verdicts": { "better": 15, "worse": 27, "same": 104 } }