{ "a": { "run_id": "20261002-2344", "target": "beta", "runner": "claude-code", "model": "sonnet", "label": "Sonnet 5", "repeats": 1, "judge_model": "claude-code:claude-sonnet-4-6", "judge_models": [ "claude-code:claude-sonnet-4-6" ], "questions_file": "input/questions.yaml", "created_at": "2026-10-02T23:44:44.004480+00:00", "agent_id": "agent_OHVSJI9Gd6gwsDnFSL-Xl", "librechat": null, "counts": { "expected": 146, "completed": 146, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "eligible": 146, "unexpected": 0 }, "routed_to": {}, "consistency": { "flaky": 0, "with_repeats": 0, "mean_pass_variance": null, "median_latency_stdev": null } }, "b": { "run_id": "20261002-0306", "target": "beta", "runner": "claude-code", "model": "sonnet", "label": "Sonnet 5", "repeats": 1, "judge_model": "claude-code:claude-sonnet-4-6", "judge_models": [ "claude-code:claude-sonnet-4-6" ], "questions_file": "input/questions.yaml", "created_at": "2026-10-02T03:06:52.093052+00:00", "agent_id": "agent_OHVSJI9Gd6gwsDnFSL-Xl", "librechat": null, "counts": { "expected": 146, "completed": 146, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "eligible": 146, "unexpected": 0 }, "routed_to": {}, "consistency": { "flaky": 0, "with_repeats": 0, "mean_pass_variance": null, "median_latency_stdev": null } }, "n_questions": 146, "only_a": [], "only_b": [], "warnings": [], "mismatched_questions": [], "metrics": [ { "key": "recall", "label": "Recall: passes / eligible turns (failed or missing = not passed)", "fmt": "pct", "better": "higher", "a": 62.32876712328767, "b": 69.86301369863014, "delta": 7.534246575342472, "verdict": "better" }, { "key": "precision", "label": "Precision: passes / attempted answers", "fmt": "pct", "better": "higher", "a": 63.19444444444444, "b": 71.83098591549296, "delta": 8.636541471048517, "verdict": "better" }, { "key": "attempt_rate", "label": "Attempt rate: attempted / eligible turns", "fmt": "pct", "better": "higher", "a": 98.63013698630137, "b": 97.26027397260275, "delta": -1.3698630136986196, "verdict": "worse" }, { "key": "pass_rate", "label": "Pass rate over graded answers (report definition; failures left out)", "fmt": "pct", "better": "higher", "a": 62.32876712328767, "b": 69.86301369863014, "delta": 7.534246575342472, "verdict": "better" }, { "key": "median_latency", "label": "Median latency, completed turns", "fmt": "secs", "better": "lower", "a": 30.105337791028433, "b": 27.482272312045097, "delta": -2.623065478983335, "verdict": "better" }, { "key": "p90_latency", "label": "p90 latency, completed turns", "fmt": "secs", "better": "lower", "a": 56.87397245806642, "b": 58.92000625003129, "delta": 2.0460337919648737, "verdict": "worse" }, { "key": "median_latency_all", "label": "Median latency, completed + failed turns", "fmt": "secs", "better": "lower", "a": 30.105337791028433, "b": 27.482272312045097, "delta": -2.623065478983335, "verdict": "better" }, { "key": "p90_latency_all", "label": "p90 latency, completed + failed turns", "fmt": "secs", "better": "lower", "a": 56.87397245806642, "b": 58.92000625003129, "delta": 2.0460337919648737, "verdict": "worse" }, { "key": "fast_share", "label": "Under 10s, of completed turns", "fmt": "pct", "better": "higher", "a": 2.73972602739726, "b": 4.109589041095891, "delta": 1.3698630136986307, "verdict": "better" }, { "key": "fast_share_all", "label": "Under 10s, of completed + failed turns", "fmt": "pct", "better": "higher", "a": 2.73972602739726, "b": 4.109589041095891, "delta": 1.3698630136986307, "verdict": "better" }, { "key": "mean_llm_calls", "label": "LLM calls / answer (mean)", "fmt": "num", "better": "lower", "a": 6.383561643835616, "b": 5.958904109589041, "delta": -0.4246575342465748, "verdict": "better" }, { "key": "median_llm_calls", "label": "LLM calls / answer (median)", "fmt": "num", "better": "lower", "a": 6.0, "b": 5.0, "delta": -1.0, "verdict": "better" }, { "key": "mean_tool_rounds", "label": "Tool rounds / answer (mean)", "fmt": "num", "better": "lower", "a": 0, "b": 0, "delta": 0, "verdict": "same" }, { "key": "mean_handoffs", "label": "Successful handoffs / answer", "fmt": "num", "better": "lower", "a": 0, "b": 0, "delta": 0, "verdict": "same" }, { "key": "failed_handoffs", "label": "Failed handoffs (total)", "fmt": "int", "better": "lower", "a": 0, "b": 0, "delta": 0, "verdict": "same" }, { "key": "tool_error_rate", "label": "Tool error rate", "fmt": "pct", "better": "lower", "a": 4.0096230954290295, "b": 3.944124897288414, "delta": -0.0654981981406153, "verdict": "better" }, { "key": "cost_per_answer", "label": "Cost / answer", "fmt": "usd", "better": "lower", "a": 0.14089268630136986, "b": 0.1307750424657534, "delta": -0.010117643835616474, "verdict": "better" } ], "by_category": [ { "key": "Study discovery", "label": "Study discovery", "a": { "expected": 11, "completed": 11, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 7, "attempted": 11, "eligible": 11, "unexpected": 0, "recall": 63.63636363636363, "precision": 63.63636363636363, "attempt_rate": 100.0, "pass_rate": 63.63636363636363, "median_latency": 15.242907125037163, "p90_latency": 28.456359874922782, "median_latency_all": 15.242907125037163, "p90_latency_all": 28.456359874922782, "fast_share": 18.181818181818183, "fast_share_all": 18.181818181818183, "mean_llm_calls": 4.090909090909091, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0680745, "latency_stdev": 7.9034423201500354, "pass_variance": 0.23140495867768596, "flaky": true, "routed_to": {} }, "b": { "expected": 11, "completed": 11, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 8, "attempted": 11, "eligible": 11, "unexpected": 0, "recall": 72.72727272727273, "precision": 72.72727272727273, "attempt_rate": 100.0, "pass_rate": 72.72727272727273, "median_latency": 16.770684791961685, "p90_latency": 25.69504424999468, "median_latency_all": 16.770684791961685, "p90_latency_all": 25.69504424999468, "fast_share": 9.090909090909092, "fast_share_all": 9.090909090909092, "mean_llm_calls": 3.4545454545454546, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06073763636363636, "latency_stdev": 6.474180217557922, "pass_variance": 0.1983471074380165, "flaky": true, "routed_to": {} }, "delta": 9.0909090909091 }, { "key": "Cohort & clinical counts", "label": "Cohort & clinical counts", "a": { "expected": 13, "completed": 13, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 7, "attempted": 13, "eligible": 13, "unexpected": 0, "recall": 53.84615384615385, "precision": 53.84615384615385, "attempt_rate": 100.0, "pass_rate": 53.84615384615385, "median_latency": 22.4792595829349, "p90_latency": 42.23939470900223, "median_latency_all": 22.4792595829349, "p90_latency_all": 42.23939470900223, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6.076923076923077, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 1.9047619047619047, "cost_per_answer": 0.11361456923076924, "latency_stdev": 11.606916811209933, "pass_variance": 0.2485207100591716, "flaky": true, "routed_to": {} }, "b": { "expected": 13, "completed": 13, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 10, "attempted": 12, "eligible": 13, "unexpected": 0, "recall": 76.92307692307692, "precision": 83.33333333333333, "attempt_rate": 92.3076923076923, "pass_rate": 76.92307692307692, "median_latency": 19.85682850005105, "p90_latency": 33.550132083008066, "median_latency_all": 19.85682850005105, "p90_latency_all": 33.550132083008066, "fast_share": 23.076923076923077, "fast_share_all": 23.076923076923077, "mean_llm_calls": 4.846153846153846, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 1.25, "cost_per_answer": 0.08775329230769231, "latency_stdev": 13.605712389829158, "pass_variance": 0.17751479289940827, "flaky": true, "routed_to": {} }, "delta": 23.076923076923073 }, { "key": "Alteration frequency", "label": "Alteration frequency", "a": { "expected": 36, "completed": 36, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 22, "attempted": 35, "eligible": 36, "unexpected": 0, "recall": 61.111111111111114, "precision": 62.857142857142854, "attempt_rate": 97.22222222222223, "pass_rate": 61.111111111111114, "median_latency": 30.2371538329171, "p90_latency": 56.80317087494768, "median_latency_all": 30.2371538329171, "p90_latency_all": 56.80317087494768, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6.75, "median_llm_calls": 6.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.424778761061947, "cost_per_answer": 0.16787212500000004, "latency_stdev": 19.56696184965026, "pass_variance": 0.23765432098765432, "flaky": true, "routed_to": {} }, "b": { "expected": 36, "completed": 36, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 23, "attempted": 35, "eligible": 36, "unexpected": 0, "recall": 63.888888888888886, "precision": 65.71428571428571, "attempt_rate": 97.22222222222223, "pass_rate": 63.888888888888886, "median_latency": 25.65666435402818, "p90_latency": 58.52062904101331, "median_latency_all": 25.65666435402818, "p90_latency_all": 58.52062904101331, "fast_share": 2.7777777777777777, "fast_share_all": 2.7777777777777777, "mean_llm_calls": 5.527777777777778, "median_llm_calls": 5.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 2.807017543859649, "cost_per_answer": 0.12554417500000004, "latency_stdev": 17.986640284792735, "pass_variance": 0.2307098765432099, "flaky": true, "routed_to": {} }, "delta": 2.7777777777777715 }, { "key": "Variants & hotspots", "label": "Variants & hotspots", "a": { "expected": 16, "completed": 16, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 5, "attempted": 16, "eligible": 16, "unexpected": 0, "recall": 31.25, "precision": 31.25, "attempt_rate": 100.0, "pass_rate": 31.25, "median_latency": 28.14037314604502, "p90_latency": 65.49161366699263, "median_latency_all": 28.14037314604502, "p90_latency_all": 65.49161366699263, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6.25, "median_llm_calls": 5.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 3.4965034965034967, "cost_per_answer": 0.16733781875, "latency_stdev": 21.293254336727387, "pass_variance": 0.21484375, "flaky": true, "routed_to": {} }, "b": { "expected": 16, "completed": 16, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 10, "attempted": 16, "eligible": 16, "unexpected": 0, "recall": 62.5, "precision": 62.5, "attempt_rate": 100.0, "pass_rate": 62.5, "median_latency": 24.79029656248167, "p90_latency": 54.66823850013316, "median_latency_all": 24.79029656248167, "p90_latency_all": 54.66823850013316, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5.6875, "median_llm_calls": 5.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 1.5037593984962405, "cost_per_answer": 0.16208258749999999, "latency_stdev": 17.979047568019453, "pass_variance": 0.234375, "flaky": true, "routed_to": {} }, "delta": 31.25 }, { "key": "Co-occurrence & exclusivity", "label": "Co-occurrence & exclusivity", "a": { "expected": 8, "completed": 8, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 4, "attempted": 8, "eligible": 8, "unexpected": 0, "recall": 50.0, "precision": 50.0, "attempt_rate": 100.0, "pass_rate": 50.0, "median_latency": 34.93324418750126, "p90_latency": 81.02412062510848, "median_latency_all": 34.93324418750126, "p90_latency_all": 81.02412062510848, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7.5, "median_llm_calls": 7.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 3.658536585365854, "cost_per_answer": 0.19528111250000002, "latency_stdev": 21.4774562592137, "pass_variance": 0.25, "flaky": true, "routed_to": {} }, "b": { "expected": 8, "completed": 8, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 5, "attempted": 8, "eligible": 8, "unexpected": 0, "recall": 62.5, "precision": 62.5, "attempt_rate": 100.0, "pass_rate": 62.5, "median_latency": 39.72822106257081, "p90_latency": 70.32496604206972, "median_latency_all": 39.72822106257081, "p90_latency_all": 70.32496604206972, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 7.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 6.185567010309279, "cost_per_answer": 0.17483959999999998, "latency_stdev": 14.022192517371975, "pass_variance": 0.234375, "flaky": true, "routed_to": {} }, "delta": 12.5 }, { "key": "Expression & multi-omics", "label": "Expression & multi-omics", "a": { "expected": 20, "completed": 20, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 14, "attempted": 19, "eligible": 20, "unexpected": 0, "recall": 70.0, "precision": 73.6842105263158, "attempt_rate": 95.0, "pass_rate": 70.0, "median_latency": 48.29655331268441, "p90_latency": 101.06706654187292, "median_latency_all": 48.29655331268441, "p90_latency_all": 101.06706654187292, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7.85, "median_llm_calls": 7.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.392156862745098, "cost_per_answer": 0.152803925, "latency_stdev": 21.367009076404038, "pass_variance": 0.21000000000000002, "flaky": true, "routed_to": {} }, "b": { "expected": 20, "completed": 20, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 14, "attempted": 19, "eligible": 20, "unexpected": 0, "recall": 70.0, "precision": 73.6842105263158, "attempt_rate": 95.0, "pass_rate": 70.0, "median_latency": 45.293677978916094, "p90_latency": 118.68866045796312, "median_latency_all": 45.293677978916094, "p90_latency_all": 118.68866045796312, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8.95, "median_llm_calls": 7.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 6.557377049180328, "cost_per_answer": 0.173615915, "latency_stdev": 37.262002238727554, "pass_variance": 0.21000000000000002, "flaky": true, "routed_to": {} }, "delta": 0.0 }, { "key": "Survival & outcomes", "label": "Survival & outcomes", "a": { "expected": 14, "completed": 14, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 12, "attempted": 14, "eligible": 14, "unexpected": 0, "recall": 85.71428571428571, "precision": 85.71428571428571, "attempt_rate": 100.0, "pass_rate": 85.71428571428571, "median_latency": 40.2368824374862, "p90_latency": 58.06169991707429, "median_latency_all": 40.2368824374862, "p90_latency_all": 58.06169991707429, "fast_share": 7.142857142857143, "fast_share_all": 7.142857142857143, "mean_llm_calls": 7.642857142857143, "median_llm_calls": 7.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.761904761904762, "cost_per_answer": 0.1657057, "latency_stdev": 19.263865758781513, "pass_variance": 0.12244897959183677, "flaky": true, "routed_to": {} }, "b": { "expected": 14, "completed": 14, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 13, "attempted": 14, "eligible": 14, "unexpected": 0, "recall": 92.85714285714286, "precision": 92.85714285714286, "attempt_rate": 100.0, "pass_rate": 92.85714285714286, "median_latency": 46.01695362501778, "p90_latency": 58.487465125042945, "median_latency_all": 46.01695362501778, "p90_latency_all": 58.487465125042945, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7.5, "median_llm_calls": 7.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 3.896103896103896, "cost_per_answer": 0.17957160714285716, "latency_stdev": 25.647589200194325, "pass_variance": 0.06632653061224487, "flaky": true, "routed_to": {} }, "delta": 7.142857142857153 }, { "key": "Treatment", "label": "Treatment", "a": { "expected": 6, "completed": 6, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 5, "attempted": 6, "eligible": 6, "unexpected": 0, "recall": 83.33333333333333, "precision": 83.33333333333333, "attempt_rate": 100.0, "pass_rate": 83.33333333333333, "median_latency": 26.57941954152193, "p90_latency": 56.87397245806642, "median_latency_all": 26.57941954152193, "p90_latency_all": 56.87397245806642, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6.166666666666667, "median_llm_calls": 5.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.660377358490566, "cost_per_answer": 0.1190505, "latency_stdev": 17.34058365878589, "pass_variance": 0.13888888888888887, "flaky": true, "routed_to": {} }, "b": { "expected": 6, "completed": 6, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 5, "attempted": 6, "eligible": 6, "unexpected": 0, "recall": 83.33333333333333, "precision": 83.33333333333333, "attempt_rate": 100.0, "pass_rate": 83.33333333333333, "median_latency": 31.41759439604357, "p90_latency": 70.73389512498397, "median_latency_all": 31.41759439604357, "p90_latency_all": 70.73389512498397, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6.5, "median_llm_calls": 6.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.407407407407407, "cost_per_answer": 0.1354165833333333, "latency_stdev": 25.571466568967537, "pass_variance": 0.13888888888888887, "flaky": true, "routed_to": {} }, "delta": 0.0 }, { "key": "Patient & sample lookup", "label": "Patient & sample lookup", "a": { "expected": 10, "completed": 10, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 8, "attempted": 10, "eligible": 10, "unexpected": 0, "recall": 80.0, "precision": 80.0, "attempt_rate": 100.0, "pass_rate": 80.0, "median_latency": 30.03904104209505, "p90_latency": 44.91181025002152, "median_latency_all": 30.03904104209505, "p90_latency_all": 44.91181025002152, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6.7, "median_llm_calls": 7.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 2.3255813953488373, "cost_per_answer": 0.13560455, "latency_stdev": 9.407727486292428, "pass_variance": 0.15999999999999998, "flaky": true, "routed_to": {} }, "b": { "expected": 10, "completed": 10, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 7, "attempted": 9, "eligible": 10, "unexpected": 0, "recall": 70.0, "precision": 77.77777777777777, "attempt_rate": 90.0, "pass_rate": 70.0, "median_latency": 29.269873499462847, "p90_latency": 55.93408775003627, "median_latency_all": 29.269873499462847, "p90_latency_all": 55.93408775003627, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5.7, "median_llm_calls": 5.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 6.024096385542169, "cost_per_answer": 0.12386854, "latency_stdev": 12.705680487926285, "pass_variance": 0.21000000000000002, "flaky": true, "routed_to": {} }, "delta": -10.0 }, { "key": "Out of scope", "label": "Out of scope", "a": { "expected": 12, "completed": 12, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 7, "attempted": 12, "eligible": 12, "unexpected": 0, "recall": 58.333333333333336, "precision": 58.333333333333336, "attempt_rate": 100.0, "pass_rate": 58.333333333333336, "median_latency": 16.50799814600032, "p90_latency": 27.122881249990314, "median_latency_all": 16.50799814600032, "p90_latency_all": 27.122881249990314, "fast_share": 8.333333333333334, "fast_share_all": 8.333333333333334, "mean_llm_calls": 3.0833333333333335, "median_llm_calls": 2.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.2631578947368425, "cost_per_answer": 0.05126383333333334, "latency_stdev": 16.157604794877432, "pass_variance": 0.24305555555555555, "flaky": true, "routed_to": {} }, "b": { "expected": 12, "completed": 12, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 7, "attempted": 12, "eligible": 12, "unexpected": 0, "recall": 58.333333333333336, "precision": 58.333333333333336, "attempt_rate": 100.0, "pass_rate": 58.333333333333336, "median_latency": 20.63091033353703, "p90_latency": 28.457912041107193, "median_latency_all": 20.63091033353703, "p90_latency_all": 28.457912041107193, "fast_share": 8.333333333333334, "fast_share_all": 8.333333333333334, "mean_llm_calls": 2.9166666666666665, "median_llm_calls": 3.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0612596, "latency_stdev": 7.645293194621528, "pass_variance": 0.24305555555555555, "flaky": true, "routed_to": {} }, "delta": 0.0 } ], "by_track": [ { "key": "data", "label": "Data", "a": { "expected": 62, "completed": 62, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 34, "attempted": 61, "eligible": 62, "unexpected": 0, "recall": 54.83870967741935, "precision": 55.73770491803279, "attempt_rate": 98.38709677419355, "pass_rate": 54.83870967741935, "median_latency": 24.67585993744433, "p90_latency": 47.248311541974545, "median_latency_all": 24.67585993744433, "p90_latency_all": 47.248311541974545, "fast_share": 3.225806451612903, "fast_share_all": 3.225806451612903, "mean_llm_calls": 5.806451612903226, "median_llm_calls": 5.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 3.1982942430703623, "cost_per_answer": 0.12387942741935484, "latency_stdev": 12.467305127429151, "pass_variance": 0.24765868886576484, "flaky": true, "routed_to": {} }, "b": { "expected": 62, "completed": 62, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 46, "attempted": 60, "eligible": 62, "unexpected": 0, "recall": 74.19354838709677, "precision": 76.66666666666667, "attempt_rate": 96.7741935483871, "pass_rate": 74.19354838709677, "median_latency": 21.281572707986925, "p90_latency": 48.039967874996364, "median_latency_all": 21.281572707986925, "p90_latency_all": 48.039967874996364, "fast_share": 6.451612903225806, "fast_share_all": 6.451612903225806, "mean_llm_calls": 5.016129032258065, "median_llm_calls": 5.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 2.863961813842482, "cost_per_answer": 0.1095444258064516, "latency_stdev": 15.352544076435542, "pass_variance": 0.1914672216441207, "flaky": true, "routed_to": {} }, "delta": 19.354838709677416 }, { "key": "navigation", "label": "Navigation", "a": { "expected": 26, "completed": 26, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 12, "attempted": 26, "eligible": 26, "unexpected": 0, "recall": 46.15384615384615, "precision": 46.15384615384615, "attempt_rate": 100.0, "pass_rate": 46.15384615384615, "median_latency": 29.446023125434294, "p90_latency": 81.02412062510848, "median_latency_all": 29.446023125434294, "p90_latency_all": 81.02412062510848, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7.038461538461538, "median_llm_calls": 6.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.958677685950414, "cost_per_answer": 0.15034128076923076, "latency_stdev": 23.77979994724948, "pass_variance": 0.2485207100591716, "flaky": true, "routed_to": {} }, "b": { "expected": 26, "completed": 26, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 12, "attempted": 25, "eligible": 26, "unexpected": 0, "recall": 46.15384615384615, "precision": 48.0, "attempt_rate": 96.15384615384616, "pass_rate": 46.15384615384615, "median_latency": 24.901463582995348, "p90_latency": 75.45076083298773, "median_latency_all": 24.901463582995348, "p90_latency_all": 75.45076083298773, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6.8076923076923075, "median_llm_calls": 5.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.545454545454546, "cost_per_answer": 0.13744958461538462, "latency_stdev": 25.22020532638144, "pass_variance": 0.2485207100591716, "flaky": true, "routed_to": {} }, "delta": 0.0 }, { "key": "analysis", "label": "Analysis", "a": { "expected": 46, "completed": 46, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 38, "attempted": 45, "eligible": 46, "unexpected": 0, "recall": 82.6086956521739, "precision": 84.44444444444444, "attempt_rate": 97.82608695652173, "pass_rate": 82.6086956521739, "median_latency": 44.457330749952234, "p90_latency": 72.28012279095128, "median_latency_all": 44.457330749952234, "p90_latency_all": 72.28012279095128, "fast_share": 2.1739130434782608, "fast_share_all": 2.1739130434782608, "mean_llm_calls": 7.6521739130434785, "median_llm_calls": 7.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.216867469879518, "cost_per_answer": 0.18186453043478262, "latency_stdev": 19.487289983790497, "pass_variance": 0.1436672967863894, "flaky": true, "routed_to": {} }, "b": { "expected": 46, "completed": 46, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 37, "attempted": 45, "eligible": 46, "unexpected": 0, "recall": 80.43478260869566, "precision": 82.22222222222223, "attempt_rate": 97.82608695652173, "pass_rate": 80.43478260869566, "median_latency": 43.89108637499157, "p90_latency": 69.91114275006112, "median_latency_all": 43.89108637499157, "p90_latency_all": 69.91114275006112, "fast_share": 2.1739130434782608, "fast_share_all": 2.1739130434782608, "mean_llm_calls": 7.543478260869565, "median_llm_calls": 7.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.863813229571984, "cost_per_answer": 0.17375211739130436, "latency_stdev": 28.01826691505167, "pass_variance": 0.15737240075614364, "flaky": true, "routed_to": {} }, "delta": -2.173913043478251 }, { "key": "out_of_scope", "label": "Out of scope", "a": { "expected": 12, "completed": 12, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 7, "attempted": 12, "eligible": 12, "unexpected": 0, "recall": 58.333333333333336, "precision": 58.333333333333336, "attempt_rate": 100.0, "pass_rate": 58.333333333333336, "median_latency": 16.50799814600032, "p90_latency": 27.122881249990314, "median_latency_all": 16.50799814600032, "p90_latency_all": 27.122881249990314, "fast_share": 8.333333333333334, "fast_share_all": 8.333333333333334, "mean_llm_calls": 3.0833333333333335, "median_llm_calls": 2.5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.2631578947368425, "cost_per_answer": 0.05126383333333334, "latency_stdev": 16.157604794877432, "pass_variance": 0.24305555555555555, "flaky": true, "routed_to": {} }, "b": { "expected": 12, "completed": 12, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 7, "attempted": 12, "eligible": 12, "unexpected": 0, "recall": 58.333333333333336, "precision": 58.333333333333336, "attempt_rate": 100.0, "pass_rate": 58.333333333333336, "median_latency": 20.63091033353703, "p90_latency": 28.457912041107193, "median_latency_all": 20.63091033353703, "p90_latency_all": 28.457912041107193, "fast_share": 8.333333333333334, "fast_share_all": 8.333333333333334, "mean_llm_calls": 2.9166666666666665, "median_llm_calls": 3.0, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0612596, "latency_stdev": 7.645293194621528, "pass_variance": 0.24305555555555555, "flaky": true, "routed_to": {} }, "delta": 0.0 } ], "questions": [ { "id": 13, "question": "What percentage of patients in the MSK-CHORD Study have at least one TP53 mutation?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 23.66706775012426, "p90_latency": 23.66706775012426, "median_latency_all": 23.66706775012426, "p90_latency_all": 23.66706775012426, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 16.666666666666668, "cost_per_answer": 0.1156369, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 12.112771999905817, "p90_latency": 12.112771999905817, "median_latency_all": 12.112771999905817, "p90_latency_all": 12.112771999905817, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0490459, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -11.554295750218444, "verdict": "worse" }, { "id": 20, "question": "What percentage of colorectal cancer samples have KRAS mutations in the MSK-CHORD Study?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 30.895173915894702, "p90_latency": 30.895173915894702, "median_latency_all": 30.895173915894702, "p90_latency_all": 30.895173915894702, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 12.5, "cost_per_answer": 0.1490423, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 16.22285804199055, "p90_latency": 16.22285804199055, "median_latency_all": 16.22285804199055, "p90_latency_all": 16.22285804199055, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1327576, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -14.672315873904154, "verdict": "worse" }, { "id": 42, "question": "In the TCGA PanCancer Atlas endometrial cancer study, what percentage of patients have co-occurring oncogenic mutations in both KRAS and NRAS?", "category": "Co-occurrence & exclusivity", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 37.827598500065506, "p90_latency": 37.827598500065506, "median_latency_all": 37.827598500065506, "p90_latency_all": 37.827598500065506, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1815161, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 30.739338999963365, "p90_latency": 30.739338999963365, "median_latency_all": 30.739338999963365, "p90_latency_all": 30.739338999963365, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1392698, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -7.08825950010214, "verdict": "worse" }, { "id": 62, "question": "In the TCGA Glioblastoma multiforme study compare the median patient age at diagnosis between patients with IDH1 R132H mutation and patients with wild-type IDH1.", "category": "Cohort & clinical counts", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 44.81809825007804, "p90_latency": 44.81809825007804, "median_latency_all": 44.81809825007804, "p90_latency_all": 44.81809825007804, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.239813, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 0, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": null, "attempt_rate": 0.0, "pass_rate": 0.0, "median_latency": 8.061707542045042, "p90_latency": 8.061707542045042, "median_latency_all": 8.061707542045042, "p90_latency_all": 8.061707542045042, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0447953, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "–" }, "delta": -100.0, "latency_delta": -36.756390708032995, "verdict": "worse" }, { "id": 64, "question": "In TCGA PanCancer Atlas, which recurrent hotspot mutations occur almost exclusively in one cancer type?", "category": "Variants & hotspots", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 65.49161366699263, "p90_latency": 65.49161366699263, "median_latency_all": 65.49161366699263, "p90_latency_all": 65.49161366699263, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.334731, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 47.56963633408304, "p90_latency": 47.56963633408304, "median_latency_all": 47.56963633408304, "p90_latency_all": 47.56963633408304, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.2295343, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -17.92197733290959, "verdict": "worse" }, { "id": 68, "question": "What are the key genomic differences between lung adenocarcinomas and squamous cell carcinomas identified in the Pan-Lung Cancer TCGA study?", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 56.80317087494768, "p90_latency": 56.80317087494768, "median_latency_all": 56.80317087494768, "p90_latency_all": 56.80317087494768, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.2177956, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 58.52062904101331, "p90_latency": 58.52062904101331, "median_latency_all": 58.52062904101331, "p90_latency_all": 58.52062904101331, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.2172265, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 1.7174581660656258, "verdict": "worse" }, { "id": 85, "question": "generate a oncoprint of kras, stk11, keap1, tp53 alterations stratified by smoking status in lung adenocarcinoma from the mskcc clincogenomic sequencing cohort. Please only select oncogenic somatic alterations and remove samples with unknown smoking status", "category": "Alteration frequency", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 46.59957333304919, "p90_latency": 46.59957333304919, "median_latency_all": 46.59957333304919, "p90_latency_all": 46.59957333304919, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.2562986, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 53.943866583984345, "p90_latency": 53.943866583984345, "median_latency_all": 53.943866583984345, "p90_latency_all": 53.943866583984345, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 6.25, "cost_per_answer": 0.1798622, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 7.344293250935152, "verdict": "worse" }, { "id": 86, "question": "what are the most common events in her2- breast cancer?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 56.32509158295579, "p90_latency": 56.32509158295579, "median_latency_all": 56.32509158295579, "p90_latency_all": 56.32509158295579, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 12, "median_llm_calls": 12, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 9.090909090909092, "cost_per_answer": 0.2751955, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 0, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": null, "attempt_rate": 0.0, "pass_rate": 0.0, "median_latency": 13.579973125015385, "p90_latency": 13.579973125015385, "median_latency_all": 13.579973125015385, "p90_latency_all": 13.579973125015385, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 50.0, "cost_per_answer": 0.0433494, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "–" }, "delta": -100.0, "latency_delta": -42.74511845794041, "verdict": "worse" }, { "id": 96, "question": "Is KRAS G12C more aggressive than G12D?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 8.78464304190129, "p90_latency": 8.78464304190129, "median_latency_all": 8.78464304190129, "p90_latency_all": 8.78464304190129, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.032387900000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 16.48411679198034, "p90_latency": 16.48411679198034, "median_latency_all": 16.48411679198034, "p90_latency_all": 16.48411679198034, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.08471, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 7.699473750079051, "verdict": "worse" }, { "id": 102, "question": "calculate median expression of ceacam5, itgb6, muc2, tpgb and muc1 mRNA in pancreatic cancer", "category": "Expression & multi-omics", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 49.38875245815143, "p90_latency": 49.38875245815143, "median_latency_all": 49.38875245815143, "p90_latency_all": 49.38875245815143, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0866307, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 0, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": null, "attempt_rate": 0.0, "pass_rate": 0.0, "median_latency": 15.432937584002502, "p90_latency": 15.432937584002502, "median_latency_all": 15.432937584002502, "p90_latency_all": 15.432937584002502, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0532725, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "–" }, "delta": -100.0, "latency_delta": -33.95581487414893, "verdict": "worse" }, { "id": 104, "question": "write me python code that can query the timeline files for msk-chord", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 20.715546749997884, "p90_latency": 20.715546749997884, "median_latency_all": 20.715546749997884, "p90_latency_all": 20.715546749997884, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0744698, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 27.74318225006573, "p90_latency": 27.74318225006573, "median_latency_all": 27.74318225006573, "p90_latency_all": 27.74318225006573, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0766676, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 7.027635500067845, "verdict": "worse" }, { "id": 114, "question": "Are there differences in chromosome arm 7p copy number between lower grade glioma molecular subtypes?", "category": "Expression & multi-omics", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 59.78105595917441, "p90_latency": 59.78105595917441, "median_latency_all": 59.78105595917441, "p90_latency_all": 59.78105595917441, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.21506679999999997, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 118.68866045796312, "p90_latency": 118.68866045796312, "median_latency_all": 118.68866045796312, "p90_latency_all": 118.68866045796312, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 20, "median_llm_calls": 20, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.0, "cost_per_answer": 0.3647904, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 58.907604498788714, "verdict": "worse" }, { "id": 123, "question": "In TCGA lower grade glioma, show me samples that are TP53 mutant or EGFR amplified.", "category": "Patient & sample lookup", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 34.15706262481399, "p90_latency": 34.15706262481399, "median_latency_all": 34.15706262481399, "p90_latency_all": 34.15706262481399, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.11111111111111, "cost_per_answer": 0.2199154, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 37.600696375011466, "p90_latency": 37.600696375011466, "median_latency_all": 37.600696375011466, "p90_latency_all": 37.600696375011466, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1353327, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 3.4436337501974776, "verdict": "worse" }, { "id": 141, "question": "is there a relatinoship between mgmt methylation and idh1 mutation in glioma?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 55.892225874820724, "p90_latency": 55.892225874820724, "median_latency_all": 55.892225874820724, "p90_latency_all": 55.892225874820724, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1764103, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 182.68183370796032, "p90_latency": 182.68183370796032, "median_latency_all": 182.68183370796032, "p90_latency_all": 182.68183370796032, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 16, "median_llm_calls": 16, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 3.225806451612903, "cost_per_answer": 0.32902770000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": 126.7896078331396, "verdict": "worse" }, { "id": 145, "question": "Which genes are enriched for mutations between NSCLC vs squamous cell carcinoma?", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 74.94556804210879, "p90_latency": 74.94556804210879, "median_latency_all": 74.94556804210879, "p90_latency_all": 74.94556804210879, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10, "median_llm_calls": 10, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.30158840000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 11.791111832950264, "p90_latency": 11.791111832950264, "median_latency_all": 11.791111832950264, "p90_latency_all": 11.791111832950264, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0432226, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": -100.0, "latency_delta": -63.154456209158525, "verdict": "worse" }, { "id": 3, "question": "How many patients and samples are in the MSK-CHORD Study?", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 14.263619208941236, "p90_latency": 14.263619208941236, "median_latency_all": 14.263619208941236, "p90_latency_all": 14.263619208941236, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0590051, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 7.998848583898507, "p90_latency": 7.998848583898507, "median_latency_all": 7.998848583898507, "p90_latency_all": 7.998848583898507, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0256229, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -6.264770625042729, "verdict": "better" }, { "id": 7, "question": "What are the top 5 most frequently copy number altered genes in the Osteosarcoma study from TARGET?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 32.63976937509142, "p90_latency": 32.63976937509142, "median_latency_all": 32.63976937509142, "p90_latency_all": 32.63976937509142, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1716377, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 33.56884737499058, "p90_latency": 33.56884737499058, "median_latency_all": 33.56884737499058, "p90_latency_all": 33.56884737499058, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.10368970000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 0.9290779998991638, "verdict": "better" }, { "id": 16, "question": "How many unique patients have both primary and metastatic samples in the MSK-CHORD Study?", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 16.970612624892965, "p90_latency": 16.970612624892965, "median_latency_all": 16.970612624892965, "p90_latency_all": 16.970612624892965, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06666860000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 24.208218083018437, "p90_latency": 24.208218083018437, "median_latency_all": 24.208218083018437, "p90_latency_all": 24.208218083018437, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.07806439999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 7.237605458125472, "verdict": "better" }, { "id": 17, "question": "What are the top 5 most common primary diagnosis sites in the MSK-CHORD Study?", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 12.684825165895745, "p90_latency": 12.684825165895745, "median_latency_all": 12.684825165895745, "p90_latency_all": 12.684825165895745, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0761518, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 20.4267931249924, "p90_latency": 20.4267931249924, "median_latency_all": 20.4267931249924, "p90_latency_all": 20.4267931249924, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0896729, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 7.741967959096655, "verdict": "better" }, { "id": 21, "question": "What are the most commonly co-occurring mutation pairs in breast cancer samples from the MSK-CHORD Study?", "category": "Co-occurrence & exclusivity", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 26.91213516611606, "p90_latency": 26.91213516611606, "median_latency_all": 26.91213516611606, "p90_latency_all": 26.91213516611606, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1533609, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 25.421190665918402, "p90_latency": 25.421190665918402, "median_latency_all": 25.421190665918402, "p90_latency_all": 25.421190665918402, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1608107, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -1.4909445001976565, "verdict": "better" }, { "id": 40, "question": "What are the most frequently altered genes in KRAS wild-type lung adenocarcinoma patients?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 31.588816583156586, "p90_latency": 31.588816583156586, "median_latency_all": 31.588816583156586, "p90_latency_all": 31.588816583156586, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.157437, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 22.10863725002855, "p90_latency": 22.10863725002855, "median_latency_all": 22.10863725002855, "p90_latency_all": 22.10863725002855, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.11289039999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -9.480179333128035, "verdict": "better" }, { "id": 49, "question": "Which cancer types have the highest aneuploidy scores and how does this correlate with mutation burden across TCGA Pan-Cancer studies?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 44.11899379105307, "p90_latency": 44.11899379105307, "median_latency_all": 44.11899379105307, "p90_latency_all": 44.11899379105307, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.21303129999999998, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 64.44293112505693, "p90_latency": 64.44293112505693, "median_latency_all": 64.44293112505693, "p90_latency_all": 64.44293112505693, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 12, "median_llm_calls": 12, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 15.384615384615385, "cost_per_answer": 0.2538098, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 20.32393733400386, "verdict": "better" }, { "id": 51, "question": "What fraction of patients were older than five when diagnosed according to the Pediatric Neuroblastoma study from TARGET?", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 23.562223999993876, "p90_latency": 23.562223999993876, "median_latency_all": 23.562223999993876, "p90_latency_all": 23.562223999993876, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.10421860000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 20.776910333079286, "p90_latency": 20.776910333079286, "median_latency_all": 20.776910333079286, "p90_latency_all": 20.776910333079286, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0954548, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -2.7853136669145897, "verdict": "better" }, { "id": 52, "question": "What is the most frequent mutation in the TP53 gene in the TCGA breast cancer study?", "category": "Variants & hotspots", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 16.796706709079444, "p90_latency": 16.796706709079444, "median_latency_all": 16.796706709079444, "p90_latency_all": 16.796706709079444, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1054695, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 12.336166541907005, "p90_latency": 12.336166541907005, "median_latency_all": 12.336166541907005, "p90_latency_all": 12.336166541907005, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0945345, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -4.460540167172439, "verdict": "better" }, { "id": 53, "question": "How many patients have an EGFR amplification in the TCGA Lung Adenocarcinoma study?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 30.00464754109271, "p90_latency": 30.00464754109271, "median_latency_all": 30.00464754109271, "p90_latency_all": 30.00464754109271, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.13734570000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 9.893151374999434, "p90_latency": 9.893151374999434, "median_latency_all": 9.893151374999434, "p90_latency_all": 9.893151374999434, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0365176, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -20.111496166093275, "verdict": "better" }, { "id": 54, "question": "Which KRAS mutations are most common in colorectal cancer?", "category": "Variants & hotspots", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 21.32978079095483, "p90_latency": 21.32978079095483, "median_latency_all": 21.32978079095483, "p90_latency_all": 21.32978079095483, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1248591, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 16.324098207987845, "p90_latency": 16.324098207987845, "median_latency_all": 16.324098207987845, "p90_latency_all": 16.324098207987845, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1104398, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -5.005682582966983, "verdict": "better" }, { "id": 55, "question": "What is the median survival time in the Pediatric Neuroblastoma study from TARGET?", "category": "Survival & outcomes", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 19.174211916979402, "p90_latency": 19.174211916979402, "median_latency_all": 19.174211916979402, "p90_latency_all": 19.174211916979402, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0694351, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 21.597525499993935, "p90_latency": 21.597525499993935, "median_latency_all": 21.597525499993935, "p90_latency_all": 21.597525499993935, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1035965, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 2.423313583014533, "verdict": "better" }, { "id": 60, "question": "In the Breast Invasive Carcinoma TCGA study what are the top 5 down-regulated genes in TP53 mutated samples compared to non-mutated ones?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 0, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": null, "attempt_rate": 0.0, "pass_rate": 0.0, "median_latency": 27.542138167191297, "p90_latency": 27.542138167191297, "median_latency_all": 27.542138167191297, "p90_latency_all": 27.542138167191297, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 16.666666666666668, "cost_per_answer": 0.0825608, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "–" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 47.792417417047545, "p90_latency": 47.792417417047545, "median_latency_all": 47.792417417047545, "p90_latency_all": 47.792417417047545, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.0, "cost_per_answer": 0.1391189, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 20.25027924985625, "verdict": "better" }, { "id": 65, "question": "In the TCGA PanCancer Atlas breast cancer study, what is the average tumor mutational burden of patients who have at least two of the five most frequently mutated genes and how does this compare to the average TMB of all patients?", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 44.36443112487905, "p90_latency": 44.36443112487905, "median_latency_all": 44.36443112487905, "p90_latency_all": 44.36443112487905, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.0, "cost_per_answer": 0.2114305, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 27.485894166049547, "p90_latency": 27.485894166049547, "median_latency_all": 27.485894166049547, "p90_latency_all": 27.485894166049547, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1741482, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -16.8785369588295, "verdict": "better" }, { "id": 73, "question": "What is the frequency of TP53 mutations in lung cancer?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 28.74757729214616, "p90_latency": 28.74757729214616, "median_latency_all": 28.74757729214616, "p90_latency_all": 28.74757729214616, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1308555, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 20.965619915979914, "p90_latency": 20.965619915979914, "median_latency_all": 20.965619915979914, "p90_latency_all": 20.965619915979914, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0678176, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -7.781957376166247, "verdict": "better" }, { "id": 76, "question": "Are TP53 mutations clinically actionable?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 9.636845583096147, "p90_latency": 9.636845583096147, "median_latency_all": 9.636845583096147, "p90_latency_all": 9.636845583096147, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.0138666, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 28.457912041107193, "p90_latency": 28.457912041107193, "median_latency_all": 28.457912041107193, "p90_latency_all": 28.457912041107193, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.095145, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 18.821066458011046, "verdict": "better" }, { "id": 81, "question": "tell me about IDH1 mutations in glioma", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 30.206028040964156, "p90_latency": 30.206028040964156, "median_latency_all": 30.206028040964156, "p90_latency_all": 30.206028040964156, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.15478429999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 29.291254916926846, "p90_latency": 29.291254916926846, "median_latency_all": 29.291254916926846, "p90_latency_all": 29.291254916926846, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.14498629999999998, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -0.9147731240373105, "verdict": "better" }, { "id": 83, "question": "Help me rank TCGA cancer studies based on median CD3 expression", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 52.24008141597733, "p90_latency": 52.24008141597733, "median_latency_all": 52.24008141597733, "p90_latency_all": 52.24008141597733, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 20.0, "cost_per_answer": 0.13610029999999998, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 41.485706542036496, "p90_latency": 41.485706542036496, "median_latency_all": 41.485706542036496, "p90_latency_all": 41.485706542036496, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1098064, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -10.754374873940833, "verdict": "better" }, { "id": 92, "question": "what is the most prevalent TP53 mutation in uterine cancer that is not a point mutation", "category": "Variants & hotspots", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 29.333154125139117, "p90_latency": 29.333154125139117, "median_latency_all": 29.333154125139117, "p90_latency_all": 29.333154125139117, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1460986, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 25.35702533298172, "p90_latency": 25.35702533298172, "median_latency_all": 25.35702533298172, "p90_latency_all": 25.35702533298172, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 12.5, "cost_per_answer": 0.1705254, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -3.9761287921573967, "verdict": "better" }, { "id": 97, "question": "give me a contingency table with the number of lung cancer patients with EGFR and/or KRAS alterations", "category": "Co-occurrence & exclusivity", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 24.238359542097896, "p90_latency": 24.238359542097896, "median_latency_all": 24.238359542097896, "p90_latency_all": 24.238359542097896, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1360086, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 46.91851979098283, "p90_latency": 46.91851979098283, "median_latency_all": 46.91851979098283, "p90_latency_all": 46.91851979098283, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.18642, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 22.68016024888493, "verdict": "better" }, { "id": 106, "question": "Which studies have RNA expression for renal cancer?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 31.82874875003472, "p90_latency": 31.82874875003472, "median_latency_all": 31.82874875003472, "p90_latency_all": 31.82874875003472, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1554142, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 20.008879625005648, "p90_latency": 20.008879625005648, "median_latency_all": 20.008879625005648, "p90_latency_all": 20.008879625005648, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0618393, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -11.819869125029072, "verdict": "better" }, { "id": 109, "question": "I receive a report with the PIK3CA p.*1069Wext*3 mutation. Can you provide details of it", "category": "Variants & hotspots", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 42.83837116602808, "p90_latency": 42.83837116602808, "median_latency_all": 42.83837116602808, "p90_latency_all": 42.83837116602808, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.20671240000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 32.33215775003191, "p90_latency": 32.33215775003191, "median_latency_all": 32.33215775003191, "p90_latency_all": 32.33215775003191, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.18923520000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -10.506213415996172, "verdict": "better" }, { "id": 116, "question": "What are the frequencies of different KRAS mutations in TCGA PanCan Lung Adenocarcinoma?", "category": "Variants & hotspots", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 15.730199332814664, "p90_latency": 15.730199332814664, "median_latency_all": 15.730199332814664, "p90_latency_all": 15.730199332814664, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.08114489999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 21.761459707980976, "p90_latency": 21.761459707980976, "median_latency_all": 21.761459707980976, "p90_latency_all": 21.761459707980976, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.10764910000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 6.031260375166312, "verdict": "better" }, { "id": 117, "question": "What cancer types in MSK-IMPACT have mutations in TP53?", "category": "Alteration frequency", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 25.30981304193847, "p90_latency": 25.30981304193847, "median_latency_all": 25.30981304193847, "p90_latency_all": 25.30981304193847, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.10896310000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 25.217266499996185, "p90_latency": 25.217266499996185, "median_latency_all": 25.217266499996185, "p90_latency_all": 25.217266499996185, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1297462, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -0.09254654194228351, "verdict": "better" }, { "id": 135, "question": "How does overall survival compare between prostate cancer patients where TMPRSS2 is acting as the upstream donor to an ERG fusion event and patients without this fusion?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 87.25240491703153, "p90_latency": 87.25240491703153, "median_latency_all": 87.25240491703153, "p90_latency_all": 87.25240491703153, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 17, "median_llm_calls": 17, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.545454545454546, "cost_per_answer": 0.34630940000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 121.54425916704349, "p90_latency": 121.54425916704349, "median_latency_all": 121.54425916704349, "p90_latency_all": 121.54425916704349, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 14, "median_llm_calls": 14, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.2631578947368425, "cost_per_answer": 0.4572877, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": 34.291854250011966, "verdict": "better" }, { "id": 143, "question": "show me P135L mutations in $p14^{ARF}$", "category": "Variants & hotspots", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 26.4499387498945, "p90_latency": 26.4499387498945, "median_latency_all": 26.4499387498945, "p90_latency_all": 26.4499387498945, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1037259, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 21.92004054086283, "p90_latency": 21.92004054086283, "median_latency_all": 21.92004054086283, "p90_latency_all": 21.92004054086283, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0907957, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 100.0, "latency_delta": -4.529898209031671, "verdict": "better" }, { "id": 1, "question": "How many studies are in cBioPortal?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 6.350463832961395, "p90_latency": 6.350463832961395, "median_latency_all": 6.350463832961395, "p90_latency_all": 6.350463832961395, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0690616, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 5.773517790948972, "p90_latency": 5.773517790948972, "median_latency_all": 5.773517790948972, "p90_latency_all": 5.773517790948972, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0731345, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -0.5769460420124233, "verdict": "same" }, { "id": 2, "question": "How many glioblastoma studies are in cBioPortal?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 13.514563292032108, "p90_latency": 13.514563292032108, "median_latency_all": 13.514563292032108, "p90_latency_all": 13.514563292032108, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0390584, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 11.535229915985838, "p90_latency": 11.535229915985838, "median_latency_all": 11.535229915985838, "p90_latency_all": 11.535229915985838, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.027692300000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -1.9793333760462701, "verdict": "same" }, { "id": 4, "question": "How many primary samples are in the MSK-CHORD Study?", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 14.676055083051324, "p90_latency": 14.676055083051324, "median_latency_all": 14.676055083051324, "p90_latency_all": 14.676055083051324, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06102110000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 8.223270458052866, "p90_latency": 8.223270458052866, "median_latency_all": 8.223270458052866, "p90_latency_all": 8.223270458052866, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0391784, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -6.452784624998458, "verdict": "same" }, { "id": 5, "question": "What treatment did most patients receive in the MSK-CHORD Study?", "category": "Treatment", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 14.940764999948442, "p90_latency": 14.940764999948442, "median_latency_all": 14.940764999948442, "p90_latency_all": 14.940764999948442, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.055306, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 13.413281915942207, "p90_latency": 13.413281915942207, "median_latency_all": 13.413281915942207, "p90_latency_all": 13.413281915942207, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0408683, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -1.527483084006235, "verdict": "same" }, { "id": 6, "question": "What are the top 5 most frequently mutated genes in the Osteosarcoma study from TARGET?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 16.125861833803356, "p90_latency": 16.125861833803356, "median_latency_all": 16.125861833803356, "p90_latency_all": 16.125861833803356, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1002285, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 15.048237917013466, "p90_latency": 15.048237917013466, "median_latency_all": 15.048237917013466, "p90_latency_all": 15.048237917013466, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.049843900000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -1.0776239167898893, "verdict": "same" }, { "id": 8, "question": "What are the top 5 most frequently altered genes in a structural variant in the Osteosarcoma study from TARGET?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 20.551563875051215, "p90_latency": 20.551563875051215, "median_latency_all": 20.551563875051215, "p90_latency_all": 20.551563875051215, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1278898, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 10.363338000024669, "p90_latency": 10.363338000024669, "median_latency_all": 10.363338000024669, "p90_latency_all": 10.363338000024669, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0283664, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -10.188225875026546, "verdict": "same" }, { "id": 9, "question": "\"What are the top 5 frequently altered genes in the Osteosarcoma study from TARGET for mutations, copy numbers and SVs combined?\"", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 47.248311541974545, "p90_latency": 47.248311541974545, "median_latency_all": 47.248311541974545, "p90_latency_all": 47.248311541974545, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1865677, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 17.706973375054076, "p90_latency": 17.706973375054076, "median_latency_all": 17.706973375054076, "p90_latency_all": 17.706973375054076, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0531487, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -29.541338166920468, "verdict": "same" }, { "id": 10, "question": "What is the median age at diagnosis for osteosarcoma patients in the TARGET study?", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 18.650896166916937, "p90_latency": 18.650896166916937, "median_latency_all": 18.650896166916937, "p90_latency_all": 18.650896166916937, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0904648, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 18.88086791697424, "p90_latency": 18.88086791697424, "median_latency_all": 18.88086791697424, "p90_latency_all": 18.88086791697424, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0824934, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 0.22997175005730242, "verdict": "same" }, { "id": 11, "question": "What are the top 5 most frequently mutated genes in the MSK-CHORD Study?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 16.198550124885514, "p90_latency": 16.198550124885514, "median_latency_all": 16.198550124885514, "p90_latency_all": 16.198550124885514, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.10157420000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 13.454406750039198, "p90_latency": 13.454406750039198, "median_latency_all": 13.454406750039198, "p90_latency_all": 13.454406750039198, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0380512, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -2.744143374846317, "verdict": "same" }, { "id": 12, "question": "What is the most common cancer type in the MSK-CHORD Study based on sample count?", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 14.114376333076507, "p90_latency": 14.114376333076507, "median_latency_all": 14.114376333076507, "p90_latency_all": 14.114376333076507, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0721407, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 11.137865125085227, "p90_latency": 11.137865125085227, "median_latency_all": 11.137865125085227, "p90_latency_all": 11.137865125085227, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0411428, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -2.9765112079912797, "verdict": "same" }, { "id": 14, "question": "Which cancer type has the highest average tumor mutational burden across all studies?", "category": "Cohort & clinical counts", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 26.237603958928958, "p90_latency": 26.237603958928958, "median_latency_all": 26.237603958928958, "p90_latency_all": 26.237603958928958, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.09379169999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 19.592461333959363, "p90_latency": 19.592461333959363, "median_latency_all": 19.592461333959363, "p90_latency_all": 19.592461333959363, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0665145, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -6.645142624969594, "verdict": "same" }, { "id": 15, "question": "How many total studies contain mutation data in the cBioPortal database?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 8.140764540992677, "p90_latency": 8.140764540992677, "median_latency_all": 8.140764540992677, "p90_latency_all": 8.140764540992677, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0252082, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 17.357679083012044, "p90_latency": 17.357679083012044, "median_latency_all": 17.357679083012044, "p90_latency_all": 17.357679083012044, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0350877, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 9.216914542019367, "verdict": "same" }, { "id": 18, "question": "What are the most frequently administered systemic therapy regimens for lung cancer patients in the MSK-CHORD Study?", "category": "Treatment", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 30.858923375140876, "p90_latency": 30.858923375140876, "median_latency_all": 30.858923375140876, "p90_latency_all": 30.858923375140876, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1107044, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 64.14496329100803, "p90_latency": 64.14496329100803, "median_latency_all": 64.14496329100803, "p90_latency_all": 64.14496329100803, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10, "median_llm_calls": 10, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 14.285714285714286, "cost_per_answer": 0.1709196, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 33.28603991586715, "verdict": "same" }, { "id": 19, "question": "What are the top 10 most frequently mutated genes across all cancer types in the MSK-CHORD Study?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 18.36835829098709, "p90_latency": 18.36835829098709, "median_latency_all": 18.36835829098709, "p90_latency_all": 18.36835829098709, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.10488220000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 14.8475469580153, "p90_latency": 14.8475469580153, "median_latency_all": 14.8475469580153, "p90_latency_all": 14.8475469580153, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0353485, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -3.520811332971789, "verdict": "same" }, { "id": 22, "question": "Do patients with PIK3CA mutations have different overall survival outcomes compared to PIK3CA wild-type patients in breast cancer from the MSK-CHORD Study?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 51.63170558284037, "p90_latency": 51.63170558284037, "median_latency_all": 51.63170558284037, "p90_latency_all": 51.63170558284037, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10, "median_llm_calls": 10, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 8.333333333333334, "cost_per_answer": 0.24775490000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 47.87033641699236, "p90_latency": 47.87033641699236, "median_latency_all": 47.87033641699236, "p90_latency_all": 47.87033641699236, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.236436, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -3.7613691658480093, "verdict": "same" }, { "id": 23, "question": "What is the correlation between tumor mutational burden and microsatellite instability status in colorectal cancer patients from the MSK-CHORD Study?", "category": "Cohort & clinical counts", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 40.77896741707809, "p90_latency": 40.77896741707809, "median_latency_all": 40.77896741707809, "p90_latency_all": 40.77896741707809, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.0, "cost_per_answer": 0.1601292, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 33.550132083008066, "p90_latency": 33.550132083008066, "median_latency_all": 33.550132083008066, "p90_latency_all": 33.550132083008066, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.140465, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -7.228835334070027, "verdict": "same" }, { "id": 24, "question": "Which genomic alterations are associated with immunotherapy response in melanoma patients from the MSK-CHORD Study?", "category": "Treatment", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 17.774771082913503, "p90_latency": 17.774771082913503, "median_latency_all": 17.774771082913503, "p90_latency_all": 17.774771082913503, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.09398619999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 18.326428584055975, "p90_latency": 18.326428584055975, "median_latency_all": 18.326428584055975, "p90_latency_all": 18.326428584055975, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0648612, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 0.551657501142472, "verdict": "same" }, { "id": 25, "question": "How does the mutation landscape differ between primary and metastatic samples from the same patients in the MSK-CHORD Study?", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 45.96724012494087, "p90_latency": 45.96724012494087, "median_latency_all": 45.96724012494087, "p90_latency_all": 45.96724012494087, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.2063863, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 66.91111895896029, "p90_latency": 66.91111895896029, "median_latency_all": 66.91111895896029, "p90_latency_all": 66.91111895896029, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10, "median_llm_calls": 10, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.11111111111111, "cost_per_answer": 0.2719769, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 20.943878834019415, "verdict": "same" }, { "id": 26, "question": "What percentage of genomic events in the MSK-CHORD Study occur in genes that are off-panel (not covered by the sequencing panel used)?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 28.661139333853498, "p90_latency": 28.661139333853498, "median_latency_all": 28.661139333853498, "p90_latency_all": 28.661139333853498, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.167219, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 19.1623243750073, "p90_latency": 19.1623243750073, "median_latency_all": 19.1623243750073, "p90_latency_all": 19.1623243750073, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.14198170000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -9.498814958846197, "verdict": "same" }, { "id": 27, "question": "Is there a correlation between ERBB2 gene amplification and ERBB2 protein expression levels in breast cancer samples from the MSK-CHORD Study?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 69.19686841708608, "p90_latency": 69.19686841708608, "median_latency_all": 69.19686841708608, "p90_latency_all": 69.19686841708608, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 6.666666666666667, "cost_per_answer": 0.1775384, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 32.90088495798409, "p90_latency": 32.90088495798409, "median_latency_all": 32.90088495798409, "p90_latency_all": 32.90088495798409, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0994247, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -36.29598345910199, "verdict": "same" }, { "id": 28, "question": "\"What percentage of glioblastoma patients have alterations in RB pathway genes (CDKN2A, CDK4, RB1)?\"", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 39.48733479203656, "p90_latency": 39.48733479203656, "median_latency_all": 39.48733479203656, "p90_latency_all": 39.48733479203656, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 22.22222222222222, "cost_per_answer": 0.15365779999999998, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 28.126059249974787, "p90_latency": 28.126059249974787, "median_latency_all": 28.126059249974787, "p90_latency_all": 28.126059249974787, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1256069, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -11.361275542061776, "verdict": "same" }, { "id": 29, "question": "\"Are mutations in CDKN2A, CDK4, and RB1 mutually exclusive in glioblastoma patients?\"", "category": "Co-occurrence & exclusivity", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 37.9576602911111, "p90_latency": 37.9576602911111, "median_latency_all": 37.9576602911111, "p90_latency_all": 37.9576602911111, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.12236739999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 49.84887670807075, "p90_latency": 49.84887670807075, "median_latency_all": 49.84887670807075, "p90_latency_all": 49.84887670807075, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 12.5, "cost_per_answer": 0.1570198, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 11.89121641695965, "verdict": "same" }, { "id": 30, "question": "In the TCGA PanCancer Atlas glioblastoma study, is CDK4 mRNA expression significantly higher in samples with CDK4 amplification compared to diploid samples?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 31.09479545801878, "p90_latency": 31.09479545801878, "median_latency_all": 31.09479545801878, "p90_latency_all": 31.09479545801878, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.11111111111111, "cost_per_answer": 0.11267170000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 38.13484112499282, "p90_latency": 38.13484112499282, "median_latency_all": 38.13484112499282, "p90_latency_all": 38.13484112499282, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 12.5, "cost_per_answer": 0.12432720000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 7.040045666974038, "verdict": "same" }, { "id": 31, "question": "Which cancer types have the highest frequency of EGFR mutations across all TCGA Pan-Cancer Atlas studies?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 0, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": null, "attempt_rate": 0.0, "pass_rate": 0.0, "median_latency": 17.13376612518914, "p90_latency": 17.13376612518914, "median_latency_all": 17.13376612518914, "p90_latency_all": 17.13376612518914, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.10474080000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "–" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 26.096062208060175, "p90_latency": 26.096062208060175, "median_latency_all": 26.096062208060175, "p90_latency_all": 26.096062208060175, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06168510000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 8.962296082871035, "verdict": "same" }, { "id": 32, "question": "What are the most frequent EGFR mutation variants in lung adenocarcinoma and what percentage are known hotspot mutations?", "category": "Variants & hotspots", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 25.226229041116312, "p90_latency": 25.226229041116312, "median_latency_all": 25.226229041116312, "p90_latency_all": 25.226229041116312, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.0, "cost_per_answer": 0.1315634, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 22.943479540990666, "p90_latency": 22.943479540990666, "median_latency_all": 22.943479540990666, "p90_latency_all": 22.943479540990666, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1093008, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -2.2827495001256466, "verdict": "same" }, { "id": 33, "question": "What is the correlation coefficient between EGFR copy number and EGFR mRNA expression in ovarian cancer?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 31.835368875181302, "p90_latency": 31.835368875181302, "median_latency_all": 31.835368875181302, "p90_latency_all": 31.835368875181302, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0981374, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 33.549347958061844, "p90_latency": 33.549347958061844, "median_latency_all": 33.549347958061844, "p90_latency_all": 33.549347958061844, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.11111111111111, "cost_per_answer": 0.11432270000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 1.7139790828805417, "verdict": "same" }, { "id": 34, "question": "In the TCGA PanCancer Atlas ovarian cancer study, what is the correlation between EGFR mRNA expression and EGFR protein (RPPA) levels?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 28.86165020801127, "p90_latency": 28.86165020801127, "median_latency_all": 28.86165020801127, "p90_latency_all": 28.86165020801127, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1010155, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 22.808635958004743, "p90_latency": 22.808635958004743, "median_latency_all": 22.808635958004743, "p90_latency_all": 22.808635958004743, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0768869, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -6.053014250006527, "verdict": "same" }, { "id": 35, "question": "In the TCGA PanCancer Atlas ovarian cancer study, do samples with TP53 truncating mutations have significantly lower TP53 mRNA expression compared to wild-type samples?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 53.24810254108161, "p90_latency": 53.24810254108161, "median_latency_all": 53.24810254108161, "p90_latency_all": 53.24810254108161, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.2824833, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 37.40148145798594, "p90_latency": 37.40148145798594, "median_latency_all": 37.40148145798594, "p90_latency_all": 37.40148145798594, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.0, "cost_per_answer": 0.131709, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -15.84662108309567, "verdict": "same" }, { "id": 36, "question": "Is BRCA1 promoter methylation associated with decreased BRCA1 mRNA expression in ovarian cancer?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 47.20435416721739, "p90_latency": 47.20435416721739, "median_latency_all": 47.20435416721739, "p90_latency_all": 47.20435416721739, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10, "median_llm_calls": 10, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.6923076923076925, "cost_per_answer": 0.15143379999999998, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 36.822616791934706, "p90_latency": 36.822616791934706, "median_latency_all": 36.822616791934706, "p90_latency_all": 36.822616791934706, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.12555539999999998, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -10.381737375282682, "verdict": "same" }, { "id": 37, "question": "Do ovarian cancer patients with BRCA1 or BRCA2 alterations have significantly different overall survival compared to wild-type patients?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 31.6062656249851, "p90_latency": 31.6062656249851, "median_latency_all": 31.6062656249851, "p90_latency_all": 31.6062656249851, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1285216, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 49.20349841704592, "p90_latency": 49.20349841704592, "median_latency_all": 49.20349841704592, "p90_latency_all": 49.20349841704592, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 8.333333333333334, "cost_per_answer": 0.1887705, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 17.597232792060822, "verdict": "same" }, { "id": 38, "question": "Do lung adenocarcinoma patients with high EGFR mRNA expression (top quartile) have different survival outcomes than those with low expression?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 34.92161716707051, "p90_latency": 34.92161716707051, "median_latency_all": 34.92161716707051, "p90_latency_all": 34.92161716707051, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1346255, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 34.73698745900765, "p90_latency": 34.73698745900765, "median_latency_all": 34.73698745900765, "p90_latency_all": 34.73698745900765, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 9.090909090909092, "cost_per_answer": 0.15267129999999998, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -0.1846297080628574, "verdict": "same" }, { "id": 39, "question": "\"What are the survival differences between EGFR-mutated, EGFR-amplified, and EGFR wild-type lung adenocarcinoma patients?\"", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 51.58470970788039, "p90_latency": 51.58470970788039, "median_latency_all": 51.58470970788039, "p90_latency_all": 51.58470970788039, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.18711270000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 56.83781029097736, "p90_latency": 56.83781029097736, "median_latency_all": 56.83781029097736, "p90_latency_all": 56.83781029097736, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10, "median_llm_calls": 10, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.20735810000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 5.253100583096966, "verdict": "same" }, { "id": 41, "question": "How does PTEN alteration (mutations or homozygous deletions) affect pAKT protein levels in lung squamous cell carcinoma?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 57.96939908294007, "p90_latency": 57.96939908294007, "median_latency_all": 57.96939908294007, "p90_latency_all": 57.96939908294007, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1439226, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 74.23893929098267, "p90_latency": 74.23893929098267, "median_latency_all": 74.23893929098267, "p90_latency_all": 74.23893929098267, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1720867, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 16.2695402080426, "verdict": "same" }, { "id": 43, "question": "In the TCGA PanCancer Atlas endometrial cancer study, in samples with both KRAS and NRAS mutations, what is the distribution of variant allele frequencies suggesting clonal vs subclonal events?", "category": "Variants & hotspots", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 44.55023037502542, "p90_latency": 44.55023037502542, "median_latency_all": 44.55023037502542, "p90_latency_all": 44.55023037502542, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.2024729, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 42.36159341596067, "p90_latency": 42.36159341596067, "median_latency_all": 42.36159341596067, "p90_latency_all": 42.36159341596067, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1903247, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -2.188636959064752, "verdict": "same" }, { "id": 44, "question": "What percentage of endometrial cancer samples have hypermutation (>5000 mutations) and how does this correlate with histological subtype?", "category": "Cohort & clinical counts", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 42.23939470900223, "p90_latency": 42.23939470900223, "median_latency_all": 42.23939470900223, "p90_latency_all": 42.23939470900223, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.142857142857143, "cost_per_answer": 0.22022690000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 58.92000625003129, "p90_latency": 58.92000625003129, "median_latency_all": 58.92000625003129, "p90_latency_all": 58.92000625003129, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10, "median_llm_calls": 10, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.882352941176471, "cost_per_answer": 0.22655779999999998, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 16.680611541029066, "verdict": "same" }, { "id": 45, "question": "What are the most frequently mutated genes in copy-number high subtype endometrial cancers compared to other subtypes?", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 45.89930370799266, "p90_latency": 45.89930370799266, "median_latency_all": 45.89930370799266, "p90_latency_all": 45.89930370799266, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 9.090909090909092, "cost_per_answer": 0.2005385, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 46.3813285829965, "p90_latency": 46.3813285829965, "median_latency_all": 46.3813285829965, "p90_latency_all": 46.3813285829965, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.2203895, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 0.4820248750038445, "verdict": "same" }, { "id": 46, "question": "Which cancer types show the highest frequency of BRAF V600E mutations across all TCGA Pan-Cancer Atlas studies?", "category": "Variants & hotspots", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 26.947592166950926, "p90_latency": 26.947592166950926, "median_latency_all": 26.947592166950926, "p90_latency_all": 26.947592166950926, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.12709410000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 24.223567791981623, "p90_latency": 24.223567791981623, "median_latency_all": 24.223567791981623, "p90_latency_all": 24.223567791981623, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1271504, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -2.7240243749693036, "verdict": "same" }, { "id": 47, "question": "How does ERBB2 mRNA expression vary across different cancer types in TCGA Pan-Cancer Atlas studies?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 38.060411375015974, "p90_latency": 38.060411375015974, "median_latency_all": 38.060411375015974, "p90_latency_all": 38.060411375015974, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1128841, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 44.591968999942765, "p90_latency": 44.591968999942765, "median_latency_all": 44.591968999942765, "p90_latency_all": 44.591968999942765, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 16.666666666666668, "cost_per_answer": 0.129436, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 6.531557624926791, "verdict": "same" }, { "id": 48, "question": "In the TCGA PanCancer Atlas breast cancer study, what is the concordance between ERBB2 copy number amplification, mRNA overexpression, and protein overexpression?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 53.296917957952246, "p90_latency": 53.296917957952246, "median_latency_all": 53.296917957952246, "p90_latency_all": 53.296917957952246, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 9.090909090909092, "cost_per_answer": 0.1466084, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 43.61860191693995, "p90_latency": 43.61860191693995, "median_latency_all": 43.61860191693995, "p90_latency_all": 43.61860191693995, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.12363770000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -9.678316041012295, "verdict": "same" }, { "id": 50, "question": "\"Are mutations in DNA repair pathway genes (BRCA1, BRCA2, ATM, CHEK2) enriched in specific cancer types across TCGA Pan-Cancer Atlas?\"", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 41.446017417125404, "p90_latency": 41.446017417125404, "median_latency_all": 41.446017417125404, "p90_latency_all": 41.446017417125404, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.21118, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 37.37168824998662, "p90_latency": 37.37168824998662, "median_latency_all": 37.37168824998662, "p90_latency_all": 37.37168824998662, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1512091, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -4.074329167138785, "verdict": "same" }, { "id": 56, "question": "In the TCGA glioblastoma study (Cell 2013), how does methylation of the MGMT gene promoter affect the prognosis and treatment response in patients with glioblastoma?", "category": "Treatment", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 49.20455591706559, "p90_latency": 49.20455591706559, "median_latency_all": 49.20455591706559, "p90_latency_all": 49.20455591706559, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.142857142857143, "cost_per_answer": 0.1750422, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 44.50876020803116, "p90_latency": 44.50876020803116, "median_latency_all": 44.50876020803116, "p90_latency_all": 44.50876020803116, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 8.333333333333334, "cost_per_answer": 0.1701586, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -4.695795709034428, "verdict": "same" }, { "id": 57, "question": "For patients with the EML4-ALK fusion gene in lung cancer how do the different fusion variants affect their long-term quality of life and risk of developing a secondary cancer?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 10.656136750010774, "p90_latency": 10.656136750010774, "median_latency_all": 10.656136750010774, "p90_latency_all": 10.656136750010774, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.0152266, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 10.926818499923684, "p90_latency": 10.926818499923684, "median_latency_all": 10.926818499923684, "p90_latency_all": 10.926818499923684, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.0157306, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 0.27068174991291016, "verdict": "same" }, { "id": 58, "question": "In the “Pan-cancer analysis of whole genomes” study what is the survival difference and the corresponding statistical significance between patients with mutations in both TP53 and KRAS versus patients with only a KRAS mutation?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 55.75963683286682, "p90_latency": 55.75963683286682, "median_latency_all": 55.75963683286682, "p90_latency_all": 55.75963683286682, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 10, "median_llm_calls": 10, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 8.333333333333334, "cost_per_answer": 0.23830020000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 50.695346541004255, "p90_latency": 50.695346541004255, "median_latency_all": 50.695346541004255, "p90_latency_all": 50.695346541004255, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1796914, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -5.064290291862562, "verdict": "same" }, { "id": 59, "question": "Are there studies that were not processed using polyA enrichment in order to explore lncRNA-related questions?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 27.122881249990314, "p90_latency": 27.122881249990314, "median_latency_all": 27.122881249990314, "p90_latency_all": 27.122881249990314, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 16.666666666666668, "cost_per_answer": 0.08886680000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 17.24265991698485, "p90_latency": 17.24265991698485, "median_latency_all": 17.24265991698485, "p90_latency_all": 17.24265991698485, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0506027, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -9.880221333005466, "verdict": "same" }, { "id": 61, "question": "In the Pediatric Neuroblastoma study from TARGET what is the survival difference and the corresponding statistical significance between patients who were older than four when diagnosed and the younger ones?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 41.60651558288373, "p90_latency": 41.60651558288373, "median_latency_all": 41.60651558288373, "p90_latency_all": 41.60651558288373, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.6923076923076925, "cost_per_answer": 0.163882, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 33.87175370811019, "p90_latency": 33.87175370811019, "median_latency_all": 33.87175370811019, "p90_latency_all": 33.87175370811019, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 9.090909090909092, "cost_per_answer": 0.143861, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -7.73476187477354, "verdict": "same" }, { "id": 63, "question": "In the TCGA PanCancer Atlas breast cancer study, what is the correlation coefficient between EGFR expression levels and PIK3CA mutation status considering only patients who also have a homozygous deletion of PTEN?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 51.91497699986212, "p90_latency": 51.91497699986212, "median_latency_all": 51.91497699986212, "p90_latency_all": 51.91497699986212, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.0, "cost_per_answer": 0.16403610000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 47.82653333398048, "p90_latency": 47.82653333398048, "median_latency_all": 47.82653333398048, "p90_latency_all": 47.82653333398048, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.2156055, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -4.088443665881641, "verdict": "same" }, { "id": 66, "question": "In colorectal cancer do patients with microsatellite instability in the TCGA dataset have a similar prognosis as those in the MSK Gastroenterology 2020 study?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 58.06169991707429, "p90_latency": 58.06169991707429, "median_latency_all": 58.06169991707429, "p90_latency_all": 58.06169991707429, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.882352941176471, "cost_per_answer": 0.1865494, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 48.874883208889514, "p90_latency": 48.874883208889514, "median_latency_all": 48.874883208889514, "p90_latency_all": 48.874883208889514, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.555555555555555, "cost_per_answer": 0.18951679999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -9.186816708184779, "verdict": "same" }, { "id": 67, "question": "In the TCGA Breast Cancer study if a patient has both a PIK3CA mutation and a PTEN deletion what is their probability of having a luminal A subtype? Is this probability different from a patient with only a PIK3CA mutation?", "category": "Co-occurrence & exclusivity", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 72.28012279095128, "p90_latency": 72.28012279095128, "median_latency_all": 72.28012279095128, "p90_latency_all": 72.28012279095128, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.2326185, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 39.110043375054374, "p90_latency": 39.110043375054374, "median_latency_all": 39.110043375054374, "p90_latency_all": 39.110043375054374, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.142857142857143, "cost_per_answer": 0.21510279999999998, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -33.17007941589691, "verdict": "same" }, { "id": 69, "question": "Which patients have a TP53 G199V mutation? Which are somatic vs germline?", "category": "Patient & sample lookup", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 34.71514558396302, "p90_latency": 34.71514558396302, "median_latency_all": 34.71514558396302, "p90_latency_all": 34.71514558396302, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.17670260000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 40.50601737503894, "p90_latency": 40.50601737503894, "median_latency_all": 40.50601737503894, "p90_latency_all": 40.50601737503894, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 9.090909090909092, "cost_per_answer": 0.2032156, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 5.790871791075915, "verdict": "same" }, { "id": 70, "question": "In MSS colorectal cancer, what is the frequency of BRAF oncogenic mutations (as defined by OncoKB)? Use the 2017 MSK study for this. Can you give me a table of all these mutations, with frequency, count and denominator?", "category": "Variants & hotspots", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 55.23376633389853, "p90_latency": 55.23376633389853, "median_latency_all": 55.23376633389853, "p90_latency_all": 55.23376633389853, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 6.666666666666667, "cost_per_answer": 0.2102288, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 50.95534791704267, "p90_latency": 50.95534791704267, "median_latency_all": 50.95534791704267, "p90_latency_all": 50.95534791704267, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.2350314, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -4.278418416855857, "verdict": "same" }, { "id": 71, "question": "What is the current and future support for storing and analyzing germline variants in cBioPortal, compared to other alternatives?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 21.955559874884784, "p90_latency": 21.955559874884784, "median_latency_all": 21.955559874884784, "p90_latency_all": 21.955559874884784, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0550714, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 27.946821750025265, "p90_latency": 27.946821750025265, "median_latency_all": 27.946821750025265, "p90_latency_all": 27.946821750025265, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.07585510000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 5.991261875140481, "verdict": "same" }, { "id": 72, "question": "Is there any study with a polygenic risk score?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 28.456359874922782, "p90_latency": 28.456359874922782, "median_latency_all": 28.456359874922782, "p90_latency_all": 28.456359874922782, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.08445860000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 26.05909158301074, "p90_latency": 26.05909158301074, "median_latency_all": 26.05909158301074, "p90_latency_all": 26.05909158301074, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0870159, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -2.3972682919120416, "verdict": "same" }, { "id": 74, "question": "What are the most mutated genes in lung cancer?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 24.81253333389759, "p90_latency": 24.81253333389759, "median_latency_all": 24.81253333389759, "p90_latency_all": 24.81253333389759, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.12775889999999998, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 27.478650458040647, "p90_latency": 27.478650458040647, "median_latency_all": 27.478650458040647, "p90_latency_all": 27.478650458040647, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1360811, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 2.6661171241430566, "verdict": "same" }, { "id": 75, "question": "How many patients have a shallow deletion for SMARCA4 in the POG study?", "category": "Patient & sample lookup", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 23.46672674990259, "p90_latency": 23.46672674990259, "median_latency_all": 23.46672674990259, "p90_latency_all": 23.46672674990259, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.13890870000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 19.41596066695638, "p90_latency": 19.41596066695638, "median_latency_all": 19.41596066695638, "p90_latency_all": 19.41596066695638, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.10766010000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -4.050766082946211, "verdict": "same" }, { "id": 77, "question": "can you show me the minerva viewer for the ohsu htan sample", "category": "Patient & sample lookup", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 32.05821941699833, "p90_latency": 32.05821941699833, "median_latency_all": 32.05821941699833, "p90_latency_all": 32.05821941699833, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.09955589999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 0, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": null, "attempt_rate": 0.0, "pass_rate": 0.0, "median_latency": 24.58566066599451, "p90_latency": 24.58566066599451, "median_latency_all": 24.58566066599451, "p90_latency_all": 24.58566066599451, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0679152, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "–" }, "delta": 0.0, "latency_delta": -7.472558751003817, "verdict": "same" }, { "id": 78, "question": "what's the best way to put clinical data into cbioportal?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 14.419031416065991, "p90_latency": 14.419031416065991, "median_latency_all": 14.419031416065991, "p90_latency_all": 14.419031416065991, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.017951599999999998, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 12.97674925008323, "p90_latency": 12.97674925008323, "median_latency_all": 12.97674925008323, "p90_latency_all": 12.97674925008323, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.018468099999999998, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -1.4422821659827605, "verdict": "same" }, { "id": 79, "question": "Can you please generate a survival curve for colorectal cancer patients based on the expression levels of IMPDH2?", "category": "Survival & outcomes", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 24.483760042116046, "p90_latency": 24.483760042116046, "median_latency_all": 24.483760042116046, "p90_latency_all": 24.483760042116046, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1029886, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 19.891569082974456, "p90_latency": 19.891569082974456, "median_latency_all": 19.891569082974456, "p90_latency_all": 19.891569082974456, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0492423, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -4.59219095914159, "verdict": "same" }, { "id": 80, "question": "Is there a cohort of NSCLC patient samples that have Kras mutations, wild-type p53, and high expression levels of c-Myc?", "category": "Patient & sample lookup", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 44.91181025002152, "p90_latency": 44.91181025002152, "median_latency_all": 44.91181025002152, "p90_latency_all": 44.91181025002152, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.142857142857143, "cost_per_answer": 0.2166951, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 55.93408775003627, "p90_latency": 55.93408775003627, "median_latency_all": 55.93408775003627, "p90_latency_all": 55.93408775003627, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.170689, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 11.022277500014752, "verdict": "same" }, { "id": 82, "question": "How many GLASS patients developed hypermutation after TMZ treatment?", "category": "Treatment", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 56.87397245806642, "p90_latency": 56.87397245806642, "median_latency_all": 56.87397245806642, "p90_latency_all": 56.87397245806642, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 12, "median_llm_calls": 12, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.764705882352942, "cost_per_answer": 0.20752240000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 70.73389512498397, "p90_latency": 70.73389512498397, "median_latency_all": 70.73389512498397, "p90_latency_all": 70.73389512498397, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.882352941176471, "cost_per_answer": 0.300533, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 13.859922666917555, "verdict": "same" }, { "id": 84, "question": "Can you analyze thyroid hormone gene expression by diagnosis in the Pediatric Brain Tumor Atlas (PBTA, Provisional) cohort?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 17.20187612506561, "p90_latency": 17.20187612506561, "median_latency_all": 17.20187612506561, "p90_latency_all": 17.20187612506561, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.052797100000000007, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 20.491996666998602, "p90_latency": 20.491996666998602, "median_latency_all": 20.491996666998602, "p90_latency_all": 20.491996666998602, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.08091530000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 3.2901205419329926, "verdict": "same" }, { "id": 87, "question": "Show me cases where PALB2 or ATM have germline mutations", "category": "Patient & sample lookup", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 28.019862667191774, "p90_latency": 28.019862667191774, "median_latency_all": 28.019862667191774, "p90_latency_all": 28.019862667191774, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0972185, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 33.95408633293118, "p90_latency": 33.95408633293118, "median_latency_all": 33.95408633293118, "p90_latency_all": 33.95408633293118, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1061934, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 5.93422366573941, "verdict": "same" }, { "id": 88, "question": "what do idh1 mutations do?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 10.539149124873802, "p90_latency": 10.539149124873802, "median_latency_all": 10.539149124873802, "p90_latency_all": 10.539149124873802, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.0150541, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 9.581963707925752, "p90_latency": 9.581963707925752, "median_latency_all": 9.581963707925752, "p90_latency_all": 9.581963707925752, "fast_share": 100.0, "fast_share_all": 100.0, "mean_llm_calls": 1, "median_llm_calls": 1, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": null, "cost_per_answer": 0.015090600000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -0.9571854169480503, "verdict": "same" }, { "id": 89, "question": "is there any imaging data?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 21.877797417109832, "p90_latency": 21.877797417109832, "median_latency_all": 21.877797417109832, "p90_latency_all": 21.877797417109832, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0664256, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 25.69504424999468, "p90_latency": 25.69504424999468, "median_latency_all": 25.69504424999468, "p90_latency_all": 25.69504424999468, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.06408069999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 3.817246832884848, "verdict": "same" }, { "id": 90, "question": "show me the distribution of mutations in the tert promoter across cancer types", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 57.51816683402285, "p90_latency": 57.51816683402285, "median_latency_all": 57.51816683402285, "p90_latency_all": 57.51816683402285, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.142857142857143, "cost_per_answer": 0.2506297, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 76.13273741700687, "p90_latency": 76.13273741700687, "median_latency_all": 76.13273741700687, "p90_latency_all": 76.13273741700687, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 12, "median_llm_calls": 12, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.11111111111111, "cost_per_answer": 0.36329740000000005, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 18.614570582984015, "verdict": "same" }, { "id": 91, "question": "show me a histogram of C228T mutations in the tert promoter across cancer types", "category": "Variants & hotspots", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 90.73768474999815, "p90_latency": 90.73768474999815, "median_latency_all": 90.73768474999815, "p90_latency_all": 90.73768474999815, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 16, "median_llm_calls": 16, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 8.695652173913043, "cost_per_answer": 0.3697269, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 81.40820345899556, "p90_latency": 81.40820345899556, "median_latency_all": 81.40820345899556, "p90_latency_all": 81.40820345899556, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 14, "median_llm_calls": 14, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.555555555555555, "cost_per_answer": 0.32176380000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -9.329481291002594, "verdict": "same" }, { "id": 93, "question": "Find patients IDs and samples in colorectal cancer that harbor the V600V alteration in BRAF", "category": "Patient & sample lookup", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 26.44245329219848, "p90_latency": 26.44245329219848, "median_latency_all": 26.44245329219848, "p90_latency_all": 26.44245329219848, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.085254, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 43.45312824996654, "p90_latency": 43.45312824996654, "median_latency_all": 43.45312824996654, "p90_latency_all": 43.45312824996654, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 27.272727272727273, "cost_per_answer": 0.2377765, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 17.01067495776806, "verdict": "same" }, { "id": 94, "question": "There is a heavily discussed driver alteration in MAP2K1 at codon 105 that significantly alters mRNA stability. Please list the expression values for the tumors that have a nucleotide change at this position", "category": "Expression & multi-omics", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 25.012162333121523, "p90_latency": 25.012162333121523, "median_latency_all": 25.012162333121523, "p90_latency_all": 25.012162333121523, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 20.0, "cost_per_answer": 0.11577080000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 53.278486250084825, "p90_latency": 53.278486250084825, "median_latency_all": 53.278486250084825, "p90_latency_all": 53.278486250084825, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 6.666666666666667, "cost_per_answer": 0.25568490000000005, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 28.2663239169633, "verdict": "same" }, { "id": 95, "question": "download MSK-CHORD study on Non-Small Cell Lung Cancer dataset", "category": "Study discovery", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 14.514922833070159, "p90_latency": 14.514922833070159, "median_latency_all": 14.514922833070159, "p90_latency_all": 14.514922833070159, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.038496600000000006, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 11.054216208052821, "p90_latency": 11.054216208052821, "median_latency_all": 11.054216208052821, "p90_latency_all": 11.054216208052821, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.020872, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -3.4607066250173375, "verdict": "same" }, { "id": 98, "question": "Can you think of any flaws in the methodology used in the MSK-CHORD paper (Jee et al., Nature 2024)?", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 23.76263991720043, "p90_latency": 23.76263991720043, "median_latency_all": 23.76263991720043, "p90_latency_all": 23.76263991720043, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0474717, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 24.07254591700621, "p90_latency": 24.07254591700621, "median_latency_all": 24.07254591700621, "p90_latency_all": 24.07254591700621, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.049740599999999996, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 0.30990599980577826, "verdict": "same" }, { "id": 99, "question": "Can you help me write a production-ready Python script using Bokeh to build an interactive clinicogenomic dashboard to analyzing the MSK-CHORD dataset? I have some specific requirements I can give you", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 12.473121833056211, "p90_latency": 12.473121833056211, "median_latency_all": 12.473121833056211, "p90_latency_all": 12.473121833056211, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0336975, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 20.76982400007546, "p90_latency": 20.76982400007546, "median_latency_all": 20.76982400007546, "p90_latency_all": 20.76982400007546, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1288484, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 8.296702167019248, "verdict": "same" }, { "id": 100, "question": "How many samples are there that have any of these mutations in SEPHS1: p.Arg371Trp, p.Arg371Gln, p.Arg371Gly?", "category": "Patient & sample lookup", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 39.06022004107945, "p90_latency": 39.06022004107945, "median_latency_all": 39.06022004107945, "p90_latency_all": 39.06022004107945, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.2001822, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 22.637327375006862, "p90_latency": 22.637327375006862, "median_latency_all": 22.637327375006862, "p90_latency_all": 22.637327375006862, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.097731, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -16.422892666072585, "verdict": "same" }, { "id": 101, "question": "List the top 20 mutated genes in study nbl_msk_2023.", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 19.06028441619128, "p90_latency": 19.06028441619128, "median_latency_all": 19.06028441619128, "p90_latency_all": 19.06028441619128, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0987194, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 15.63799558300525, "p90_latency": 15.63799558300525, "median_latency_all": 15.63799558300525, "p90_latency_all": 15.63799558300525, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.044539600000000006, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -3.4222888331860304, "verdict": "same" }, { "id": 103, "question": "in salivary cancer (adenoid cystic carcinoma), what are the expected drivers ? Classify them by actionability. What about BCOR mutations, either somatic or germline ?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 53.073531250003725, "p90_latency": 53.073531250003725, "median_latency_all": 53.073531250003725, "p90_latency_all": 53.073531250003725, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.20272120000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 48.039967874996364, "p90_latency": 48.039967874996364, "median_latency_all": 48.039967874996364, "p90_latency_all": 48.039967874996364, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.16162870000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -5.033563375007361, "verdict": "same" }, { "id": 105, "question": "list the portal studies for pediatric cancers that were published in the last 5 years", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 15.242907125037163, "p90_latency": 15.242907125037163, "median_latency_all": 15.242907125037163, "p90_latency_all": 15.242907125037163, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0448439, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 16.770684791961685, "p90_latency": 16.770684791961685, "median_latency_all": 16.770684791961685, "p90_latency_all": 16.770684791961685, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0544835, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 1.5277776669245213, "verdict": "same" }, { "id": 107, "question": "What kind of cancer are there in the database?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 13.911781874950975, "p90_latency": 13.911781874950975, "median_latency_all": 13.911781874950975, "p90_latency_all": 13.911781874950975, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0593539, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 13.241077917045914, "p90_latency": 13.241077917045914, "median_latency_all": 13.241077917045914, "p90_latency_all": 13.241077917045914, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0525038, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -0.6707039579050615, "verdict": "same" }, { "id": 108, "question": "Can you find a study that I may emulate that has a data_clinical_outcomes.txt file and associated meta file to interogate", "category": "Out of scope", "track": "out_of_scope", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 15.814120166935027, "p90_latency": 15.814120166935027, "median_latency_all": 15.814120166935027, "p90_latency_all": 15.814120166935027, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.023154400000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 15.25608366704546, "p90_latency": 15.25608366704546, "median_latency_all": 15.25608366704546, "p90_latency_all": 15.25608366704546, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0286265, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -0.5580364998895675, "verdict": "same" }, { "id": 110, "question": "Can you explore the difference in mutation frequency between left-sided and right-sided CRC?", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 114.03543437505141, "p90_latency": 114.03543437505141, "median_latency_all": 114.03543437505141, "p90_latency_all": 114.03543437505141, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 20, "median_llm_calls": 20, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.407407407407407, "cost_per_answer": 0.496342, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 73.78941741702147, "p90_latency": 73.78941741702147, "median_latency_all": 73.78941741702147, "p90_latency_all": 73.78941741702147, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.526315789473685, "cost_per_answer": 0.33831690000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -40.24601695802994, "verdict": "same" }, { "id": 111, "question": "can you show me a study with longitudinal data and a patient that has multiple samples over time?", "category": "Cohort & clinical counts", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 33.42426245799288, "p90_latency": 33.42426245799288, "median_latency_all": 33.42426245799288, "p90_latency_all": 33.42426245799288, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.11309939999999999, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 24.354648374952376, "p90_latency": 24.354648374952376, "median_latency_all": 24.354648374952376, "p90_latency_all": 24.354648374952376, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0969399, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -9.069614083040506, "verdict": "same" }, { "id": 112, "question": "Which cBioPortal studies include lung adenocarcinoma samples with mutation and copy-number data?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 22.546767333056778, "p90_latency": 22.546767333056778, "median_latency_all": 22.546767333056778, "p90_latency_all": 22.546767333056778, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0806122, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 23.680228415993042, "p90_latency": 23.680228415993042, "median_latency_all": 23.680228415993042, "p90_latency_all": 23.680228415993042, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1096776, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 1.1334610829362646, "verdict": "same" }, { "id": 113, "question": "In lower grade glioma, are there genes which are overexpressed in any of the molecular subtypes?", "category": "Expression & multi-omics", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 101.06706654187292, "p90_latency": 101.06706654187292, "median_latency_all": 101.06706654187292, "p90_latency_all": 101.06706654187292, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 9.090909090909092, "cost_per_answer": 0.1841466, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 69.91114275006112, "p90_latency": 69.91114275006112, "median_latency_all": 69.91114275006112, "p90_latency_all": 69.91114275006112, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 12, "median_llm_calls": 12, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 5.882352941176471, "cost_per_answer": 0.2535811, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -31.1559237918118, "verdict": "same" }, { "id": 115, "question": "Are there DNA methylation differences between lower grade glioma molecular subtypes?", "category": "Expression & multi-omics", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 102.17114837490954, "p90_latency": 102.17114837490954, "median_latency_all": 102.17114837490954, "p90_latency_all": 102.17114837490954, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 20, "median_llm_calls": 20, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.538461538461538, "cost_per_answer": 0.34482280000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 75.45076083298773, "p90_latency": 75.45076083298773, "median_latency_all": 75.45076083298773, "p90_latency_all": 75.45076083298773, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 12, "median_llm_calls": 12, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 4.761904761904762, "cost_per_answer": 0.25021140000000003, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -26.72038754192181, "verdict": "same" }, { "id": 118, "question": "In TCGA lower grade glioma, show me IDH1 mRNA expression by IDH1 mutation status.", "category": "Expression & multi-omics", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 29.568687583785504, "p90_latency": 29.568687583785504, "median_latency_all": 29.568687583785504, "p90_latency_all": 29.568687583785504, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.090126, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 35.71219520899467, "p90_latency": 35.71219520899467, "median_latency_all": 35.71219520899467, "p90_latency_all": 35.71219520899467, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.11111111111111, "cost_per_answer": 0.1212945, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 6.143507625209168, "verdict": "same" }, { "id": 119, "question": "In lower grade glioma, how do outcomes differ for IDH1 altered vs EGFR altered patients?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 35.03495900006965, "p90_latency": 35.03495900006965, "median_latency_all": 35.03495900006965, "p90_latency_all": 35.03495900006965, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.14514020000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 35.88036666705739, "p90_latency": 35.88036666705739, "median_latency_all": 35.88036666705739, "p90_latency_all": 35.88036666705739, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1382436, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 0.8454076669877395, "verdict": "same" }, { "id": 120, "question": "is there a relationship between cic mutation and 19q del in lgg?", "category": "Co-occurrence & exclusivity", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 81.02412062510848, "p90_latency": 81.02412062510848, "median_latency_all": 81.02412062510848, "p90_latency_all": 81.02412062510848, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 17, "median_llm_calls": 17, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 13.043478260869565, "cost_per_answer": 0.5049868000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 70.32496604206972, "p90_latency": 70.32496604206972, "median_latency_all": 70.32496604206972, "p90_latency_all": 70.32496604206972, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 17, "median_llm_calls": 17, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 8.333333333333334, "cost_per_answer": 0.2691815, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -10.699154583038762, "verdict": "same" }, { "id": 121, "question": "In TCGA lower grade glioma, show me samples with EGFR gains.", "category": "Alteration frequency", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 29.323358667083085, "p90_latency": 29.323358667083085, "median_latency_all": 29.323358667083085, "p90_latency_all": 29.323358667083085, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1166055, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 40.96278170798905, "p90_latency": 40.96278170798905, "median_latency_all": 40.96278170798905, "p90_latency_all": 40.96278170798905, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1154228, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 11.639423040905967, "verdict": "same" }, { "id": 122, "question": "In TCGA lower grade glioma, filter to samples that are both IDH1 and TP53 mutant and show me the summary page.", "category": "Patient & sample lookup", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 10.779196915915236, "p90_latency": 10.779196915915236, "median_latency_all": 10.779196915915236, "p90_latency_all": 10.779196915915236, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.032970599999999996, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 16.62497458292637, "p90_latency": 16.62497458292637, "median_latency_all": 16.62497458292637, "p90_latency_all": 16.62497458292637, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0347948, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 5.845777667011134, "verdict": "same" }, { "id": 124, "question": "show me EGFR expression across cancer types", "category": "Expression & multi-omics", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 32.94333366607316, "p90_latency": 32.94333366607316, "median_latency_all": 32.94333366607316, "p90_latency_all": 32.94333366607316, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0982192, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 45.99538695788942, "p90_latency": 45.99538695788942, "median_latency_all": 45.99538695788942, "p90_latency_all": 45.99538695788942, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 12.5, "cost_per_answer": 0.1281536, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 13.052053291816264, "verdict": "same" }, { "id": 125, "question": "show me point mutations in EGFR in lung cancer", "category": "Variants & hotspots", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 11.681318166898564, "p90_latency": 11.681318166898564, "median_latency_all": 11.681318166898564, "p90_latency_all": 11.681318166898564, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0729775, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 23.985106375068426, "p90_latency": 23.985106375068426, "median_latency_all": 23.985106375068426, "p90_latency_all": 23.985106375068426, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.121364, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 12.303788208169863, "verdict": "same" }, { "id": 126, "question": "are there cdkn2a het losses in gbm tcga study?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 24.539186540991068, "p90_latency": 24.539186540991068, "median_latency_all": 24.539186540991068, "p90_latency_all": 24.539186540991068, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 20.0, "cost_per_answer": 0.1180341, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 23.114730792003684, "p90_latency": 23.114730792003684, "median_latency_all": 23.114730792003684, "p90_latency_all": 23.114730792003684, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.11723360000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -1.4244557489873841, "verdict": "same" }, { "id": 127, "question": "show me gbm with mgmt hypermethylation", "category": "Patient & sample lookup", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 25.559600374894217, "p90_latency": 25.559600374894217, "median_latency_all": 25.559600374894217, "p90_latency_all": 25.559600374894217, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0886425, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 21.741750916931778, "p90_latency": 21.741750916931778, "median_latency_all": 21.741750916931778, "p90_latency_all": 21.741750916931778, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 12.5, "cost_per_answer": 0.0773771, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -3.8178494579624385, "verdict": "same" }, { "id": 128, "question": "Is there a lower grade glioma study with race data?", "category": "Study discovery", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 17.20051029184833, "p90_latency": 17.20051029184833, "median_latency_all": 17.20051029184833, "p90_latency_all": 17.20051029184833, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0858863, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 15.500020082923584, "p90_latency": 15.500020082923584, "median_latency_all": 15.500020082923584, "p90_latency_all": 15.500020082923584, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0817267, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -1.700490208924748, "verdict": "same" }, { "id": 129, "question": "compare atrx mutant vs cic mutant lgg - are there different outcomes?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 38.86724929208867, "p90_latency": 38.86724929208867, "median_latency_all": 38.86724929208867, "p90_latency_all": 38.86724929208867, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 10.0, "cost_per_answer": 0.1851026, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 44.163570833043195, "p90_latency": 44.163570833043195, "median_latency_all": 44.163570833043195, "p90_latency_all": 44.163570833043195, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.13446660000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 5.296321540954523, "verdict": "same" }, { "id": 130, "question": "are there different outcomes for idh1 mutant vs egfr amp in lgg?", "category": "Survival & outcomes", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 42.39770883414894, "p90_latency": 42.39770883414894, "median_latency_all": 42.39770883414894, "p90_latency_all": 42.39770883414894, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 11.11111111111111, "cost_per_answer": 0.1517697, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 58.487465125042945, "p90_latency": 58.487465125042945, "median_latency_all": 58.487465125042945, "p90_latency_all": 58.487465125042945, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 11, "median_llm_calls": 11, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 6.666666666666667, "cost_per_answer": 0.2481507, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 16.089756290894, "verdict": "same" }, { "id": 131, "question": "what other genes are altered in kras mutant crc or luad", "category": "Co-occurrence & exclusivity", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 32.03888987493701, "p90_latency": 32.03888987493701, "median_latency_all": 32.03888987493701, "p90_latency_all": 32.03888987493701, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.11628580000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 33.31051354086958, "p90_latency": 33.31051354086958, "median_latency_all": 33.31051354086958, "p90_latency_all": 33.31051354086958, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1185078, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 1.271623665932566, "verdict": "same" }, { "id": 132, "question": "Create an OncoPrint with a merged track for the EGFR family genes (EGFR, ERBB2, ERBB3, ERBB4) across TCGA PanCancer Atlas studies.", "category": "Alteration frequency", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 21.41708845901303, "p90_latency": 21.41708845901303, "median_latency_all": 21.41708845901303, "p90_latency_all": 21.41708845901303, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0966133, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 24.177622540853918, "p90_latency": 24.177622540853918, "median_latency_all": 24.177622540853918, "p90_latency_all": 24.177622540853918, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0980413, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 2.7605340818408877, "verdict": "same" }, { "id": 133, "question": "What are the EGFR mutation frequencies across cancer types in the MSK-IMPACT 50K study?", "category": "Alteration frequency", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 23.12484629196115, "p90_latency": 23.12484629196115, "median_latency_all": 23.12484629196115, "p90_latency_all": 23.12484629196115, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 25.0, "cost_per_answer": 0.1004722, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 38.67706716712564, "p90_latency": 38.67706716712564, "median_latency_all": 38.67706716712564, "p90_latency_all": 38.67706716712564, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1602664, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": 15.552220875164494, "verdict": "same" }, { "id": 134, "question": "compare egfr mutations between lung and brain cancer", "category": "Alteration frequency", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 30.268279624870047, "p90_latency": 30.268279624870047, "median_latency_all": 30.268279624870047, "p90_latency_all": 30.268279624870047, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.16103920000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 28.020336749963462, "p90_latency": 28.020336749963462, "median_latency_all": 28.020336749963462, "p90_latency_all": 28.020336749963462, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1397868, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -2.2479428749065846, "verdict": "same" }, { "id": 136, "question": "In the TCGA Breast Cancer study, do TP53 mutations and high MYC expression co-occur or are they mutually exclusive?", "category": "Co-occurrence & exclusivity", "track": "analysis", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 31.053415999980643, "p90_latency": 31.053415999980643, "median_latency_all": 31.053415999980643, "p90_latency_all": 31.053415999980643, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 7, "median_llm_calls": 7, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.11510480000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 40.346398750087246, "p90_latency": 40.346398750087246, "median_latency_all": 40.346398750087246, "p90_latency_all": 40.346398750087246, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 9.090909090909092, "cost_per_answer": 0.1524044, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 9.292982750106603, "verdict": "same" }, { "id": 137, "question": "In the TCGA Glioblastoma multiforme study (gbm_tcga_pan_can_atlas_2018), how many patients have an IDH1 R132H mutation and how many are IDH1 wild-type.", "category": "Cohort & clinical counts", "track": "data", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 22.4792595829349, "p90_latency": 22.4792595829349, "median_latency_all": 22.4792595829349, "p90_latency_all": 22.4792595829349, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1202585, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 19.85682850005105, "p90_latency": 19.85682850005105, "median_latency_all": 19.85682850005105, "p90_latency_all": 19.85682850005105, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.11389070000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -2.6224310828838497, "verdict": "same" }, { "id": 138, "question": "show me all KRAS mutations in colorectal cancer that are not at position 12", "category": "Variants & hotspots", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 45.75624216697179, "p90_latency": 45.75624216697179, "median_latency_all": 45.75624216697179, "p90_latency_all": 45.75624216697179, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 9, "median_llm_calls": 9, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 7.6923076923076925, "cost_per_answer": 0.2047973, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 21.52621087501757, "p90_latency": 21.52621087501757, "median_latency_all": 21.52621087501757, "p90_latency_all": 21.52621087501757, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.11166960000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -24.23003129195422, "verdict": "same" }, { "id": 139, "question": "show me cholangio with idh1 mutations other than r132", "category": "Variants & hotspots", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 15.212785459123552, "p90_latency": 15.212785459123552, "median_latency_all": 15.212785459123552, "p90_latency_all": 15.212785459123552, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 2, "median_llm_calls": 2, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0890902, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 29.903875707881525, "p90_latency": 29.903875707881525, "median_latency_all": 29.903875707881525, "p90_latency_all": 29.903875707881525, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 5, "median_llm_calls": 5, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.1536896, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 14.691090248757973, "verdict": "same" }, { "id": 140, "question": "what's the frequency of different idh1 mutations in glioma vs cholangio vs chndrosarcoma? Give me a link to compare the frequency of the specific IDH1 mutations in those cancer types.", "category": "Variants & hotspots", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 36.604484958108515, "p90_latency": 36.604484958108515, "median_latency_all": 36.604484958108515, "p90_latency_all": 36.604484958108515, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.16671260000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 54.66823850013316, "p90_latency": 54.66823850013316, "median_latency_all": 54.66823850013316, "p90_latency_all": 54.66823850013316, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.2303131, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": 18.063753542024642, "verdict": "same" }, { "id": 142, "question": "Give me an OncoPrint for RTK genes in lung cancer, limited to driver events.", "category": "Alteration frequency", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 17.879792750114575, "p90_latency": 17.879792750114575, "median_latency_all": 17.879792750114575, "p90_latency_all": 17.879792750114575, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0515765, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 17.814572375034913, "p90_latency": 17.814572375034913, "median_latency_all": 17.814572375034913, "p90_latency_all": 17.814572375034913, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 3, "median_llm_calls": 3, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.051863400000000004, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -0.06522037507966161, "verdict": "same" }, { "id": 144, "question": "are lung carcinosarcomas associated with BRIP1 mutations?", "category": "Alteration frequency", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 47.538901292020455, "p90_latency": 47.538901292020455, "median_latency_all": 47.538901292020455, "p90_latency_all": 47.538901292020455, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 8, "median_llm_calls": 8, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 18.181818181818183, "cost_per_answer": 0.1700086, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 0, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 0.0, "precision": 0.0, "attempt_rate": 100.0, "pass_rate": 0.0, "median_latency": 41.74122683308087, "p90_latency": 41.74122683308087, "median_latency_all": 41.74122683308087, "p90_latency_all": 41.74122683308087, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 6, "median_llm_calls": 6, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.18024420000000002, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✗" }, "delta": 0.0, "latency_delta": -5.797674458939582, "verdict": "same" }, { "id": 146, "question": "In the MSK-CHORD study, is it possible to see which patients received radiation therapy?", "category": "Treatment", "track": "navigation", "gradeable": true, "mismatch": [], "a": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 22.299915707902983, "p90_latency": 22.299915707902983, "median_latency_all": 22.299915707902983, "p90_latency_all": 22.299915707902983, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.07174180000000001, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "b": { "expected": 1, "completed": 1, "failed": 0, "missing": 0, "ungraded": 0, "no_reference": 0, "passes": 1, "attempted": 1, "eligible": 1, "unexpected": 0, "recall": 100.0, "precision": 100.0, "attempt_rate": 100.0, "pass_rate": 100.0, "median_latency": 16.076999417040497, "p90_latency": 16.076999417040497, "median_latency_all": 16.076999417040497, "p90_latency_all": 16.076999417040497, "fast_share": 0.0, "fast_share_all": 0.0, "mean_llm_calls": 4, "median_llm_calls": 4, "mean_tool_rounds": 0, "mean_handoffs": 0, "failed_handoffs": 0, "tool_error_rate": 0.0, "cost_per_answer": 0.0651588, "latency_stdev": null, "pass_variance": null, "flaky": false, "routed_to": [], "icons": "✓" }, "delta": 0.0, "latency_delta": -6.222916290862486, "verdict": "same" } ], "question_verdicts": { "better": 26, "worse": 15, "same": 105 } }