3365 lines
No EOL
66 KiB
JSON
3365 lines
No EOL
66 KiB
JSON
{
|
|
"n": 70,
|
|
"results": [
|
|
{
|
|
"trap": "T-ana-2",
|
|
"type": "T-ana",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.594,
|
|
"arm_accuracy": 0.938,
|
|
"paired_diff_arm_minus_a0": 0.344,
|
|
"pairwise": {
|
|
"arm_wins": 6,
|
|
"a0_wins": 0,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 6,
|
|
"arm": 1
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.917,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.9,
|
|
"a0_acc": 0.9,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0.083,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-2",
|
|
"type": "T-ana",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.312,
|
|
"arm_accuracy": 0.688,
|
|
"paired_diff_arm_minus_a0": 0.375,
|
|
"pairwise": {
|
|
"arm_wins": 6,
|
|
"a0_wins": 0,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 11,
|
|
"arm": 5
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 1.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.5,
|
|
"a0_acc": 0.5,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-2",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.525,
|
|
"arm_accuracy": 0.85,
|
|
"paired_diff_arm_minus_a0": 0.325,
|
|
"pairwise": {
|
|
"arm_wins": 8,
|
|
"a0_wins": 0,
|
|
"tie": 12
|
|
},
|
|
"catastrophe": {
|
|
"a0": 8,
|
|
"arm": 3
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.65,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 7
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.7,
|
|
"a0_acc": 0.7,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0.35,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-5",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-5",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-5",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-9",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.775,
|
|
"arm_accuracy": 0.9,
|
|
"paired_diff_arm_minus_a0": 0.125,
|
|
"pairwise": {
|
|
"arm_wins": 5,
|
|
"a0_wins": 3,
|
|
"tie": 12
|
|
},
|
|
"catastrophe": {
|
|
"a0": 4,
|
|
"arm": 1
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.25,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 3,
|
|
"tie": 7
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.8,
|
|
"a0_acc": 0.55,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-9",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.75,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.25,
|
|
"pairwise": {
|
|
"arm_wins": 5,
|
|
"a0_wins": 0,
|
|
"tie": 15
|
|
},
|
|
"catastrophe": {
|
|
"a0": 5,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.5,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0.5,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-9",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-16",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.725,
|
|
"arm_accuracy": 0.725,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 4,
|
|
"arm": 4
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.45,
|
|
"a0_acc": 0.45,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-16",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.675,
|
|
"arm_accuracy": 0.675,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 4,
|
|
"arm": 4
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.35,
|
|
"a0_acc": 0.35,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-16",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.6,
|
|
"arm_accuracy": 0.6,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 6,
|
|
"arm": 6
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.2,
|
|
"a0_acc": 0.2,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-17",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.25,
|
|
"arm_accuracy": 0.3,
|
|
"paired_diff_arm_minus_a0": 0.05,
|
|
"pairwise": {
|
|
"arm_wins": 5,
|
|
"a0_wins": 4,
|
|
"tie": 11
|
|
},
|
|
"catastrophe": {
|
|
"a0": 15,
|
|
"arm": 14
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.1,
|
|
"without_mistral-large-latest": 0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 4,
|
|
"tie": 6
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.6,
|
|
"a0_acc": 0.5,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-17",
|
|
"type": "T-ana",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.625,
|
|
"arm_accuracy": 0.688,
|
|
"paired_diff_arm_minus_a0": 0.062,
|
|
"pairwise": {
|
|
"arm_wins": 6,
|
|
"a0_wins": 5,
|
|
"tie": 5
|
|
},
|
|
"catastrophe": {
|
|
"a0": 6,
|
|
"arm": 5
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -0.5,
|
|
"without_mistral-large-latest": 1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 3,
|
|
"a0": 5,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.5,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-17",
|
|
"type": "T-ana",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.5,
|
|
"arm_accuracy": 0.875,
|
|
"paired_diff_arm_minus_a0": 0.375,
|
|
"pairwise": {
|
|
"arm_wins": 8,
|
|
"a0_wins": 4,
|
|
"tie": 4
|
|
},
|
|
"catastrophe": {
|
|
"a0": 8,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 3
|
|
},
|
|
"BA": {
|
|
"arm": 3,
|
|
"a0": 4,
|
|
"tie": 1
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.8,
|
|
"a0_acc": 0.8,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-22",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.75,
|
|
"arm_accuracy": 0.75,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 5,
|
|
"a0_wins": 5,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 5,
|
|
"arm": 5
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 5,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.5,
|
|
"a0_acc": 0.5,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-22",
|
|
"type": "T-ana",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.75,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.25,
|
|
"pairwise": {
|
|
"arm_wins": 5,
|
|
"a0_wins": 0,
|
|
"tie": 11
|
|
},
|
|
"catastrophe": {
|
|
"a0": 3,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.667,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 2,
|
|
"a0": 0,
|
|
"tie": 6
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0.333,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-22",
|
|
"type": "T-ana",
|
|
"n_votes": 19,
|
|
"parse_fail": 1,
|
|
"a0_accuracy": 0.737,
|
|
"arm_accuracy": 0.947,
|
|
"paired_diff_arm_minus_a0": 0.211,
|
|
"pairwise": {
|
|
"arm_wins": 5,
|
|
"a0_wins": 1,
|
|
"tie": 13
|
|
},
|
|
"catastrophe": {
|
|
"a0": 5,
|
|
"arm": 1
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.4,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 1,
|
|
"tie": 8
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 9
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.9,
|
|
"a0_acc": 0.5,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-23",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.8,
|
|
"arm_accuracy": 0.975,
|
|
"paired_diff_arm_minus_a0": 0.175,
|
|
"pairwise": {
|
|
"arm_wins": 6,
|
|
"a0_wins": 2,
|
|
"tie": 12
|
|
},
|
|
"catastrophe": {
|
|
"a0": 3,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.1,
|
|
"without_mistral-large-latest": 0.25
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 2,
|
|
"a0": 1,
|
|
"tie": 7
|
|
},
|
|
"BA": {
|
|
"arm": 4,
|
|
"a0": 1,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.95,
|
|
"a0_acc": 0.7,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0.9,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-23",
|
|
"type": "T-ana",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.656,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.344,
|
|
"pairwise": {
|
|
"arm_wins": 8,
|
|
"a0_wins": 0,
|
|
"tie": 8
|
|
},
|
|
"catastrophe": {
|
|
"a0": 2,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.2,
|
|
"without_mistral-large-latest": 0.583
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 3
|
|
},
|
|
"BA": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0.417,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0.8,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-23",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.8,
|
|
"arm_accuracy": 0.925,
|
|
"paired_diff_arm_minus_a0": 0.125,
|
|
"pairwise": {
|
|
"arm_wins": 6,
|
|
"a0_wins": 2,
|
|
"tie": 12
|
|
},
|
|
"catastrophe": {
|
|
"a0": 1,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.25
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 1,
|
|
"a0": 2,
|
|
"tie": 7
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.85,
|
|
"a0_acc": 0.6,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-31",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-31",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-31",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-32",
|
|
"type": "T-ana",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.344,
|
|
"arm_accuracy": 0.656,
|
|
"paired_diff_arm_minus_a0": 0.312,
|
|
"pairwise": {
|
|
"arm_wins": 11,
|
|
"a0_wins": 5,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 8,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.2,
|
|
"without_mistral-large-latest": 0.5
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 8,
|
|
"a0": 0,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 3,
|
|
"a0": 5,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.5,
|
|
"a0_acc": 0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.75,
|
|
"a0_acc": 0.55,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-32",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.275,
|
|
"arm_accuracy": 0.35,
|
|
"paired_diff_arm_minus_a0": 0.075,
|
|
"pairwise": {
|
|
"arm_wins": 5,
|
|
"a0_wins": 5,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 3,
|
|
"arm": 1
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.15,
|
|
"without_mistral-large-latest": 0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 5,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.7,
|
|
"a0_acc": 0.55,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-32",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.325,
|
|
"arm_accuracy": 0.375,
|
|
"paired_diff_arm_minus_a0": 0.05,
|
|
"pairwise": {
|
|
"arm_wins": 5,
|
|
"a0_wins": 5,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 2,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.1,
|
|
"without_mistral-large-latest": 0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 5,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.75,
|
|
"a0_acc": 0.65,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-34",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-34",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-34",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.9,
|
|
"paired_diff_arm_minus_a0": -0.1,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 4,
|
|
"tie": 16
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -0.2,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 4,
|
|
"tie": 6
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.8,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-35",
|
|
"type": "T-ana",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.594,
|
|
"arm_accuracy": 0.969,
|
|
"paired_diff_arm_minus_a0": 0.375,
|
|
"pairwise": {
|
|
"arm_wins": 6,
|
|
"a0_wins": 0,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 6,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 1.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.95,
|
|
"a0_acc": 0.95,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-35",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -1.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 12
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-35",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -1.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 12
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-36",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.4,
|
|
"arm_accuracy": 0.3,
|
|
"paired_diff_arm_minus_a0": -0.1,
|
|
"pairwise": {
|
|
"arm_wins": 7,
|
|
"a0_wins": 5,
|
|
"tie": 8
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 5
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -0.25,
|
|
"without_mistral-large-latest": 0.05
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 7,
|
|
"a0": 0,
|
|
"tie": 3
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 5,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.1,
|
|
"a0_acc": 0.05,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.5,
|
|
"a0_acc": 0.75,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-36",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.25,
|
|
"arm_accuracy": 0.3,
|
|
"paired_diff_arm_minus_a0": 0.05,
|
|
"pairwise": {
|
|
"arm_wins": 5,
|
|
"a0_wins": 5,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 4,
|
|
"arm": 3
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.1,
|
|
"without_mistral-large-latest": 0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 5,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.6,
|
|
"a0_acc": 0.5,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-36",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.35,
|
|
"arm_accuracy": 0.375,
|
|
"paired_diff_arm_minus_a0": 0.025,
|
|
"pairwise": {
|
|
"arm_wins": 6,
|
|
"a0_wins": 5,
|
|
"tie": 9
|
|
},
|
|
"catastrophe": {
|
|
"a0": 1,
|
|
"arm": 1
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.05
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 6,
|
|
"a0": 0,
|
|
"tie": 4
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 5,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.05,
|
|
"a0_acc": 0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.7,
|
|
"a0_acc": 0.7,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-37",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.95,
|
|
"arm_accuracy": 0.925,
|
|
"paired_diff_arm_minus_a0": -0.025,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 1,
|
|
"tie": 19
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": -0.05
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 1,
|
|
"tie": 9
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.85,
|
|
"a0_acc": 0.9,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-37",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.975,
|
|
"arm_accuracy": 0.95,
|
|
"paired_diff_arm_minus_a0": -0.025,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 1,
|
|
"tie": 19
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": -0.05
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 1,
|
|
"tie": 9
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.9,
|
|
"a0_acc": 0.95,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-37",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.85,
|
|
"paired_diff_arm_minus_a0": -0.15,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 7,
|
|
"tie": 13
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -0.25,
|
|
"without_mistral-large-latest": -0.05
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 1,
|
|
"tie": 9
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 4
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.95,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.75,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-38",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-38",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-38",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-39",
|
|
"type": "T-ana",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.688,
|
|
"arm_accuracy": 0.312,
|
|
"paired_diff_arm_minus_a0": -0.375,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 6,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 6
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 3,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 3,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.5,
|
|
"a0_acc": 0.5,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-39",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.55,
|
|
"arm_accuracy": 0.475,
|
|
"paired_diff_arm_minus_a0": -0.075,
|
|
"pairwise": {
|
|
"arm_wins": 4,
|
|
"a0_wins": 6,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 4,
|
|
"arm": 5
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -0.15,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 4,
|
|
"a0": 1,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 5,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.5,
|
|
"a0_acc": 0.5,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.45,
|
|
"a0_acc": 0.6,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-39",
|
|
"type": "T-ana",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.75,
|
|
"arm_accuracy": 0.375,
|
|
"paired_diff_arm_minus_a0": -0.375,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 6,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 6
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 3,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 3,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.6,
|
|
"a0_acc": 0.6,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ell-40",
|
|
"type": "T-ell",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.15,
|
|
"arm_accuracy": 0.3,
|
|
"paired_diff_arm_minus_a0": 0.15,
|
|
"pairwise": {
|
|
"arm_wins": 6,
|
|
"a0_wins": 3,
|
|
"tie": 11
|
|
},
|
|
"catastrophe": {
|
|
"a0": 8,
|
|
"arm": 5
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.3,
|
|
"without_mistral-large-latest": 0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 1,
|
|
"a0": 3,
|
|
"tie": 6
|
|
},
|
|
"BA": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.6,
|
|
"a0_acc": 0.3,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ell-40",
|
|
"type": "T-ell",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.0,
|
|
"arm_accuracy": 0.375,
|
|
"paired_diff_arm_minus_a0": 0.375,
|
|
"pairwise": {
|
|
"arm_wins": 6,
|
|
"a0_wins": 0,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 7,
|
|
"arm": 1
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 1.0,
|
|
"without_mistral-large-latest": 0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ell-40",
|
|
"type": "T-ell",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.0,
|
|
"arm_accuracy": 0.375,
|
|
"paired_diff_arm_minus_a0": 0.375,
|
|
"pairwise": {
|
|
"arm_wins": 6,
|
|
"a0_wins": 0,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 6,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 1.0,
|
|
"without_mistral-large-latest": 0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-41",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.875,
|
|
"paired_diff_arm_minus_a0": -0.125,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 5,
|
|
"tie": 15
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -0.25,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 5,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.75,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-41",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.9,
|
|
"paired_diff_arm_minus_a0": -0.1,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 4,
|
|
"tie": 16
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -0.2,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 4,
|
|
"tie": 6
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.8,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-41",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-42",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.25,
|
|
"arm_accuracy": 0.25,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 5,
|
|
"a0_wins": 5,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 14,
|
|
"arm": 14
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 5,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.5,
|
|
"a0_acc": 0.5,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-42",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.25,
|
|
"arm_accuracy": 0.375,
|
|
"paired_diff_arm_minus_a0": 0.125,
|
|
"pairwise": {
|
|
"arm_wins": 10,
|
|
"a0_wins": 5,
|
|
"tie": 5
|
|
},
|
|
"catastrophe": {
|
|
"a0": 9,
|
|
"arm": 7
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.1,
|
|
"without_mistral-large-latest": 0.15
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 6,
|
|
"a0": 0,
|
|
"tie": 4
|
|
},
|
|
"BA": {
|
|
"arm": 4,
|
|
"a0": 5,
|
|
"tie": 1
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.25,
|
|
"a0_acc": 0.1,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.5,
|
|
"a0_acc": 0.4,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-42",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.325,
|
|
"arm_accuracy": 0.325,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 6,
|
|
"a0_wins": 5,
|
|
"tie": 9
|
|
},
|
|
"catastrophe": {
|
|
"a0": 12,
|
|
"arm": 12
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 1,
|
|
"a0": 5,
|
|
"tie": 4
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.15,
|
|
"a0_acc": 0.15,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.5,
|
|
"a0_acc": 0.5,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ell-43",
|
|
"type": "T-ell",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -1.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 11
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ell-43",
|
|
"type": "T-ell",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.125,
|
|
"paired_diff_arm_minus_a0": -0.875,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 8
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -0.75,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.25,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ell-43",
|
|
"type": "T-ell",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.042,
|
|
"paired_diff_arm_minus_a0": -0.958,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 9
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -0.917,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.083,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ana-44",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-src"
|
|
},
|
|
{
|
|
"trap": "T-ana-44",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-draft"
|
|
},
|
|
{
|
|
"trap": "T-ana-44",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ent-1",
|
|
"type": "T-ent",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.75,
|
|
"arm_accuracy": 0.75,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 5,
|
|
"a0_wins": 5,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 5,
|
|
"arm": 5
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 5,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 5,
|
|
"a0": 0,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.5,
|
|
"a0_acc": 0.5,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ent-3",
|
|
"type": "T-ent",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ent-4",
|
|
"type": "T-ent",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.85,
|
|
"arm_accuracy": 0.925,
|
|
"paired_diff_arm_minus_a0": 0.075,
|
|
"pairwise": {
|
|
"arm_wins": 5,
|
|
"a0_wins": 2,
|
|
"tie": 13
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.05,
|
|
"without_mistral-large-latest": 0.1
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 1,
|
|
"tie": 9
|
|
},
|
|
"BA": {
|
|
"arm": 5,
|
|
"a0": 1,
|
|
"tie": 4
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.85,
|
|
"a0_acc": 0.75,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0.95,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ent-6",
|
|
"type": "T-ent",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.975,
|
|
"arm_accuracy": 0.975,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.95,
|
|
"a0_acc": 0.95,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ent-7",
|
|
"type": "T-ent",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ent-8",
|
|
"type": "T-ent",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ent-10",
|
|
"type": "T-ent",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ent-11",
|
|
"type": "T-ent",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.75,
|
|
"paired_diff_arm_minus_a0": -0.25,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 10,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -0.25,
|
|
"without_mistral-large-latest": -0.25
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 1,
|
|
"tie": 9
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 9,
|
|
"tie": 1
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.75,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.75,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ent-12",
|
|
"type": "T-ent",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.167,
|
|
"arm_accuracy": 0.917,
|
|
"paired_diff_arm_minus_a0": 0.75,
|
|
"pairwise": {
|
|
"arm_wins": 12,
|
|
"a0_wins": 0,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 5,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.667,
|
|
"without_mistral-large-latest": 0.833
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 6,
|
|
"a0": 0,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 6,
|
|
"a0": 0,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.833,
|
|
"a0_acc": 0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0.333,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
},
|
|
{
|
|
"trap": "T-ent-13",
|
|
"type": "T-ent",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q3a",
|
|
"arm": "q3a-final"
|
|
}
|
|
]
|
|
} |