1445 lines
No EOL
28 KiB
JSON
1445 lines
No EOL
28 KiB
JSON
{
|
|
"n": 30,
|
|
"results": [
|
|
{
|
|
"trap": "T-ana-2",
|
|
"type": "T-ana",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.125,
|
|
"arm_accuracy": 0.562,
|
|
"paired_diff_arm_minus_a0": 0.438,
|
|
"pairwise": {
|
|
"arm_wins": 7,
|
|
"a0_wins": 0,
|
|
"tie": 9
|
|
},
|
|
"catastrophe": {
|
|
"a0": 12,
|
|
"arm": 6
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 1.0,
|
|
"without_mistral-large-latest": 0.1
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 4,
|
|
"a0": 0,
|
|
"tie": 4
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.3,
|
|
"a0_acc": 0.2,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-5",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -1.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 9
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-9",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -1.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 12
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-16",
|
|
"type": "T-ana",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.594,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -0.594,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 14,
|
|
"tie": 2
|
|
},
|
|
"catastrophe": {
|
|
"a0": 3,
|
|
"arm": 15
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -0.35
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 2
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 8,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 0.35,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-17",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 1.0,
|
|
"pairwise": {
|
|
"arm_wins": 12,
|
|
"a0_wins": 0,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 12,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 1.0,
|
|
"without_mistral-large-latest": 1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 6,
|
|
"a0": 0,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 6,
|
|
"a0": 0,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-22",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.917,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -0.917,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 10
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -0.833
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 0.833,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-23",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -1.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 10
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-31",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -1.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 12
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-32",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.0,
|
|
"arm_accuracy": 0.75,
|
|
"paired_diff_arm_minus_a0": 0.75,
|
|
"pairwise": {
|
|
"arm_wins": 12,
|
|
"a0_wins": 0,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 7,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 1.0,
|
|
"without_mistral-large-latest": 0.5
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 6,
|
|
"a0": 0,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 6,
|
|
"a0": 0,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.5,
|
|
"a0_acc": 0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-34",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-35",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -1.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 11
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-36",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.708,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -0.708,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 12
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -0.417
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 0.417,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-37",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -1.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 12
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-38",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -1.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 12
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-39",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.792,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -0.792,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 9
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -0.583
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 0.583,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ell-40",
|
|
"type": "T-ell",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.667,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -0.667,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 8
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -0.333
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 0.333,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-41",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -1.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 10
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-42",
|
|
"type": "T-ana",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.325,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -0.325,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 11,
|
|
"tie": 9
|
|
},
|
|
"catastrophe": {
|
|
"a0": 9,
|
|
"arm": 16
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -0.5,
|
|
"without_mistral-large-latest": -0.15
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 5,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 4
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 0.15,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 0.5,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ell-43",
|
|
"type": "T-ell",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -1.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 11
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ana-44",
|
|
"type": "T-ana",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -1.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 12
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ent-1",
|
|
"type": "T-ent",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.75,
|
|
"arm_accuracy": 0.938,
|
|
"paired_diff_arm_minus_a0": 0.188,
|
|
"pairwise": {
|
|
"arm_wins": 5,
|
|
"a0_wins": 1,
|
|
"tie": 10
|
|
},
|
|
"catastrophe": {
|
|
"a0": 3,
|
|
"arm": 1
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.5,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 2,
|
|
"a0": 1,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.833,
|
|
"a0_acc": 0.333,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ent-3",
|
|
"type": "T-ent",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -1.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 10
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ent-4",
|
|
"type": "T-ent",
|
|
"n_votes": 12,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.0,
|
|
"paired_diff_arm_minus_a0": -1.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 12,
|
|
"tie": 0
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 8
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -1.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ent-6",
|
|
"type": "T-ent",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.0,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 0,
|
|
"tie": 20
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ent-7",
|
|
"type": "T-ent",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.75,
|
|
"paired_diff_arm_minus_a0": -0.25,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 5,
|
|
"tie": 15
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 5
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -0.5,
|
|
"without_mistral-large-latest": 0.0
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 5,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.5,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ent-8",
|
|
"type": "T-ent",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.975,
|
|
"arm_accuracy": 0.45,
|
|
"paired_diff_arm_minus_a0": -0.525,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 11,
|
|
"tie": 9
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 10
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -0.5,
|
|
"without_mistral-large-latest": -0.55
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 5,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 4
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.4,
|
|
"a0_acc": 0.95,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.5,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ent-10",
|
|
"type": "T-ent",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.656,
|
|
"paired_diff_arm_minus_a0": -0.344,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 7,
|
|
"tie": 9
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 4
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -0.75,
|
|
"without_mistral-large-latest": -0.1
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 4,
|
|
"tie": 4
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 3,
|
|
"tie": 5
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.9,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0.25,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ent-11",
|
|
"type": "T-ent",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.344,
|
|
"paired_diff_arm_minus_a0": -0.656,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 11,
|
|
"tie": 5
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 10
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": -1.0,
|
|
"without_mistral-large-latest": -0.45
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 5,
|
|
"tie": 3
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 6,
|
|
"tie": 2
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.55,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 0,
|
|
"a0_acc": 1.0,
|
|
"n": 6
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ent-12",
|
|
"type": "T-ent",
|
|
"n_votes": 16,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 0.344,
|
|
"arm_accuracy": 1.0,
|
|
"paired_diff_arm_minus_a0": 0.656,
|
|
"pairwise": {
|
|
"arm_wins": 11,
|
|
"a0_wins": 0,
|
|
"tie": 5
|
|
},
|
|
"catastrophe": {
|
|
"a0": 6,
|
|
"arm": 0
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.5,
|
|
"without_mistral-large-latest": 0.917
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 3,
|
|
"a0": 0,
|
|
"tie": 5
|
|
},
|
|
"BA": {
|
|
"arm": 8,
|
|
"a0": 0,
|
|
"tie": 0
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0.083,
|
|
"n": 6
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 0.5,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
},
|
|
{
|
|
"trap": "T-ent-13",
|
|
"type": "T-ent",
|
|
"n_votes": 20,
|
|
"parse_fail": 0,
|
|
"a0_accuracy": 1.0,
|
|
"arm_accuracy": 0.85,
|
|
"paired_diff_arm_minus_a0": -0.15,
|
|
"pairwise": {
|
|
"arm_wins": 0,
|
|
"a0_wins": 4,
|
|
"tie": 16
|
|
},
|
|
"catastrophe": {
|
|
"a0": 0,
|
|
"arm": 1
|
|
},
|
|
"leave_one_judge_out": {
|
|
"without_grok-4.3": 0.0,
|
|
"without_mistral-large-latest": -0.3
|
|
},
|
|
"order_bias": {
|
|
"AB": {
|
|
"arm": 0,
|
|
"a0": 0,
|
|
"tie": 10
|
|
},
|
|
"BA": {
|
|
"arm": 0,
|
|
"a0": 4,
|
|
"tie": 6
|
|
}
|
|
},
|
|
"per_judge": {
|
|
"grok-4.3": {
|
|
"arm_acc": 0.7,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
},
|
|
"mistral-large-latest": {
|
|
"arm_acc": 1.0,
|
|
"a0_acc": 1.0,
|
|
"n": 10
|
|
}
|
|
},
|
|
"block": "Q1b",
|
|
"arm": "oracle"
|
|
}
|
|
]
|
|
} |