{
  "app": "jcode-bench-v1-gpt56",
  "benchmark_commit": "a9bfcdd9ed6cba355bef1025b552ee3da70ce2c0",
  "runner_commit": "11fb1990f41ca8bbb16996f3e1ac7cbbb8933c6e",
  "model": "gpt-5.6-sol",
  "reasoning_effort": "high",
  "swarm_concurrency": 8,
  "codex_version": "0.144.1",
  "jcode_version": "v0.41.1-dev (825c96f16)",
  "run_count": 12,
  "completed_count": 12,
  "runs": [
    {
      "agent": "codex",
      "swarm": false,
      "task": "json-unescape",
      "run_id": "20260710T205336Z-codex-solo-json-unescape",
      "function_call_id": "fc-01KX6WX9S4CS5V6PJ65H7ME5HQ",
      "status": "completed",
      "agent_exit_code": 0,
      "final_grade_exit_code": 0,
      "agent_duration_s": 1189.558,
      "grade_count": 21,
      "final_score": 2.4228,
      "best_score": 2.4228,
      "helper_events": 0
    },
    {
      "agent": "codex",
      "swarm": true,
      "task": "json-unescape",
      "run_id": "20260710T205336Z-codex-swarm-json-unescape",
      "function_call_id": "fc-01KX6WX9X5F1DVXR9WG9ZERC2Y",
      "status": "completed",
      "agent_exit_code": 0,
      "final_grade_exit_code": 0,
      "agent_duration_s": 1020.126,
      "grade_count": 22,
      "final_score": 2.608,
      "best_score": 2.6135,
      "helper_events": 0
    },
    {
      "agent": "jcode",
      "swarm": false,
      "task": "json-unescape",
      "run_id": "20260710T205523Z-jcode-solo-json-unescape",
      "function_call_id": "fc-01KX6X0J5JR9J6C7AENY9X6KCX",
      "status": "completed",
      "agent_exit_code": 0,
      "final_grade_exit_code": 0,
      "agent_duration_s": 554.391,
      "grade_count": 21,
      "final_score": 1.9157,
      "best_score": 1.9173,
      "helper_events": 0
    },
    {
      "agent": "jcode",
      "swarm": true,
      "task": "json-unescape",
      "run_id": "20260710T205523Z-jcode-swarm-json-unescape",
      "function_call_id": "fc-01KX6X0JA8SMJJADB47GSF8GDS",
      "status": "completed",
      "agent_exit_code": 0,
      "final_grade_exit_code": 0,
      "agent_duration_s": 561.931,
      "grade_count": 29,
      "final_score": 1.9172,
      "best_score": 1.921,
      "helper_events": 0
    },
    {
      "agent": "codex",
      "swarm": false,
      "task": "float-print",
      "run_id": "20260710T210724Z-codex-solo-float-print",
      "function_call_id": "fc-01KX6XPJ3W4CFPW9S6FPAWZVWY",
      "status": "completed",
      "agent_exit_code": 0,
      "final_grade_exit_code": 0,
      "agent_duration_s": 2531.193,
      "grade_count": 30,
      "final_score": 7.4165,
      "best_score": 7.4165,
      "helper_events": 0,
      "score_curve": [
        [
          0.0,
          0.0
        ],
        [
          1.617,
          6.0733
        ],
        [
          2.45,
          6.0064
        ],
        [
          2.867,
          6.0254
        ],
        [
          3.6,
          6.1009
        ],
        [
          4.333,
          6.1388
        ],
        [
          5.05,
          6.1258
        ],
        [
          5.983,
          6.0895
        ],
        [
          6.55,
          6.1354
        ],
        [
          7.25,
          6.4187
        ],
        [
          8.117,
          6.4495
        ],
        [
          8.983,
          6.4731
        ],
        [
          9.65,
          6.4214
        ],
        [
          10.4,
          6.4703
        ],
        [
          11.133,
          6.4868
        ],
        [
          12.067,
          6.526
        ],
        [
          12.533,
          6.542
        ],
        [
          13.1,
          6.5389
        ],
        [
          13.533,
          6.5413
        ],
        [
          14.15,
          6.5421
        ],
        [
          14.75,
          6.5503
        ],
        [
          15.333,
          6.596
        ],
        [
          16.267,
          6.5623
        ],
        [
          17.033,
          6.5971
        ],
        [
          17.65,
          6.6118
        ],
        [
          18.333,
          6.5624
        ],
        [
          19.117,
          6.547
        ],
        [
          19.667,
          6.613
        ],
        [
          42.133,
          6.6133
        ],
        [
          42.467,
          7.4165
        ]
      ]
    },
    {
      "agent": "codex",
      "swarm": true,
      "task": "float-print",
      "run_id": "20260710T205625Z-codex-swarm-float-print",
      "function_call_id": "fc-01KX6X2FBAJSXSKSZCDDBHHVTF",
      "status": "completed",
      "agent_exit_code": 0,
      "final_grade_exit_code": 0,
      "agent_duration_s": 2904.081,
      "grade_count": 38,
      "final_score": 7.5363,
      "best_score": 7.5363,
      "helper_events": 0,
      "score_curve": [
        [
          0.0,
          0.0
        ],
        [
          0.683,
          0.0005
        ],
        [
          2.367,
          5.8078
        ],
        [
          3.533,
          5.943
        ],
        [
          4.0,
          5.8237
        ],
        [
          5.267,
          6.0853
        ],
        [
          5.883,
          6.0945
        ],
        [
          6.283,
          6.0703
        ],
        [
          6.85,
          6.0305
        ],
        [
          7.833,
          6.1276
        ],
        [
          8.6,
          6.0576
        ],
        [
          9.817,
          6.4741
        ],
        [
          10.9,
          6.4617
        ],
        [
          11.417,
          6.4356
        ],
        [
          12.45,
          6.4844
        ],
        [
          13.4,
          6.465
        ],
        [
          13.817,
          6.4805
        ],
        [
          14.617,
          6.5214
        ],
        [
          15.217,
          6.5144
        ],
        [
          15.717,
          6.5171
        ],
        [
          16.383,
          6.5281
        ],
        [
          16.933,
          6.5426
        ],
        [
          17.167,
          6.5375
        ],
        [
          18.2,
          6.6122
        ],
        [
          18.633,
          6.6248
        ],
        [
          19.067,
          6.6322
        ],
        [
          19.483,
          6.6357
        ],
        [
          20.1,
          6.634
        ],
        [
          21.6,
          6.7012
        ],
        [
          22.117,
          6.7072
        ],
        [
          22.95,
          6.7213
        ],
        [
          23.283,
          6.7196
        ],
        [
          23.9,
          6.7293
        ],
        [
          24.317,
          6.7265
        ],
        [
          25.033,
          6.7287
        ],
        [
          25.75,
          6.7291
        ],
        [
          47.733,
          6.7293
        ],
        [
          48.683,
          7.5363
        ]
      ]
    },
    {
      "agent": "jcode",
      "swarm": false,
      "task": "float-print",
      "run_id": "20260710T205625Z-jcode-solo-float-print",
      "function_call_id": "fc-01KX6X2FFKAN82CA999XK4NY3Q",
      "status": "completed",
      "agent_exit_code": 0,
      "final_grade_exit_code": 0,
      "agent_duration_s": 2537.034,
      "grade_count": 48,
      "final_score": 7.1501,
      "best_score": 7.1501,
      "helper_events": 0,
      "score_curve": [
        [
          0.0,
          0.0
        ],
        [
          0.517,
          0.0
        ],
        [
          1.7,
          6.0021
        ],
        [
          2.317,
          6.3201
        ],
        [
          3.333,
          6.3201
        ],
        [
          3.783,
          6.3146
        ],
        [
          4.0,
          6.313
        ],
        [
          4.167,
          6.313
        ],
        [
          4.467,
          6.2271
        ],
        [
          4.633,
          6.2313
        ],
        [
          5.867,
          6.1122
        ],
        [
          6.017,
          6.1113
        ],
        [
          6.217,
          6.313
        ],
        [
          7.0,
          6.1828
        ],
        [
          7.167,
          6.1868
        ],
        [
          7.333,
          6.313
        ],
        [
          7.583,
          6.252
        ],
        [
          7.733,
          6.2593
        ],
        [
          8.033,
          6.2822
        ],
        [
          8.167,
          6.2862
        ],
        [
          8.317,
          6.3125
        ],
        [
          8.65,
          6.2985
        ],
        [
          8.8,
          6.2989
        ],
        [
          9.133,
          6.2412
        ],
        [
          9.283,
          6.2455
        ],
        [
          9.567,
          6.2509
        ],
        [
          9.717,
          6.2549
        ],
        [
          10.033,
          6.3307
        ],
        [
          10.2,
          6.332
        ],
        [
          10.517,
          6.3207
        ],
        [
          10.667,
          6.324
        ],
        [
          11.117,
          6.3335
        ],
        [
          11.283,
          6.3354
        ],
        [
          11.5,
          6.3335
        ],
        [
          11.667,
          6.3375
        ],
        [
          11.883,
          6.3429
        ],
        [
          12.033,
          6.3462
        ],
        [
          12.283,
          6.3285
        ],
        [
          12.433,
          6.3311
        ],
        [
          12.867,
          6.3196
        ],
        [
          13.017,
          6.3217
        ],
        [
          13.283,
          6.3429
        ],
        [
          13.717,
          6.3335
        ],
        [
          13.883,
          6.3384
        ],
        [
          14.067,
          6.3429
        ],
        [
          14.367,
          6.346
        ],
        [
          42.083,
          6.3452
        ],
        [
          42.483,
          7.1501
        ]
      ]
    },
    {
      "agent": "jcode",
      "swarm": true,
      "task": "float-print",
      "run_id": "20260710T205625Z-jcode-swarm-float-print",
      "function_call_id": "fc-01KX6X2FMDK5X8P0T9F8442EPP",
      "status": "completed",
      "agent_exit_code": 0,
      "final_grade_exit_code": 0,
      "agent_duration_s": 2233.628,
      "grade_count": 12,
      "final_score": 7.1052,
      "best_score": 7.1054,
      "helper_events": 0,
      "score_curve": [
        [
          0.0,
          0.0
        ],
        [
          0.55,
          0.0
        ],
        [
          1.6,
          6.7825
        ],
        [
          2.233,
          6.6795
        ],
        [
          2.933,
          7.0979
        ],
        [
          3.517,
          6.993
        ],
        [
          4.267,
          7.1041
        ],
        [
          4.7,
          7.1015
        ],
        [
          5.117,
          7.0205
        ],
        [
          5.683,
          7.0626
        ],
        [
          6.7,
          7.1054
        ],
        [
          37.517,
          7.1052
        ]
      ]
    },
    {
      "agent": "codex",
      "swarm": false,
      "task": "utf16-transcode",
      "run_id": "20260710T205625Z-codex-solo-utf16-transcode",
      "function_call_id": "fc-01KX6X2FSTJ03BNZZM27V1YKZ2",
      "status": "completed",
      "agent_exit_code": 0,
      "final_grade_exit_code": 0,
      "agent_duration_s": 1008.807,
      "grade_count": 24,
      "final_score": 1.8319,
      "best_score": 1.8327,
      "helper_events": 0
    },
    {
      "agent": "codex",
      "swarm": true,
      "task": "utf16-transcode",
      "run_id": "20260710T210726Z-codex-swarm-utf16-transcode",
      "function_call_id": "fc-01KX6XPMSCHJAJ5XHCN2T6R1G6",
      "status": "completed",
      "agent_exit_code": 0,
      "final_grade_exit_code": 0,
      "agent_duration_s": 1139.283,
      "grade_count": 29,
      "final_score": 2.4996,
      "best_score": 2.5029,
      "helper_events": 0
    },
    {
      "agent": "jcode",
      "swarm": false,
      "task": "utf16-transcode",
      "run_id": "20260710T205625Z-jcode-solo-utf16-transcode",
      "function_call_id": "fc-01KX6X2G2FY5HXPCWHGP2PNMY6",
      "status": "completed",
      "agent_exit_code": 0,
      "final_grade_exit_code": 0,
      "agent_duration_s": 493.192,
      "grade_count": 17,
      "final_score": 1.4711,
      "best_score": 1.4713,
      "helper_events": 0
    },
    {
      "agent": "jcode",
      "swarm": true,
      "task": "utf16-transcode",
      "run_id": "20260710T210858Z-jcode-swarm-utf16-transcode",
      "function_call_id": "fc-01KX6XSE8YQ4CDR51K20671M98",
      "status": "completed",
      "agent_exit_code": 0,
      "final_grade_exit_code": 0,
      "agent_duration_s": 590.284,
      "grade_count": 24,
      "final_score": 1.4653,
      "best_score": 1.4653,
      "helper_events": 0
    }
  ],
  "aggregates": [
    {
      "agent": "codex",
      "swarm": false,
      "completed_tasks": 3,
      "mean_final_score": 3.8904,
      "mean_best_score": 3.8907,
      "total_agent_duration_s": 4729.558,
      "helper_events": 0
    },
    {
      "agent": "codex",
      "swarm": true,
      "completed_tasks": 3,
      "mean_final_score": 4.2146,
      "mean_best_score": 4.2176,
      "total_agent_duration_s": 5063.49,
      "helper_events": 0
    },
    {
      "agent": "jcode",
      "swarm": false,
      "completed_tasks": 3,
      "mean_final_score": 3.5123,
      "mean_best_score": 3.5129,
      "total_agent_duration_s": 3584.617,
      "helper_events": 0
    },
    {
      "agent": "jcode",
      "swarm": true,
      "completed_tasks": 3,
      "mean_final_score": 3.4959,
      "mean_best_score": 3.4972,
      "total_agent_duration_s": 3385.843,
      "helper_events": 0
    }
  ],
  "comparisons": {
    "codex_swarm_vs_solo": {
      "mean_score_delta": 0.3242,
      "geomean_efficiency_factor": 1.252,
      "total_agent_time_delta_percent": 7.06
    },
    "jcode_swarm_vs_solo": {
      "mean_score_delta": -0.0164,
      "geomean_efficiency_factor": 0.9887,
      "total_agent_time_delta_percent": -5.55
    },
    "codex_vs_jcode_solo": {
      "mean_score_delta": 0.3781,
      "geomean_efficiency_factor": 1.2996,
      "total_agent_time_delta_percent": 31.94
    },
    "codex_vs_jcode_swarm": {
      "mean_score_delta": 0.7187,
      "geomean_efficiency_factor": 1.6457,
      "total_agent_time_delta_percent": 49.55
    }
  }
}