{ "benchmark": { "name": "DeepSWE", "scoring": "DataCurve official scoring", "tasks": 113, "version": "v1.1" }, "effort_rows": [ { "ci_half": 0.0238819467145892, "ci_hi": 0.47742176972343875, "ci_lo": 0.42965787629426033, "mean_agent_steps": 23.358407079646017, "mean_cost_usd": 1.0742664646017699, "mean_output_tokens": 10579.141592920354, "n_attempted": 452, "n_passed": 205, "n_runs": 4, "n_tasks_attempted": 113, "pass_rate": 0.45353982300884954, "reasoning_effort": "low" }, { "ci_half": 0.01583357649307215, "ci_hi": 0.6264530455196208, "ci_lo": 0.5947858925334765, "mean_agent_steps": 30.913716814159294, "mean_cost_usd": 1.8620318075221238, "mean_output_tokens": 18425.216814159292, "n_attempted": 452, "n_passed": 276, "n_runs": 4, "n_tasks_attempted": 113, "pass_rate": 0.6106194690265486, "reasoning_effort": "medium" }, { "ci_half": 0.014321452203772355, "ci_hi": 0.7083347559731736, "ci_lo": 0.679691851565629, "mean_agent_steps": 36.889135254988915, "mean_cost_usd": 3.4698331108647453, "mean_output_tokens": 28450.31929046563, "n_attempted": 451, "n_passed": 313, "n_runs": 4, "n_tasks_attempted": 113, "pass_rate": 0.6940133037694013, "reasoning_effort": "high" }, { "ci_half": 0.008191117217343103, "ci_hi": 0.7155081903880748, "ci_lo": 0.6991259559533886, "mean_agent_steps": 44.0, "mean_cost_usd": 4.703663860310422, "mean_output_tokens": 40744.59201773836, "n_attempted": 451, "n_passed": 319, "n_runs": 4, "n_tasks_attempted": 113, "pass_rate": 0.7073170731707317, "reasoning_effort": "xhigh" }, { "ci_half": 0.02829822249837175, "ci_hi": 0.7549648891650385, "ci_lo": 0.6983684441682949, "mean_agent_steps": 61.25333333333333, "mean_cost_usd": 8.386436346666667, "mean_output_tokens": 60013.64444444444, "n_attempted": 450, "n_passed": 327, "n_runs": 4, "n_tasks_attempted": 113, "pass_rate": 0.7266666666666667, "reasoning_effort": "max" } ], "generated_at": "2026-07-17", "harness": "mini-swe-agent", "high_per_task": [ { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "abs-module-cache-flags__8wyDK9j" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "abs-module-cache-flags__GhyyLQB" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "abs-module-cache-flags__XFYUpsy" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "abs-module-cache-flags__ydUEWnX" } ], "pass_rate": 0.75, "passes": 3, "task": "abs-module-cache-flags", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "abs-stepped-slices__ajvrsDA" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "abs-stepped-slices__hffCKsJ" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "abs-stepped-slices__o8BqiiM" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "abs-stepped-slices__vjgipDk" } ], "pass_rate": 1.0, "passes": 4, "task": "abs-stepped-slices", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "actionlint-action-pinning-lint__3u3dqdz" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "actionlint-action-pinning-lint__HN9cLyE" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "actionlint-action-pinning-lint__qEuCH9U" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "actionlint-action-pinning-lint__th937iW" } ], "pass_rate": 0.75, "passes": 3, "task": "actionlint-action-pinning-lint", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "adaptix-name-mapping-aliases__37vL2eX" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "adaptix-name-mapping-aliases__CMXYnAR" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "adaptix-name-mapping-aliases__FaK5eYC" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "adaptix-name-mapping-aliases__tU3yga6" } ], "pass_rate": 1.0, "passes": 4, "task": "adaptix-name-mapping-aliases", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "aiomonitor-task-snapshots-diff__PNnRbi4" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "aiomonitor-task-snapshots-diff__fhkFZkz" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "aiomonitor-task-snapshots-diff__p4yTjLt" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "aiomonitor-task-snapshots-diff__pFmKD3q" } ], "pass_rate": 1.0, "passes": 4, "task": "aiomonitor-task-snapshots-diff", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "anko-default-function-arguments__Ar5qDFm" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "anko-default-function-arguments__HrpExEJ" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "anko-default-function-arguments__TdTjti8" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "anko-default-function-arguments__rF9PASo" } ], "pass_rate": 1.0, "passes": 4, "task": "anko-default-function-arguments", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "anko-typed-variable-bindings__3ig4gQP" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "anko-typed-variable-bindings__4AZV4r4" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "anko-typed-variable-bindings__S4w9YQR" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "anko-typed-variable-bindings__fGq2Ryx" } ], "pass_rate": 0.5, "passes": 2, "task": "anko-typed-variable-bindings", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "arcane-drift-detection-baselines__BpxXAtU" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "arcane-drift-detection-baselines__PQUs8qy" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "arcane-drift-detection-baselines__dAiAs3J" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "arcane-drift-detection-baselines__mjwU3FN" } ], "pass_rate": 1.0, "passes": 4, "task": "arcane-drift-detection-baselines", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "arktype-json-schema-refs-depende__HeUG6ed" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "arktype-json-schema-refs-depende__Jxa5k7o" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "arktype-json-schema-refs-depende__nCCukMT" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "arktype-json-schema-refs-depende__pCxzu2h" } ], "pass_rate": 0.5, "passes": 2, "task": "arktype-json-schema-refs-dependencies", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "awilix-async-container-initializ__89FuSZ2" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "awilix-async-container-initializ__GwQ5NME" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "awilix-async-container-initializ__oGh2p5H" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "awilix-async-container-initializ__pvTfbxp" } ], "pass_rate": 0.25, "passes": 1, "task": "awilix-async-container-initialization", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "bandit-incremental-cache-control__4N5B3ZT" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "bandit-incremental-cache-control__BxYhSKA" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "bandit-incremental-cache-control__XXhzzjW" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "bandit-incremental-cache-control__zJqSGGw" } ], "pass_rate": 0.75, "passes": 3, "task": "bandit-incremental-cache-control", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "bandit-interprocedural-taint-che__3pkwUJ2" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "bandit-interprocedural-taint-che__Raje9n9" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "bandit-interprocedural-taint-che__hMYLVQe" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "bandit-interprocedural-taint-che__jixVjvd" } ], "pass_rate": 1.0, "passes": 4, "task": "bandit-interprocedural-taint-checks", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "bandit-structured-nosec-directiv__5NKsQPK" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "bandit-structured-nosec-directiv__NF4JuXa" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "bandit-structured-nosec-directiv__bJsBahe" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "bandit-structured-nosec-directiv__orowa7S" } ], "pass_rate": 0.0, "passes": 0, "task": "bandit-structured-nosec-directives", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "boa-hierarchical-evaluation-canc__VYyKGW5" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "boa-hierarchical-evaluation-canc__Y5noqnB" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "boa-hierarchical-evaluation-canc__kSspc8a" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "boa-hierarchical-evaluation-canc__ziAPEUu" } ], "pass_rate": 0.5, "passes": 2, "task": "boa-hierarchical-evaluation-cancellation", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "cattrs-partial-structuring-recov__7jKJkDG" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "cattrs-partial-structuring-recov__QULr54J" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "cattrs-partial-structuring-recov__qsa8Cex" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "cattrs-partial-structuring-recov__vMG3ULs" } ], "pass_rate": 1.0, "passes": 4, "task": "cattrs-partial-structuring-recovery", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "clack-async-autocomplete-options__BbiszZu" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "clack-async-autocomplete-options__GgjwTex" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "clack-async-autocomplete-options__QHzqZyM" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "clack-async-autocomplete-options__emwJUj9" } ], "pass_rate": 0.0, "passes": 0, "task": "clack-async-autocomplete-options", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "claude-code-by-agents-recursive__2mggmXA" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "claude-code-by-agents-recursive__Hi9oD4Q" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "claude-code-by-agents-recursive__fw8wz2U" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "claude-code-by-agents-recursive__kzHVzTs" } ], "pass_rate": 1.0, "passes": 4, "task": "claude-code-by-agents-recursive-delegation", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "cliffy-config-file-parsing__2gZZxgb" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "cliffy-config-file-parsing__WdCQzhA" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "cliffy-config-file-parsing__inQVM3x" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "cliffy-config-file-parsing__zWCNzWT" } ], "pass_rate": 0.5, "passes": 2, "task": "cliffy-config-file-parsing", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "csstree-shorthand-expansion-comp__J7AnBgC" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "csstree-shorthand-expansion-comp__RqcicAo" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "csstree-shorthand-expansion-comp__bYLeyxo" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "csstree-shorthand-expansion-comp__qAN8vQY" } ], "pass_rate": 1.0, "passes": 4, "task": "csstree-shorthand-expansion-compression", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "dasel-html-document-format__SfdXkvA" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "dasel-html-document-format__T8EXjAu" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "dasel-html-document-format__Vkh4zZ4" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "dasel-html-document-format__YuQ2bkn" } ], "pass_rate": 0.75, "passes": 3, "task": "dasel-html-document-format", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "dateutil-rfc5545-timezone-intero__7GkEahp" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "dateutil-rfc5545-timezone-intero__pHUdLb7" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "dateutil-rfc5545-timezone-intero__rDKcHVJ" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "dateutil-rfc5545-timezone-intero__zDmnQPg" } ], "pass_rate": 1.0, "passes": 4, "task": "dateutil-rfc5545-timezone-interop", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "drizzle-orm-window-function-buil__7oYnRKP" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "drizzle-orm-window-function-buil__8AZhcgb" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "drizzle-orm-window-function-buil__jDAHbQX" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "drizzle-orm-window-function-buil__jzr86Yq" } ], "pass_rate": 1.0, "passes": 4, "task": "drizzle-orm-window-function-builders", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "dynamodb-toolbox-conditional-att__6iGEh7J" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "dynamodb-toolbox-conditional-att__J92Z29T" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "dynamodb-toolbox-conditional-att__WWPyi4W" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "dynamodb-toolbox-conditional-att__ZBW2DmG" } ], "pass_rate": 1.0, "passes": 4, "task": "dynamodb-toolbox-conditional-attribute-requirements", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "dynamodb-toolbox-lazy-recursive__Evsv7Gx" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "dynamodb-toolbox-lazy-recursive__MyT39d6" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "dynamodb-toolbox-lazy-recursive__NzNTa63" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "dynamodb-toolbox-lazy-recursive__XUZy839" } ], "pass_rate": 1.0, "passes": 4, "task": "dynamodb-toolbox-lazy-recursive-schemas", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "effect-sse-httpapi-streaming__7UtqUEa" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "effect-sse-httpapi-streaming__Q3ztbux" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "effect-sse-httpapi-streaming__nEmWJXV" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "effect-sse-httpapi-streaming__sm6z247" } ], "pass_rate": 0.5, "passes": 2, "task": "effect-sse-httpapi-streaming", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "eicrud-keyset-pagination-cursor__5kmKiUk" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "eicrud-keyset-pagination-cursor__ZVgTpzp" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "eicrud-keyset-pagination-cursor__cGV2XGF" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "eicrud-keyset-pagination-cursor__wtu5r9D" } ], "pass_rate": 0.25, "passes": 1, "task": "eicrud-keyset-pagination-cursor", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "etree-xml-diff-patch__27hxCVY" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "etree-xml-diff-patch__8MTcKv7" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "etree-xml-diff-patch__QNmEnRY" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "etree-xml-diff-patch__fWPBBDV" } ], "pass_rate": 1.0, "passes": 4, "task": "etree-xml-diff-patch", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "expr-try-catch-errors__PLZP9Jn" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "expr-try-catch-errors__VHxMtpA" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "expr-try-catch-errors__b8VYvJA" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "expr-try-catch-errors__rGQXmWa" } ], "pass_rate": 1.0, "passes": 4, "task": "expr-try-catch-errors", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "fastapi-deprecation-response-hea__ChyGMka" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "fastapi-deprecation-response-hea__K9mBFnn" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "fastapi-deprecation-response-hea__PXqLmBn" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "fastapi-deprecation-response-hea__S4y9u28" } ], "pass_rate": 0.5, "passes": 2, "task": "fastapi-deprecation-response-headers", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "fastapi-implicit-head-options__C3G6NE5" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "fastapi-implicit-head-options__S9VYZMy" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "fastapi-implicit-head-options__ooTfdiX" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "fastapi-implicit-head-options__rLiEWWH" } ], "pass_rate": 0.5, "passes": 2, "task": "fastapi-implicit-head-options", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "fd-deterministic-multi-key-sorti__HuF5Lgp" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "fd-deterministic-multi-key-sorti__HwTeqPV" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "fd-deterministic-multi-key-sorti__aX4fH9T" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "fd-deterministic-multi-key-sorti__vchcoqP" } ], "pass_rate": 1.0, "passes": 4, "task": "fd-deterministic-multi-key-sorting", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "geo-shapeindex-serialization__8BZbSUN" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "geo-shapeindex-serialization__Zvc2seU" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "geo-shapeindex-serialization__sRnmpxd" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "geo-shapeindex-serialization__viRmTYa" } ], "pass_rate": 1.0, "passes": 4, "task": "geo-shapeindex-serialization", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "go-critic-doc-link-checker__8DUdwF4" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "go-critic-doc-link-checker__SzDCDA8" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "go-critic-doc-link-checker__aYcxREL" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "go-critic-doc-link-checker__fNqSmWA" } ], "pass_rate": 0.5, "passes": 2, "task": "go-critic-doc-link-checker", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "go-genai-streamed-function-args__9ptMcvr" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "go-genai-streamed-function-args__e6RGJCK" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "go-genai-streamed-function-args__gnXqdYh" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "go-genai-streamed-function-args__nxgedQP" } ], "pass_rate": 1.0, "passes": 4, "task": "go-genai-streamed-function-args", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "go-git-worktree-merge-conflicts__3sM972Q" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "go-git-worktree-merge-conflicts__DbAHWRz" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "go-git-worktree-merge-conflicts__RKXAvDt" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "go-git-worktree-merge-conflicts__w3MXeDN" } ], "pass_rate": 1.0, "passes": 4, "task": "go-git-worktree-merge-conflicts", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "goreleaser-retry-publish-auditin__Ko7UHsf" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "goreleaser-retry-publish-auditin__SSwFsNn" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "goreleaser-retry-publish-auditin__ZGAohNk" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "goreleaser-retry-publish-auditin__abCu3rP" } ], "pass_rate": 1.0, "passes": 4, "task": "goreleaser-retry-publish-auditing", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "gql-incremental-graphql-delivery__GxWgRS8" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "gql-incremental-graphql-delivery__s7csNWY" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "gql-incremental-graphql-delivery__xQ9bTMP" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "gql-incremental-graphql-delivery__zfUG6rM" } ], "pass_rate": 0.0, "passes": 0, "task": "gql-incremental-graphql-delivery", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "happy-dom-abort-pending-body-rea__QWxr2X7" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "happy-dom-abort-pending-body-rea__kUAdThW" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "happy-dom-abort-pending-body-rea__p9yozYu" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "happy-dom-abort-pending-body-rea__rc44LBU" } ], "pass_rate": 1.0, "passes": 4, "task": "happy-dom-abort-pending-body-reads", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "happy-dom-deterministic-intersec__3iBAqh3" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "happy-dom-deterministic-intersec__7zaoKLY" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "happy-dom-deterministic-intersec__YUrXyfC" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "happy-dom-deterministic-intersec__d3ayGAN" } ], "pass_rate": 1.0, "passes": 4, "task": "happy-dom-deterministic-intersectionobserver", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "helm-array-merge-strategies__Hew5wHY" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "helm-array-merge-strategies__QprHCas" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "helm-array-merge-strategies__k5aezqm" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "helm-array-merge-strategies__wVKRWzu" } ], "pass_rate": 0.0, "passes": 0, "task": "helm-array-merge-strategies", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "helm-unified-manifest-stream__3WoFCun" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "helm-unified-manifest-stream__M9iVdwq" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "helm-unified-manifest-stream__PribGhB" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "helm-unified-manifest-stream__prvyFRh" } ], "pass_rate": 0.75, "passes": 3, "task": "helm-unified-manifest-stream", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "httpx-deterministic-cookie-store__FA3hruU" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "httpx-deterministic-cookie-store__Gry6UPE" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "httpx-deterministic-cookie-store__kfVZwHp" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "httpx-deterministic-cookie-store__w7sJ6mQ" } ], "pass_rate": 1.0, "passes": 4, "task": "httpx-deterministic-cookie-store", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "httpx-multipart-response-parsing__77J6YN2" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "httpx-multipart-response-parsing__AYBfF3A" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "httpx-multipart-response-parsing__FVGyYq6" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "httpx-multipart-response-parsing__iXscexG" } ], "pass_rate": 0.75, "passes": 3, "task": "httpx-multipart-response-parsing", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "httpx-streaming-json-iteration__6ZmRyyq" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "httpx-streaming-json-iteration__DUEnEFr" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "httpx-streaming-json-iteration__GCtakXN" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "httpx-streaming-json-iteration__PW5MprM" } ], "pass_rate": 1.0, "passes": 4, "task": "httpx-streaming-json-iteration", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "igel-persist-feature-schema__GDXcQyB" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "igel-persist-feature-schema__Gsq3e4x" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "igel-persist-feature-schema__exdjb6R" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "igel-persist-feature-schema__jnqK6FB" } ], "pass_rate": 0.5, "passes": 2, "task": "igel-persist-feature-schema", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "ink-grid-box-layout__MQEB2mo" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "ink-grid-box-layout__MqRmVmn" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "ink-grid-box-layout__gd4SBYJ" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "ink-grid-box-layout__zfLEGfX" } ], "pass_rate": 0.5, "passes": 2, "task": "ink-grid-box-layout", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "ipython-session-bundle-replay__42sng9X" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "ipython-session-bundle-replay__6HnLxio" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "ipython-session-bundle-replay__bAWA53z" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "ipython-session-bundle-replay__jbbfmS7" } ], "pass_rate": 0.75, "passes": 3, "task": "ipython-session-bundle-replay", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "katex-multicolumn-array-spans__AdGAJft" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "katex-multicolumn-array-spans__BQP2Zg2" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "katex-multicolumn-array-spans__ENfjXra" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "katex-multicolumn-array-spans__c9ihphT" } ], "pass_rate": 0.75, "passes": 3, "task": "katex-multicolumn-array-spans", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kcp-go-multiplexed-kcp-streams__2tbA4C9" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kcp-go-multiplexed-kcp-streams__GLaHFDC" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kcp-go-multiplexed-kcp-streams__MMSb9K7" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kcp-go-multiplexed-kcp-streams__TXW57ur" } ], "pass_rate": 1.0, "passes": 4, "task": "kcp-go-multiplexed-kcp-streams", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kea-atomic-signal-selectors__AUzDnRv" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kea-atomic-signal-selectors__F2eWqLn" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kea-atomic-signal-selectors__Pa94fm7" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "kea-atomic-signal-selectors__xEfzUCv" } ], "pass_rate": 0.75, "passes": 3, "task": "kea-atomic-signal-selectors", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "kgateway-consistent-hash-policy__AL5z6dF" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "kgateway-consistent-hash-policy__LyAtAzg" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kgateway-consistent-hash-policy__Qeh3xUQ" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kgateway-consistent-hash-policy__WGZqoJp" } ], "pass_rate": 0.5, "passes": 2, "task": "kgateway-consistent-hash-policy", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kombu-single-active-consumer-pri__ABLcBMP" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kombu-single-active-consumer-pri__J8CGbMH" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kombu-single-active-consumer-pri__XtPEqN5" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kombu-single-active-consumer-pri__mb3FrUd" } ], "pass_rate": 1.0, "passes": 4, "task": "kombu-single-active-consumer-priority", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "kombu-virtual-queue-dead-letteri__EezGR5a" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "kombu-virtual-queue-dead-letteri__RS2yRv9" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "kombu-virtual-queue-dead-letteri__fDxTM6q" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "kombu-virtual-queue-dead-letteri__wvurmiC" } ], "pass_rate": 0.0, "passes": 0, "task": "kombu-virtual-queue-dead-lettering", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "koota-composite-trait-aspects__LfDAfst" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "koota-composite-trait-aspects__YMZULqR" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "koota-composite-trait-aspects__bXgj3sf" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "koota-composite-trait-aspects__uYrCL4Q" } ], "pass_rate": 1.0, "passes": 4, "task": "koota-composite-trait-aspects", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "koota-deferred-mutation-buffer__2z9Z4jc" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "koota-deferred-mutation-buffer__BUej39Z" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "koota-deferred-mutation-buffer__hYdk4CU" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "koota-deferred-mutation-buffer__joNfpnx" } ], "pass_rate": 0.5, "passes": 2, "task": "koota-deferred-mutation-buffer", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "koota-entity-snapshot-rollback__SMgs9ez" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "koota-entity-snapshot-rollback__Xx2sZmW" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "koota-entity-snapshot-rollback__txeT2x8" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "koota-entity-snapshot-rollback__vjrvQY2" } ], "pass_rate": 0.75, "passes": 3, "task": "koota-entity-snapshot-rollback", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "koota-pair-relation-tracking__NSf9xAE" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "koota-pair-relation-tracking__UFnT2ko" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "koota-pair-relation-tracking__dZCdmyU" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "koota-pair-relation-tracking__oZZxPje" } ], "pass_rate": 0.0, "passes": 0, "task": "koota-pair-relation-tracking", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "koota-query-predicates__DmoGk7x" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "koota-query-predicates__YaZYXAU" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "koota-query-predicates__cGoKjYF" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "koota-query-predicates__ujZQUxD" } ], "pass_rate": 0.0, "passes": 0, "task": "koota-query-predicates", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kysely-window-grouping-helpers__4iry3Sf" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kysely-window-grouping-helpers__YAPRJTZ" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kysely-window-grouping-helpers__hmzdgMi" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "kysely-window-grouping-helpers__tp2rF7x" } ], "pass_rate": 1.0, "passes": 4, "task": "kysely-window-grouping-helpers", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "langchain-request-coalescing__2DBHxyp" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "langchain-request-coalescing__MZryp3b" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "langchain-request-coalescing__fYjbbhJ" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "langchain-request-coalescing__hjM84vj" } ], "pass_rate": 0.25, "passes": 1, "task": "langchain-request-coalescing", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "mashumaro-flattened-dataclass-fi__7Y4JsWh" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "mashumaro-flattened-dataclass-fi__RcWJE28" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "mashumaro-flattened-dataclass-fi__UYNoQdC" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "mashumaro-flattened-dataclass-fi__Vw6hfJS" } ], "pass_rate": 1.0, "passes": 4, "task": "mashumaro-flattened-dataclass-fields", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "meriyah-explicit-resource-declar__ChVyJLR" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "meriyah-explicit-resource-declar__LbPpyP9" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "meriyah-explicit-resource-declar__fUmAmUk" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "meriyah-explicit-resource-declar__x44Zgpz" } ], "pass_rate": 0.25, "passes": 1, "task": "meriyah-explicit-resource-declarations", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "mnamer-daemon-watch-lifecycle__EG2MSyU" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "mnamer-daemon-watch-lifecycle__YQbjZeD" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "mnamer-daemon-watch-lifecycle__eFUaRWY" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "mnamer-daemon-watch-lifecycle__yny9fBs" } ], "pass_rate": 1.0, "passes": 4, "task": "mnamer-daemon-watch-lifecycle", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "mobly-grouped-test-barriers__Bqq4ZEu" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "mobly-grouped-test-barriers__Le2akKa" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "mobly-grouped-test-barriers__a9YBek2" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "mobly-grouped-test-barriers__cp2rfQ6" } ], "pass_rate": 1.0, "passes": 4, "task": "mobly-grouped-test-barriers", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "narwhals-rolling-window-suite__7ae4Pqi" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "narwhals-rolling-window-suite__U4Cvyxp" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "narwhals-rolling-window-suite__XYnj4e3" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "narwhals-rolling-window-suite__XZQtMii" } ], "pass_rate": 1.0, "passes": 4, "task": "narwhals-rolling-window-suite", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "numba-stencil-boundary-modes__8hj4rdM" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "numba-stencil-boundary-modes__8z9CjaG" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "numba-stencil-boundary-modes__qC3dcVQ" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "numba-stencil-boundary-modes__rqJ59as" } ], "pass_rate": 0.75, "passes": 3, "task": "numba-stencil-boundary-modes", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "obsidian-linter-auto-table-of-co__FVfXGfZ" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "obsidian-linter-auto-table-of-co__LKPXycx" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "obsidian-linter-auto-table-of-co__VJx5j4s" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "obsidian-linter-auto-table-of-co__gQhLXc5" } ], "pass_rate": 0.0, "passes": 0, "task": "obsidian-linter-auto-table-of-contents", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "obsidian-linter-link-format-conv__3hmKoCL" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "obsidian-linter-link-format-conv__79666Ab" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "obsidian-linter-link-format-conv__J3AfGLL" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "obsidian-linter-link-format-conv__fQrFJ5H" } ], "pass_rate": 0.5, "passes": 2, "task": "obsidian-linter-link-format-conversion", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "obsidian-linter-scoped-ignore-ma__YMkwQzD" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "obsidian-linter-scoped-ignore-ma__ZBXhCYs" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "obsidian-linter-scoped-ignore-ma__tVyiPck" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "obsidian-linter-scoped-ignore-ma__u33XBVZ" } ], "pass_rate": 1.0, "passes": 4, "task": "obsidian-linter-scoped-ignore-markers", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "ofetch-per-origin-circuit-breake__8t8iZcj" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "ofetch-per-origin-circuit-breake__LqYdHhm" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "ofetch-per-origin-circuit-breake__PhaSURe" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "ofetch-per-origin-circuit-breake__uZyAMi6" } ], "pass_rate": 1.0, "passes": 4, "task": "ofetch-per-origin-circuit-breaker", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "onedump-dump-encryption-pipeline__CQyy8SN" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "onedump-dump-encryption-pipeline__R76kTPr" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "onedump-dump-encryption-pipeline__YfF9pyV" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "onedump-dump-encryption-pipeline__ZdcLdGq" } ], "pass_rate": 0.0, "passes": 0, "task": "onedump-dump-encryption-pipeline", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "opa-rego-rule-profiling__TAck6LX" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "opa-rego-rule-profiling__ZEChRGk" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "opa-rego-rule-profiling__hFXtnoi" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "opa-rego-rule-profiling__riqcqGa" } ], "pass_rate": 0.5, "passes": 2, "task": "opa-rego-rule-profiling", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "opa-template-string-reconstructi__9vgdzvn" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "opa-template-string-reconstructi__K5j6a78" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "opa-template-string-reconstructi__rWoXbsY" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "opa-template-string-reconstructi__reoRFG3" } ], "pass_rate": 0.25, "passes": 1, "task": "opa-template-string-reconstruction", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "optique-conditional-option-depen__7Qgt8xS" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "optique-conditional-option-depen__aRsmrpT" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "optique-conditional-option-depen__j7qrntr" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "optique-conditional-option-depen__zptF8hh" } ], "pass_rate": 0.75, "passes": 3, "task": "optique-conditional-option-dependencies", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "oxvg-structural-selector-preserv__N5gtWrQ" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "oxvg-structural-selector-preserv__TAnXefm" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "oxvg-structural-selector-preserv__rbWLh8f" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "oxvg-structural-selector-preserv__snJirQe" } ], "pass_rate": 0.0, "passes": 0, "task": "oxvg-structural-selector-preservation", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "participle-grammar-conflict-anal__9MGsH2Z" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "participle-grammar-conflict-anal__XQXwtDf" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "participle-grammar-conflict-anal__iZXauGc" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "participle-grammar-conflict-anal__j7vjNbW" } ], "pass_rate": 0.25, "passes": 1, "task": "participle-grammar-conflict-analysis", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "pebble-durability-wait-apis__HMtBLQk" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "pebble-durability-wait-apis__SZP9jrk" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "pebble-durability-wait-apis__uXgkfMi" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "pebble-durability-wait-apis__wBtPPZM" } ], "pass_rate": 0.75, "passes": 3, "task": "pebble-durability-wait-apis", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "pest-character-class-coalescing__9RVhW6a" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "pest-character-class-coalescing__9v9Vg7c" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "pest-character-class-coalescing__JJTAzTJ" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "pest-character-class-coalescing__yuTEdtf" } ], "pass_rate": 0.0, "passes": 0, "task": "pest-character-class-coalescing", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "prometheus-transactional-reload__A6nCCWL" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "prometheus-transactional-reload__JGzYNSb" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "prometheus-transactional-reload__RnNbNdD" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "prometheus-transactional-reload__j2Bezfc" } ], "pass_rate": 0.0, "passes": 0, "task": "prometheus-transactional-reload-status", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "prometheus-typed-label-sorting__J4eNFyd" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "prometheus-typed-label-sorting__JViYHKs" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "prometheus-typed-label-sorting__jdDDLFw" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "prometheus-typed-label-sorting__s82vXGJ" } ], "pass_rate": 1.0, "passes": 4, "task": "prometheus-typed-label-sorting", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "psd-tools-blend-range-api__HRQQfuC" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "psd-tools-blend-range-api__TNFzXhG" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "psd-tools-blend-range-api__mYLZpZk" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "psd-tools-blend-range-api__wBPKKaK" } ], "pass_rate": 0.5, "passes": 2, "task": "psd-tools-blend-range-api", "trials_scored": 4, "trials_total": 4 }, { "excluded": 1, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "pwntools-tube-multiplexing__93sTAQS" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "pwntools-tube-multiplexing__A8qL77j" }, { "error_category": "verifier_timeout", "included_in_score": false, "outcome": "excluded_error", "trial_name": "pwntools-tube-multiplexing__iAVLrCa" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "pwntools-tube-multiplexing__iKXnQuS" } ], "pass_rate": 0.3333333333333333, "passes": 1, "task": "pwntools-tube-multiplexing", "trials_scored": 3, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "python-statemachine-state-data-s__DTSCFUc" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "python-statemachine-state-data-s__aUysWYR" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "python-statemachine-state-data-s__gxRR8W4" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "python-statemachine-state-data-s__hr7v6T8" } ], "pass_rate": 0.0, "passes": 0, "task": "python-statemachine-state-data-scoping", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "query-persist-restored-query-sta__N8sm7Ea" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "query-persist-restored-query-sta__nUKWpAP" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "query-persist-restored-query-sta__pFviL5d" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "query-persist-restored-query-sta__ww2TC8P" } ], "pass_rate": 1.0, "passes": 4, "task": "query-persist-restored-query-state", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "quill-shared-toolbar-focus__AnNRNzi" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "quill-shared-toolbar-focus__FnQZDV4" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "quill-shared-toolbar-focus__oid38aF" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "quill-shared-toolbar-focus__q9Qf6uE" } ], "pass_rate": 0.25, "passes": 1, "task": "quill-shared-toolbar-focus", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "returns-validated-error-accumula__3wBHr4T" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "returns-validated-error-accumula__6wGH7yu" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "returns-validated-error-accumula__BwVYWaQ" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "returns-validated-error-accumula__uxKsjee" } ], "pass_rate": 1.0, "passes": 4, "task": "returns-validated-error-accumulation", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "scc-bounded-memory-spilling__2yjvN5R" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "scc-bounded-memory-spilling__cD3gBPy" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "scc-bounded-memory-spilling__fvGSKWD" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "scc-bounded-memory-spilling__u5QWYDx" } ], "pass_rate": 0.75, "passes": 3, "task": "scc-bounded-memory-spilling", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "scriggo-method-declarations__AeKiaQ4" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "scriggo-method-declarations__PdCzKsj" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "scriggo-method-declarations__UoUkqBV" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "scriggo-method-declarations__oZQ6ZvQ" } ], "pass_rate": 1.0, "passes": 4, "task": "scriggo-method-declarations", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "skrub-duration-encoding__3ifWDH7" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "skrub-duration-encoding__FRanuj2" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "skrub-duration-encoding__ggPmNka" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "skrub-duration-encoding__nkh679c" } ], "pass_rate": 1.0, "passes": 4, "task": "skrub-duration-encoding", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "sql-formatter-bigquery-pipe-form__8Fe84xk" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "sql-formatter-bigquery-pipe-form__MKUFMsZ" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "sql-formatter-bigquery-pipe-form__jQYoAFw" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "sql-formatter-bigquery-pipe-form__pyrTo7u" } ], "pass_rate": 1.0, "passes": 4, "task": "sql-formatter-bigquery-pipe-formatting", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "sqlfmt-create-table-ddl-formatti__DTdTFzC" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "sqlfmt-create-table-ddl-formatti__bgo2rkv" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "sqlfmt-create-table-ddl-formatti__r4fPipr" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "sqlfmt-create-table-ddl-formatti__xoH8tBo" } ], "pass_rate": 0.0, "passes": 0, "task": "sqlfmt-create-table-ddl-formatting", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "sqlite-utils-safe-import-checkpo__PhvTiJg" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "sqlite-utils-safe-import-checkpo__Vfr2BzS" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "sqlite-utils-safe-import-checkpo__dQskUF5" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "sqlite-utils-safe-import-checkpo__uGyHMmp" } ], "pass_rate": 1.0, "passes": 4, "task": "sqlite-utils-safe-import-checkpoints", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "superjson-error-stack-serializat__2Hms8J6" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "superjson-error-stack-serializat__KSdZZQd" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "superjson-error-stack-serializat__QvLYTKJ" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "superjson-error-stack-serializat__e2VrXxZ" } ], "pass_rate": 1.0, "passes": 4, "task": "superjson-error-stack-serialization", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "task-task-graph-export__JUXCn5b" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "task-task-graph-export__e7WqkyL" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "task-task-graph-export__nKroLPW" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "task-task-graph-export__p76dppm" } ], "pass_rate": 1.0, "passes": 4, "task": "task-task-graph-export", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "tengo-callable-instance-isolatio__GMGndhs" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "tengo-callable-instance-isolatio__ir4SvdL" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "tengo-callable-instance-isolatio__svs7d95" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "tengo-callable-instance-isolatio__vdyzwVQ" } ], "pass_rate": 1.0, "passes": 4, "task": "tengo-callable-instance-isolation", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "tengo-destructuring-bindings__3hydh4o" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "tengo-destructuring-bindings__5epmwBJ" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "tengo-destructuring-bindings__hxCeNRF" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "tengo-destructuring-bindings__n3iupaP" } ], "pass_rate": 1.0, "passes": 4, "task": "tengo-destructuring-bindings", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "termenv-preserve-ansi-resets__DmM2G4W" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "termenv-preserve-ansi-resets__JrCgRFj" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "termenv-preserve-ansi-resets__ULNfZLA" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "termenv-preserve-ansi-resets__cr84hWx" } ], "pass_rate": 0.25, "passes": 1, "task": "termenv-preserve-ansi-resets", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "testem-bail-on-test-failure__3ELqdYp" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "testem-bail-on-test-failure__HcYH2QZ" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "testem-bail-on-test-failure__d7X4ZVq" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "testem-bail-on-test-failure__mKjVBXr" } ], "pass_rate": 0.0, "passes": 0, "task": "testem-bail-on-test-failure", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "testem-per-launcher-reports__8goSkHd" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "testem-per-launcher-reports__DWFbWbk" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "testem-per-launcher-reports__JVufjUg" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "testem-per-launcher-reports__ceH8Equ" } ], "pass_rate": 1.0, "passes": 4, "task": "testem-per-launcher-reports", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "textual-kitty-key-phases__XUagcdE" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "textual-kitty-key-phases__kW4oyPF" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "textual-kitty-key-phases__pier74V" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "textual-kitty-key-phases__rH9sLLj" } ], "pass_rate": 0.75, "passes": 3, "task": "textual-kitty-key-phases", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "textual-richlog-follow-state__BvFVfu6" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "textual-richlog-follow-state__P3waKkk" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "textual-richlog-follow-state__Y8e4AxD" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "textual-richlog-follow-state__h5VfKQy" } ], "pass_rate": 1.0, "passes": 4, "task": "textual-richlog-follow-state", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "tomlkit-toml-table-converters__GD7TB2M" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "tomlkit-toml-table-converters__MJPS3wj" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "tomlkit-toml-table-converters__cwbARDD" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "tomlkit-toml-table-converters__v7wEGJo" } ], "pass_rate": 1.0, "passes": 4, "task": "tomlkit-toml-table-converters", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "true-myth-iterable-collection-co__7VVbNRM" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "true-myth-iterable-collection-co__A6mcZz5" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "true-myth-iterable-collection-co__SARewQb" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "true-myth-iterable-collection-co__ua85L9q" } ], "pass_rate": 1.0, "passes": 4, "task": "true-myth-iterable-collection-combinators", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "ts-pattern-match-each__6DKPFFv" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "ts-pattern-match-each__Dw7nFXG" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "ts-pattern-match-each__PPLebrc" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "ts-pattern-match-each__nzz4oDs" } ], "pass_rate": 0.5, "passes": 2, "task": "ts-pattern-match-each", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "updo-policy-alerting__4mbSaUV" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "updo-policy-alerting__ED8XwjP" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "updo-policy-alerting__TQrDVrW" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "updo-policy-alerting__yijbWki" } ], "pass_rate": 0.5, "passes": 2, "task": "updo-policy-alerting", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "valibot-recursive-schema-composi__HxkrpBc" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "valibot-recursive-schema-composi__RuhXq5h" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "valibot-recursive-schema-composi__a2P8GHk" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "valibot-recursive-schema-composi__ft6zoJS" } ], "pass_rate": 1.0, "passes": 4, "task": "valibot-recursive-schema-composition", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "vitest-duration-sharding__7DGyabJ" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "vitest-duration-sharding__FTkBbeh" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "vitest-duration-sharding__Xtkibm9" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "vitest-duration-sharding__tzGGa9m" } ], "pass_rate": 1.0, "passes": 4, "task": "vitest-duration-sharding", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "vulture-persistent-analysis-cach__gd25Ab8" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "vulture-persistent-analysis-cach__hAwZSKz" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "vulture-persistent-analysis-cach__hp9yrEV" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "vulture-persistent-analysis-cach__jiLGYze" } ], "pass_rate": 0.5, "passes": 2, "task": "vulture-persistent-analysis-cache", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "wasmi-trap-coredumps__4zaQcag" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "wasmi-trap-coredumps__D2CpJHU" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "wasmi-trap-coredumps__Hs3bJud" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "wasmi-trap-coredumps__VfDbXb6" } ], "pass_rate": 1.0, "passes": 4, "task": "wasmi-trap-coredumps", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "wazero-multi-module-snapshots__6y3ydZW" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "wazero-multi-module-snapshots__fPqUPa8" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "wazero-multi-module-snapshots__pqV3JPh" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "wazero-multi-module-snapshots__tjs43A8" } ], "pass_rate": 1.0, "passes": 4, "task": "wazero-multi-module-snapshots", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "yaegi-go-embed-directives__DFjUfnk" }, { "error_category": null, "included_in_score": true, "outcome": "fail", "trial_name": "yaegi-go-embed-directives__PcVoywu" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "yaegi-go-embed-directives__p4Fsndh" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "yaegi-go-embed-directives__pJThWna" } ], "pass_rate": 0.75, "passes": 3, "task": "yaegi-go-embed-directives", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "yjs-map-conflict-detection__GgtARtM" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "yjs-map-conflict-detection__L3kSSYB" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "yjs-map-conflict-detection__mQ8Gph4" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "yjs-map-conflict-detection__nq9QUfB" } ], "pass_rate": 1.0, "passes": 4, "task": "yjs-map-conflict-detection", "trials_scored": 4, "trials_total": 4 }, { "excluded": 0, "outcomes": [ { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "ytt-jsonpath-query-api__P4eGAPX" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "ytt-jsonpath-query-api__RSiDCR8" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "ytt-jsonpath-query-api__SD86RRm" }, { "error_category": null, "included_in_score": true, "outcome": "pass", "trial_name": "ytt-jsonpath-query-api__hSMWKG3" } ], "pass_rate": 1.0, "passes": 4, "task": "ytt-jsonpath-query-api", "trials_scored": 4, "trials_total": 4 } ], "model": "gpt-5.6-sol", "schema_version": 1, "source": { "artifact_generated_at": "2026-07-13T21:41:35.160831+00:00", "data_url": "https://deepswe.datacurve.ai/data/v1.1", "leaderboard_artifact_url": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "leaderboard_sha256": "67ac2a5e9a78f5671d0fcd839611aee4ccb66c6c519ce3ac59497f23b2a1f067", "leaderboard_url": "https://deepswe.datacurve.ai/", "methodology_url": "https://deepswe.datacurve.ai/blog/deepswe-v1-1", "publisher": "DataCurve", "trials_artifact_url": "https://deepswe.datacurve.ai/artifacts/v1.1/trials.json", "trials_sha256": "450d6191068a23d8f3ac45e619607ffe6a4b9f3a25615218af0c243c831b9cc8" }, "title": "DataCurve DeepSWE v1.1 mini-SWE-agent \u00b7 GPT-5.6 Sol" }