Update SWE-Bench Lite eval results (official harness resolved-rate + submission-rate, vLLM 0.24.0)

every_eval_ever/swebench_lite_dev.json

@@ -1,19 +1,19 @@
 {
   "schema_version": "0.2.2",
-  "evaluation_id": "swebench_lite_dev/RedHatAI/Kimi-K2.6-FP8-BLOCK/1782913700",
-  "retrieved_timestamp": "1782913700",
+  "evaluation_id": "swebench_lite_dev/RedHatAI/Kimi-K2.6-FP8-BLOCK/1784225722.2845917",
+  "retrieved_timestamp": "1784225722.2845917",
   "source_metadata": {
-    "source_name": "mini-swe-agent",
+    "source_name": "SWE-bench harness",
     "source_type": "evaluation_run",
     "source_organization_name": "RedHatAI",
-    "evaluator_relationship": "third_party",
+    "evaluator_relationship": "first_party",
     "additional_details": {
-      "note": "submission-rate aggregation across reruns"
+      "note": "predictions generated with mini-swe-agent, resolved-rate scored with the official SWE-bench harness (swebench.harness.run_evaluation)"
     }
   },
   "eval_library": {
-    "name": "mini-swe-agent",
-    "version": "2.4.3"
+    "name": "swebench",
+    "version": "4.1.0"
   },
   "model_info": {
     "name": "RedHatAI/Kimi-K2.6-FP8-BLOCK",
@@ -21,7 +21,7 @@
     "developer": "RedHatAI",
     "inference_engine": {
       "name": "vllm",
-      "version": "0.22.1"
+      "version": "0.24.0"
     },
     "additional_details": {
       "profile": "kimi_fp8",
@@ -31,14 +31,33 @@
   },
   "evaluation_results": [
     {
-      "evaluation_name": "swebench_lite_dev/submission_rate",
+      "evaluation_name": "swebench_lite_dev/resolved_rate",
       "source_data": {
         "dataset_name": "SWE-bench Lite dev",
         "source_type": "hf_dataset",
         "hf_repo": "princeton-nlp/SWE-Bench_Lite",
         "hf_split": "dev",
-        "samples_number": 23,
-        "sample_ids": [
+        "samples_number": 23
+      },
+      "metric_config": {
+        "evaluation_description": "Resolve rate: fraction of benchmark instances where the submitted patch passes verification (0.0-1.0), scored with the official SWE-bench harness.",
+        "lower_is_better": false,
+        "score_type": "continuous",
+        "min_score": 0.0,
+        "max_score": 1.0
+      },
+      "score_details": {
+        "score": 0.34782608695652173,
+        "details": {
+          "total_instances": "23",
+          "resolved_instances": "8",
+          "submitted_instances": "23",
+          "completed_instances": "19",
+          "unresolved_instances": "11",
+          "empty_patch_instances": "4",
+          "error_instances": "0"
+        },
+        "completed_ids": [
           "marshmallow-code__marshmallow-1343",
           "marshmallow-code__marshmallow-1359",
           "pvlib__pvlib-python-1072",
@@ -53,35 +72,13 @@
           "pydicom__pydicom-901",
           "pylint-dev__astroid-1196",
           "pylint-dev__astroid-1268",
-          "pylint-dev__astroid-1333",
           "pylint-dev__astroid-1866",
           "pylint-dev__astroid-1978",
           "pyvista__pyvista-4315",
-          "sqlfluff__sqlfluff-1517",
-          "sqlfluff__sqlfluff-1625",
-          "sqlfluff__sqlfluff-1733",
           "sqlfluff__sqlfluff-1763",
           "sqlfluff__sqlfluff-2419"
-        ]
-      },
-      "metric_config": {
-        "evaluation_description": "Submission rate (Submitted / total instances).",
-        "lower_is_better": false,
-        "score_type": "continuous",
-        "min_score": 0.0,
-        "max_score": 1.0
-      },
-      "score_details": {
-        "score": 0.9565217391304348,
-        "details": {
-          "total_instances": "23",
-          "submitted_instances": "22",
-          "unsubmitted_instances": "1",
-          "status_counts": "{\"RepeatedFormatError\": 1, \"Submitted\": 22}",
-          "unsubmitted_ids": "[\"sqlfluff__sqlfluff-1733\"]",
-          "source_exit_status_files": "[\"/home/shubhra/kimik2.6_evals/runs/swebench/20260629T164926Z_kimi_fp8_lite_dev/output/exit_statuses_1782751843.1975355.yaml\", \"/home/shubhra/kimik2.6_evals/runs/swebench/20260629T165142Z_kimi_fp8_lite_dev/output/exit_statuses_1782752579.4915643.yaml\", \"/home/shubhra/kimik2.6_evals/runs/swebench/20260629T171225Z_kimi_fp8_lite_dev/output/exit_statuses_1782782345.286938.yaml\", \"/home/shubhra/kimik2.6_evals/runs/swebench/20260630T135435Z_kimi_fp8_lite_dev/output/exit_statuses_1782828817.9969146.yaml\", \"/home/shubhra/kimik2.6_evals/runs/swebench/20260630T143148Z_kimi_fp8_lite_dev/output/exit_statuses_1782830886.6200304.yaml\", \"/home/shubhra/kimik2.6_evals/runs/swebench/20260630T151900Z_kimi_fp8_lite_dev/output/exit_statuses_1782832932.487142.yaml\", \"/home/shubhra/kimik2.6_evals/runs/swebench/20260630T153423Z_kimi_fp8_lite_dev/output/exit_statuses_1782834676.7102883.yaml\", \"/home/shubhra/kimik2.6_evals/runs/swebench/20260630T192452Z_kimi_fp8_lite_dev/output/exit_statuses_1782847729.0120144.yaml\", \"/home/shubhra/kimik2.6_evals/runs/swebench/20260630T193746Z_kimi_fp8_lite_dev/output/exit_statuses_1782849584.3892286.yaml\", \"/home/shubhra/kimik2.6_evals/runs/swebench/20260630T200653Z_kimi_fp8_lite_dev/output/exit_statuses_1782850700.8285635.yaml\"]"
-        },
-        "completed_ids": [
+        ],
+        "submitted_ids": [
           "marshmallow-code__marshmallow-1343",
           "marshmallow-code__marshmallow-1359",
           "pvlib__pvlib-python-1072",
@@ -106,6 +103,53 @@
           "sqlfluff__sqlfluff-1763",
           "sqlfluff__sqlfluff-2419"
         ],
+        "resolved_ids": [
+          "marshmallow-code__marshmallow-1343",
+          "pydicom__pydicom-1256",
+          "pydicom__pydicom-1694",
+          "pydicom__pydicom-901",
+          "pylint-dev__astroid-1196",
+          "pylint-dev__astroid-1866",
+          "sqlfluff__sqlfluff-1763",
+          "sqlfluff__sqlfluff-2419"
+        ]
+      },
+      "generation_config": {
+        "generation_args": {
+          "agentic_eval_config": {
+            "available_tools": [
+              {
+                "name": "bash"
+              }
+            ]
+          },
+          "max_attempts": 1
+        }
+      }
+    },
+    {
+      "evaluation_name": "swebench_lite_dev/submission_rate",
+      "source_data": {
+        "dataset_name": "SWE-bench Lite dev",
+        "source_type": "hf_dataset",
+        "hf_repo": "princeton-nlp/SWE-Bench_Lite",
+        "hf_split": "dev",
+        "samples_number": 23
+      },
+      "metric_config": {
+        "evaluation_description": "Submission rate (Submitted / total instances).",
+        "lower_is_better": false,
+        "score_type": "continuous",
+        "min_score": 0.0,
+        "max_score": 1.0
+      },
+      "score_details": {
+        "score": 1.0,
+        "details": {
+          "total_instances": "23",
+          "submitted_instances": "23",
+          "unsubmitted_instances": "0"
+        },
         "submitted_ids": [
           "marshmallow-code__marshmallow-1343",
           "marshmallow-code__marshmallow-1359",
@@ -127,6 +171,7 @@
           "pyvista__pyvista-4315",
           "sqlfluff__sqlfluff-1517",
           "sqlfluff__sqlfluff-1625",
+          "sqlfluff__sqlfluff-1733",
           "sqlfluff__sqlfluff-1763",
           "sqlfluff__sqlfluff-2419"
         ]
@@ -145,4 +190,4 @@
       }
     }
   ]
-}
+}