Skip to content

Harness API

swebench.harness

__all__ module-attribute 

__all__ = ['docker_utils', 'grading', 'reporting', 'utils', 'constants', 'log_parsers', 'modal_eval']

constants

INFERENCE_LOG_DIR module-attribute 
INFERENCE_LOG_DIR = Path('logs/inference')
RUN_EVALUATION_LOG_DIR module-attribute 
RUN_EVALUATION_LOG_DIR = Path('logs/evaluation')
FAIL_TO_PASS module-attribute 
FAIL_TO_PASS = 'FAIL_TO_PASS'
FAIL_TO_FAIL module-attribute 
FAIL_TO_FAIL = 'FAIL_TO_FAIL'
PASS_TO_PASS module-attribute 
PASS_TO_PASS = 'PASS_TO_PASS'
PASS_TO_FAIL module-attribute 
PASS_TO_FAIL = 'PASS_TO_FAIL'
CONTAINER_PATCH_FILE module-attribute 
CONTAINER_PATCH_FILE = '/tmp/patch.diff'
LOG_REPORT module-attribute 
LOG_REPORT = 'report.json'
LOG_RUN_METADATA module-attribute 
LOG_RUN_METADATA = 'run.json'
LOG_INSTANCE module-attribute 
LOG_INSTANCE = 'run_instance.log'
LOG_TEST_OUTPUT module-attribute 
LOG_TEST_OUTPUT = 'test_output.txt'
APPLY_PATCH_FAIL module-attribute 
APPLY_PATCH_FAIL = '>>>>> Patch Apply Failed'
APPLY_PATCH_PASS module-attribute 
APPLY_PATCH_PASS = '>>>>> Applied Patch'
RESET_FAILED module-attribute 
RESET_FAILED = '>>>>> Reset Failed'
TESTS_ERROR module-attribute 
TESTS_ERROR = '>>>>> Tests Errored'
TESTS_FAILED module-attribute 
TESTS_FAILED = '>>>>> Some Tests Failed'
TESTS_PASSED module-attribute 
TESTS_PASSED = '>>>>> All Tests Passed'
TESTS_TIMEOUT module-attribute 
TESTS_TIMEOUT = '>>>>> Tests Timed Out'
START_TEST_OUTPUT module-attribute 
START_TEST_OUTPUT = '>>>>> Start Test Output'
END_TEST_OUTPUT module-attribute 
END_TEST_OUTPUT = '>>>>> End Test Output'
TEST_EXIT_CODE module-attribute 
TEST_EXIT_CODE = '>>>>> Test Exit Code'
TEST_EXIT_CODE_VAR module-attribute 
TEST_EXIT_CODE_VAR = 'SWEBENCH_TEST_EXIT_CODE'
NON_TEST_EXTS module-attribute 
NON_TEST_EXTS = ['.json', '.png', 'csv', '.txt', '.md', '.jpg', '.jpeg', '.pkl', '.yml', '.yaml', '.toml']
SWE_BENCH_URL_RAW module-attribute 
SWE_BENCH_URL_RAW = 'https://raw.githubusercontent.com/'
FAIL_ONLY_REPOS module-attribute 
FAIL_ONLY_REPOS = {'chartjs/Chart.js', 'processing/p5.js', 'markedjs/marked', 'bpmn-io/bpmn-js', 'openlayers/openlayers', 'eslint/eslint'}
ResolvedStatus

Bases: Enum

NO class-attribute instance-attribute 
NO = 'RESOLVED_NO'
PARTIAL class-attribute instance-attribute 
PARTIAL = 'RESOLVED_PARTIAL'
FULL class-attribute instance-attribute 
FULL = 'RESOLVED_FULL'
TestStatus

Bases: Enum

FAILED class-attribute instance-attribute 
FAILED = 'FAILED'
PASSED class-attribute instance-attribute 
PASSED = 'PASSED'
SKIPPED class-attribute instance-attribute 
SKIPPED = 'SKIPPED'
ERROR class-attribute instance-attribute 
ERROR = 'ERROR'
XFAIL class-attribute instance-attribute 
XFAIL = 'XFAIL'
EvalType

Bases: Enum

PASS_AND_FAIL class-attribute instance-attribute 
PASS_AND_FAIL = 'pass_and_fail'
FAIL_ONLY class-attribute instance-attribute 
FAIL_ONLY = 'fail_only'

docker_utils

copy_to_container 
copy_to_container(container: Container, src: Path, dst: Path)

Copy a file from local to a docker container

Parameters:

Name Type Description Default
container Container

Docker container to copy to

 required
src Path

Source file path

 required
dst Path

Destination file path in the container

 required
Source code in swebench/harness/docker_utils.py 
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
def copy_to_container(container: Container, src: Path, dst: Path):
    """
    Copy a file from local to a docker container

    Args:
        container (Container): Docker container to copy to
        src (Path): Source file path
        dst (Path): Destination file path in the container
    """
    if os.path.dirname(dst) == "":
        raise ValueError(
            f"Destination path parent directory cannot be empty!, dst: {dst}"
        )
    tar_path = src.with_suffix(".tar")
    with tarfile.open(tar_path, "w") as tar:
        tar.add(src, arcname=dst.name)
    with open(tar_path, "rb") as tar_file:
        data = tar_file.read()
    container.exec_run(f"mkdir -p {dst.parent}")
    container.put_archive(os.path.dirname(dst), data)
    tar_path.unlink()
write_to_container 
write_to_container(container: Container, data: str, dst: Path)

Write a string to a file in a docker container

Source code in swebench/harness/docker_utils.py 
39
40
41
42
43
44
45
def write_to_container(container: Container, data: str, dst: Path):
    """
    Write a string to a file in a docker container
    """
    delimiter = generate_heredoc_delimiter(data)
    command = f"cat <<'{delimiter}' > {dst}\n{data}\n{delimiter}"
    container.exec_run(command)
cleanup_container 
cleanup_container(client, container, logger)

Stop and remove a Docker container using subprocess commands. Performs this forcefully if the container cannot be stopped with the standard docker stop.

Parameters:

Name Type Description Default
client DockerClient

Docker client (unused, kept for compatibility).

 required
container Container

Container to remove.

 required
logger Logger

Logger to use for output. If None, print to stdout

 required
Source code in swebench/harness/docker_utils.py 
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
def cleanup_container(client, container, logger):
    """
    Stop and remove a Docker container using subprocess commands.
    Performs this forcefully if the container cannot be stopped with the standard docker stop.

    Args:
        client (docker.DockerClient): Docker client (unused, kept for compatibility).
        container (docker.models.containers.Container): Container to remove.
        logger (logging.Logger): Logger to use for output. If None, print to stdout
    """
    if not container:
        return

    container_id = container.id
    container_name = container.name

    log_info, log_error, raise_error = _get_log_objects(logger)

    try:
        log_info(f"Attempting to stop container {container_name}...")
        cmd = ["docker", "stop", "--time=15", container_id]
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
        if result.returncode != 0:
            raise subprocess.CalledProcessError(result.returncode, cmd, result.stderr)
        log_info(f"Container {container_name} stopped successfully.")
    except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
        log_error(
            f"Failed to stop container {container_name}: {e}. Trying to forcefully kill..."
        )
        try:
            log_info(f"Forcefully killing container {container_name}...")
            cmd = ["docker", "kill", container_id]
            result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
            if result.returncode == 0:
                log_info(f"Container {container_name} killed successfully.")
            else:
                log_error(f"Failed to kill container {container_name}: {result.stderr}")
        except Exception as e2:
            if raise_error:
                raise e2
            log_error(
                f"Failed to forcefully kill container {container_name}: {e2}\n"
                f"{traceback.format_exc()}"
            )
    try:
        log_info(f"Attempting to remove container {container_name}...")
        cmd = ["docker", "rm", "--force", container_id]
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
        if result.returncode == 0:
            log_info(f"Container {container_name} removed successfully.")
        else:
            raise subprocess.CalledProcessError(result.returncode, cmd, result.stderr)
    except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
        if raise_error:
            raise e
        log_error(
            f"Failed to remove container {container_name}: {e}\n"
            f"Command output: {getattr(e, 'stderr', 'N/A')}"
        )
exec_run_with_timeout 
exec_run_with_timeout(container, cmd, timeout: int | None = 60)

Run a command in a container with a timeout.

Parameters:

Name Type Description Default
container Container

Container to run the command in.

 required
cmd str

Command to run.

 required
timeout int

Timeout in seconds.

 60
Source code in swebench/harness/docker_utils.py 
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
def exec_run_with_timeout(container, cmd, timeout: int | None = 60):
    """
    Run a command in a container with a timeout.

    Args:
        container (docker.Container): Container to run the command in.
        cmd (str): Command to run.
        timeout (int): Timeout in seconds.
    """
    exec_result = b""
    exec_id = None
    exec_stream = None
    exception = None
    timed_out = False

    def run_command():
        nonlocal exec_result, exec_id, exec_stream, exception
        try:
            exec_id = container.client.api.exec_create(container.id, cmd)["Id"]
            exec_stream = container.client.api.exec_start(exec_id, stream=True)
            for chunk in exec_stream:
                exec_result += chunk
        except Exception as e:
            exception = e
        finally:
            _close_exec_stream(exec_stream)

    thread = threading.Thread(target=run_command)
    start_time = time.time()
    thread.start()
    thread.join(timeout)
    if exception:
        raise exception
    if thread.is_alive():
        if exec_id is not None:
            exec_pid = container.client.api.exec_inspect(exec_id)["Pid"]
            container.exec_run(f"kill -TERM {exec_pid}", detach=True)
        # the reader is still blocked on the stream; closing it lets that thread end
        _close_exec_stream(exec_stream)
        timed_out = True
    end_time = time.time()
    # test output is arbitrary bytes; a stray non-UTF-8 byte must not kill the run
    return exec_result.decode(errors="replace"), timed_out, end_time - start_time

grading

SUITE_RAN module-attribute 
SUITE_RAN = re.compile("Executed [1-9]\\d* of \\d+|TOTAL: [1-9]\\d* (?:SUCCESS|FAILED)|[1-9]\\d* passing|Tests:\\s+[1-9]\\d*|Test Suites:\\s+(?:\\d+ \\w+, )*[1-9]\\d* total|^# tests [1-9]\\d*|[1-9]\\d* specs?, \\d+ failures?|': ok$", re.M)
TEST_EXIT_CODE_RE module-attribute 
TEST_EXIT_CODE_RE = re.compile(f'{re.escape(TEST_EXIT_CODE)}:\\s*(-?\\d+)')
parse_test_exit_code 
parse_test_exit_code(content: str) -> int | None

Return the recorded test command exit status, or None if absent.

Source code in swebench/harness/grading.py 
55
56
57
58
def parse_test_exit_code(content: str) -> int | None:
    """Return the recorded test command exit status, or None if absent."""
    match = TEST_EXIT_CODE_RE.search(content)
    return int(match.group(1)) if match else None
test_passed 
test_passed(case: str, sm: dict[str, str]) -> bool
Source code in swebench/harness/grading.py 
85
86
87
88
89
90
def test_passed(case: str, sm: dict[str, str]) -> bool:
    key = _resolve_case(case, sm)
    return key is not None and sm[key] in [
        TestStatus.PASSED.value,
        TestStatus.XFAIL.value,
    ]
test_maintained 
test_maintained(case: str, sm: dict[str, str]) -> bool

P2P semantics: a skipped test is not a regression, unlike for F2P.

Source code in swebench/harness/grading.py 
93
94
95
96
97
98
def test_maintained(case: str, sm: dict[str, str]) -> bool:
    """P2P semantics: a skipped test is not a regression, unlike for F2P."""
    key = _resolve_case(case, sm)
    return test_passed(case, sm) or (
        key is not None and sm[key] == TestStatus.SKIPPED.value
    )
test_failed 
test_failed(case: str, sm: dict[str, str]) -> bool
Source code in swebench/harness/grading.py 
101
102
103
104
105
106
107
108
109
def test_failed(case: str, sm: dict[str, str]) -> bool:
    key = _resolve_case(case, sm)
    return key is None or sm[key] in [
        TestStatus.FAILED.value,
        TestStatus.ERROR.value,
        # a skipped F2P test is not a resolution; without this, a patch that makes
        # every F2P test skip lands in neither list and scores RESOLVED_FULL
        TestStatus.SKIPPED.value,
    ]
get_logs_eval 
get_logs_eval(test_spec: TestSpec, log_fp: str) -> tuple[dict[str, str], bool]

Retrieve evaluation results for a task instance from its corresponding log file

Parameters:

Name Type Description Default
log_fp str

path to log file

 required

Returns: bool: whether the patch applied successfully dict: status map

TODO(john-b-yang): Check this is working properly...

Source code in swebench/harness/grading.py 
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
def get_logs_eval(test_spec: TestSpec, log_fp: str) -> tuple[dict[str, str], bool]:
    """
    Retrieve evaluation results for a task instance from its corresponding log file

    Args:
        log_fp (str): path to log file
    Returns:
        bool: whether the patch applied successfully
        dict: status map

    TODO(john-b-yang): Check this is working properly...
    """
    log_parser = PARSER_REGISTRY[test_spec.log_parser]

    with open(log_fp) as f:
        content = f.read()
        # TODO fix constant here
        bad_codes = list(
            filter(
                lambda x: x in content,
                [
                    APPLY_PATCH_FAIL,
                    RESET_FAILED,
                    TESTS_ERROR,
                    TESTS_TIMEOUT,
                ],
            )
        )
        if bad_codes:
            return {}, False
        elif not (START_TEST_OUTPUT in content and END_TEST_OUTPUT in content):
            # Test patch did not apply (should not happen at all)
            return {}, False

        # Get status map of evaluation results
        sliced = content.split(START_TEST_OUTPUT)[1].split(END_TEST_OUTPUT)[0]
        status_map = log_parser(sliced, test_spec)
        if not status_map:
            # Some runners emit results outside the markers (stdout/stderr ordering
            # differs, e.g. on Modal), so fall back to the whole log rather than
            # reporting a run with no results at all.
            status_map = log_parser(content, test_spec)
        if not status_map and not SUITE_RAN.search(content):
            # No parsed results *and* no sign the suite ran: the run is invalid, not
            # a pass. Under EvalType.FAIL_ONLY an absent test counts as success, so
            # without this a suite that never started (e.g. a browser that fails to
            # launch) scores every F2P test as resolved.
            return {}, False

        # A patch can print its own "PASSED" lines (e.g. from a conftest.py hook),
        # so cross-check the log against the test command's exit status, recorded
        # by the eval script. Exiting non-zero while reporting no failure at all
        # means the log is not describing the run that actually happened.
        exit_code = parse_test_exit_code(content)
        if (
            exit_code not in (None, 0)
            and status_map
            and not any(
                status in (TestStatus.FAILED.value, TestStatus.ERROR.value)
                for status in status_map.values()
            )
        ):
            return {}, False
        return status_map, True
get_eval_tests_report 
get_eval_tests_report(eval_status_map: dict[str, str], gold_results: dict[str, str], calculate_to_fail: bool = False, eval_type: EvalType = PASS_AND_FAIL) -> dict[str, dict[str, list[str]]]

Create a report based on failure/pass change from gold results to eval results.

Parameters:

Name Type Description Default
eval_sm dict

evaluation status map

 required
gold_results dict

gold results

 required
calculate_to_fail bool

whether to calculate metrics for "x to fail" tests

 False

Returns: report (dict): report of metrics

Metric Definitions (Gold Result Pair + Eval Result): - Fail-Pass (F2P) + P: Success (Resolution) - Pass-Pass (P2P) + P: Success (Maintenance) - Fail-Pass (F2P) + F: Failure - Pass-Pass (P2P) + F: Failure

Miscellaneous Definitions - Fail-Fail (F2F) + F: Failure Maintenance - Pass-Fail (P2F) + F: Not considered - Fail-Fail (F2F) + P: Success (Extra Credit) - Pass-Fail (P2F) + P: Not considered

Source code in swebench/harness/grading.py 
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
def get_eval_tests_report(
    eval_status_map: dict[str, str],
    gold_results: dict[str, str],
    calculate_to_fail: bool = False,
    eval_type: EvalType = EvalType.PASS_AND_FAIL,
) -> dict[str, dict[str, list[str]]]:
    """
    Create a report based on failure/pass change from gold results to eval results.

    Args:
        eval_sm (dict): evaluation status map
        gold_results (dict): gold results
        calculate_to_fail (bool): whether to calculate metrics for "x to fail" tests
    Returns:
        report (dict): report of metrics

    Metric Definitions (Gold Result Pair + Eval Result):
    - Fail-Pass (F2P) + P: Success (Resolution)
    - Pass-Pass (P2P) + P: Success (Maintenance)
    - Fail-Pass (F2P) + F: Failure
    - Pass-Pass (P2P) + F: Failure

    Miscellaneous Definitions
    - Fail-Fail (F2F) + F: Failure Maintenance
    - Pass-Fail (P2F) + F: Not considered
    - Fail-Fail (F2F) + P: Success (Extra Credit)
    - Pass-Fail (P2F) + P: Not considered
    """

    def check_pass_and_fail(test_case, eval_status_map, success, failed):
        if test_passed(test_case, eval_status_map):
            # Assume silent success for now (test case not in eval_sm)
            success.append(test_case)
        elif test_failed(test_case, eval_status_map):
            failed.append(test_case)

    def check_maintained(test_case, eval_status_map, success, failed):
        if test_maintained(test_case, eval_status_map):
            success.append(test_case)
        elif test_failed(test_case, eval_status_map):
            failed.append(test_case)

    def check_fail_only(test_case, eval_status_map, success, failed):
        if (
            test_case in eval_status_map
            and eval_status_map[test_case] == TestStatus.FAILED.value
        ):
            failed.append(test_case)
        else:
            success.append(test_case)

    check_test_case = (
        check_pass_and_fail if eval_type == EvalType.PASS_AND_FAIL else check_fail_only
    )

    # Calculate resolution metrics
    f2p_success = []
    f2p_failure = []
    for test_case in gold_results[FAIL_TO_PASS]:
        check_test_case(test_case, eval_status_map, f2p_success, f2p_failure)

    # Calculate maintenance metrics
    check_p2p = (
        check_maintained if eval_type == EvalType.PASS_AND_FAIL else check_fail_only
    )
    p2p_success = []
    p2p_failure = []
    for test_case in gold_results[PASS_TO_PASS]:
        check_p2p(test_case, eval_status_map, p2p_success, p2p_failure)

    results = {
        FAIL_TO_PASS: {
            "success": f2p_success,
            "failure": f2p_failure,
        },
        PASS_TO_PASS: {
            "success": p2p_success,
            "failure": p2p_failure,
        },
    }

    f2f_success = []
    f2f_failure = []
    p2f_success = []
    p2f_failure = []
    if calculate_to_fail:
        # Calculate "extra credit" metrics
        for test_case in gold_results[FAIL_TO_FAIL]:
            check_test_case(test_case, eval_status_map, f2f_success, f2f_failure)

        # Calculate not considered metrics
        for test_case in gold_results[PASS_TO_FAIL]:
            check_test_case(test_case, eval_status_map, p2f_success, p2f_failure)

    results.update(
        {
            FAIL_TO_FAIL: {
                "success": f2f_success,
                "failure": f2f_failure,
            },
            PASS_TO_FAIL: {
                "success": p2f_success,
                "failure": p2f_failure,
            },
        }
    )
    return results
compute_fail_to_pass 
compute_fail_to_pass(report: dict[str, dict[str, Any]]) -> float

Compute fail-to-pass metric. Accepts single report as argument.

Source code in swebench/harness/grading.py 
288
289
290
291
292
293
294
295
def compute_fail_to_pass(report: dict[str, dict[str, Any]]) -> float:
    """
    Compute fail-to-pass metric. Accepts single report as argument.
    """
    total = len(report[FAIL_TO_PASS]["success"]) + len(report[FAIL_TO_PASS]["failure"])
    if total == 0:
        return 1
    return len(report[FAIL_TO_PASS]["success"]) / total
compute_pass_to_pass 
compute_pass_to_pass(report: dict[str, dict[str, Any]]) -> float

Compute pass-to-pass metric. Accepts single report as argument.

Source code in swebench/harness/grading.py 
298
299
300
301
302
303
304
305
306
def compute_pass_to_pass(report: dict[str, dict[str, Any]]) -> float:
    """
    Compute pass-to-pass metric. Accepts single report as argument.
    """
    total = len(report[PASS_TO_PASS]["success"]) + len(report[PASS_TO_PASS]["failure"])
    if total == 0:
        # TODO: Don't factor in p2p metrics
        return 1
    return len(report[PASS_TO_PASS]["success"]) / total
get_resolution_status 
get_resolution_status(report: dict[str, dict[str, Any]]) -> str

Determine resolved status of an evaluation instance

Criteria
  • If fail-to-pass (Resolution) = 1 and pass-to-pass (Maintenance) = 1 -> FULL
  • If (fail-to-pass (Resolution) < 1 and > 0) and pass-to-pass (Maintenance) = 1 -> PARTIAL
  • Otherwise -> NO
Source code in swebench/harness/grading.py 
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
def get_resolution_status(report: dict[str, dict[str, Any]]) -> str:
    """
    Determine resolved status of an evaluation instance

    Criteria:
        - If fail-to-pass (Resolution) = 1 and pass-to-pass (Maintenance) = 1 -> FULL
        - If (fail-to-pass (Resolution) < 1 and > 0) and pass-to-pass (Maintenance) = 1 -> PARTIAL
        - Otherwise -> NO
    """
    f2p = compute_fail_to_pass(report)
    p2p = compute_pass_to_pass(report)

    if f2p == 1 and p2p == 1:
        return ResolvedStatus.FULL.value
    elif f2p < 1 and f2p > 0 and p2p == 1:
        return ResolvedStatus.PARTIAL.value
    else:
        return ResolvedStatus.NO.value
get_eval_report 
get_eval_report(test_spec: TestSpec, prediction: dict[str, str], test_log_path: str, include_tests_status: bool) -> dict[str, Any]

Generate a report of model evaluation results from a prediction, task instance, and evaluation log.

Parameters:

Name Type Description Default
test_spec dict

test spec containing keys "instance_id", "FAIL_TO_PASS", and "PASS_TO_PASS"

 required
prediction dict

prediction containing keys "instance_id", "model_name_or_path", and "model_patch"

 required
log_path str

path to evaluation log

 required
include_tests_status bool

whether to include the status of each test in the returned report

 required

Returns: report (dict): report of metrics

Source code in swebench/harness/grading.py 
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
def get_eval_report(
    test_spec: TestSpec,
    prediction: dict[str, str],
    test_log_path: str,
    include_tests_status: bool,
) -> dict[str, Any]:
    """
    Generate a report of model evaluation results from a prediction, task instance,
    and evaluation log.

    Args:
        test_spec (dict): test spec containing keys "instance_id", "FAIL_TO_PASS", and "PASS_TO_PASS"
        prediction (dict): prediction containing keys "instance_id", "model_name_or_path", and "model_patch"
        log_path (str): path to evaluation log
        include_tests_status (bool): whether to include the status of each test in the returned report
    Returns:
        report (dict): report of metrics
    """
    report_map = {}

    instance_id = prediction["instance_id"]
    report_map[instance_id] = {
        "patch_is_None": False,
        "patch_exists": False,
        "patch_successfully_applied": False,
        "resolved": False,
        "infra_failure": False,
    }

    # Check if the model patch exists
    if prediction["model_patch"] is None:
        report_map[instance_id]["patch_is_None"] = True
        return report_map
    report_map[instance_id]["patch_exists"] = True

    # Get evaluation logs
    eval_status_map, found = get_logs_eval(test_spec, test_log_path)

    if not found:
        # No parseable test output: flag a likely environment fault for triage.
        # Advisory only -- `resolved` stays False either way (#586).
        classification = classify_logs(test_log_path)
        if classification:
            reason, tier = classification
            report_map[instance_id]["infra_failure"] = tier == TIER_ENVIRONMENT
            report_map[instance_id]["infra_failure_reason"] = reason
        return report_map
    report_map[instance_id]["patch_successfully_applied"] = True

    eval_ref = {
        "instance_id": test_spec.instance_id,
        FAIL_TO_PASS: test_spec.FAIL_TO_PASS,
        PASS_TO_PASS: test_spec.PASS_TO_PASS,
    }

    eval_type = EvalType(test_spec.eval_type)

    report = get_eval_tests_report(eval_status_map, eval_ref, eval_type=eval_type)
    if get_resolution_status(report) == ResolvedStatus.FULL.value:
        report_map[instance_id]["resolved"] = True

    if include_tests_status:
        report_map[instance_id]["tests_status"] = report  # type: ignore

    return report_map

infra_failure

Post-hoc classification of infrastructure failures (#586).

Separates "the environment broke" from "the model's patch was wrong" using signatures in logs the harness already writes, so a broken image is not silently counted as a model failure.

Two properties are deliberate:

  1. Classification is post-hoc and read-only. It never runs commands in a container and never decides whether an instance gets evaluated, so it cannot drop an instance from a run.
  2. It is advisory. A flagged instance stays in unresolved_ids / error_ids exactly as before, so the scoring denominator is unchanged.
TIER_ENVIRONMENT module-attribute 
TIER_ENVIRONMENT = 'environment'
TIER_AMBIGUOUS module-attribute 
TIER_AMBIGUOUS = 'ambiguous'
INFRA_FAILURE_SIGNATURES module-attribute 
INFRA_FAILURE_SIGNATURES: tuple[tuple[str, str, str], ...] = (('browser_launch_failed', TIER_ENVIRONMENT, 'Failed to launch|Failed to connect to the bus'), ('display_unavailable', TIER_ENVIRONMENT, 'cannot open display|Missing X server|unable to open X display'), ('out_of_memory', TIER_ENVIRONMENT, 'Cannot allocate memory|OutOfMemoryError|^Killed$'), ('container_unavailable', TIER_ENVIRONMENT, 'Error response from daemon|Cannot connect to the Docker daemon'), ('network_unreachable', TIER_ENVIRONMENT, 'Could not resolve host|Temporary failure in name resolution'), ('missing_module', TIER_AMBIGUOUS, 'Cannot find module|MODULE_NOT_FOUND|ModuleNotFoundError'), ('no_tests_collected', TIER_AMBIGUOUS, 'no tests ran|collected 0 items'), ('tests_timed_out', TIER_AMBIGUOUS, 'Timeout error: \\d+ seconds exceeded'))
classify_text 
classify_text(text: str) -> tuple[str, str] | None

Return (reason, tier) for the first matching signature, else None.

Source code in swebench/harness/infra_failure.py 
69
70
71
72
73
74
75
76
def classify_text(text: str) -> tuple[str, str] | None:
    """Return (reason, tier) for the first matching signature, else None."""
    if not text:
        return None
    for reason, tier, pattern in _COMPILED:
        if pattern.search(text):
            return reason, tier
    return None
classify_logs 
classify_logs(*log_paths: str | Path) -> tuple[str, str] | None

Classify the concatenated contents of whichever log paths exist.

Source code in swebench/harness/infra_failure.py 
79
80
81
82
83
84
85
86
def classify_logs(*log_paths: str | Path) -> tuple[str, str] | None:
    """Classify the concatenated contents of whichever log paths exist."""
    chunks = []
    for log_path in log_paths:
        path = Path(log_path)
        if path.exists():
            chunks.append(path.read_text(errors="replace"))
    return classify_text("\n".join(chunks))

log_parsers

PARSER_REGISTRY module-attribute 
PARSER_REGISTRY = {'parse_log_pytest': parse_log_pytest, 'parse_log_pytest_options': parse_log_pytest_options, 'parse_log_django': parse_log_django, 'parse_log_pytest_v2': parse_log_pytest_v2, 'parse_log_seaborn': parse_log_seaborn, 'parse_log_sympy': parse_log_sympy, 'parse_log_matplotlib': parse_log_matplotlib, 'parse_log_astroid': parse_log_astroid, 'parse_log_flask': parse_log_flask, 'parse_log_marshmallow': parse_log_marshmallow, 'parse_log_pvlib': parse_log_pvlib, 'parse_log_pyvista': parse_log_pyvista, 'parse_log_sqlfluff': parse_log_sqlfluff, 'parse_log_xarray': parse_log_xarray, 'parse_log_pydicom': parse_log_pydicom, 'parse_log_requests': parse_log_requests, 'parse_log_pylint': parse_log_pylint, 'parse_log_astropy': parse_log_astropy, 'parse_log_scikit': parse_log_scikit, 'parse_log_sphinx': parse_log_sphinx, 'parse_log_calypso': parse_log_calypso, 'parse_log_bpmn_js': parse_log_bpmn_js, 'parse_log_carbon': parse_log_carbon, 'parse_log_eslint': parse_log_eslint, 'parse_log_grommet': parse_log_grommet, 'parse_log_highlightjs': parse_log_highlightjs, 'parse_log_lighthouse': parse_log_lighthouse, 'parse_log_next': parse_log_next, 'parse_log_openlayers': parse_log_openlayers, 'parse_log_prismjs': parse_log_prismjs, 'parse_log_quarto_cli': parse_log_quarto_cli, 'parse_log_chart_js': parse_log_chart_js, 'parse_log_marked': parse_log_marked, 'parse_log_p5js': parse_log_p5js, 'parse_log_react_pdf': parse_log_react_pdf, 'parse_log_jest': parse_log_jest, 'parse_log_jest_json': parse_log_jest_json, 'parse_log_vitest': parse_log_vitest, 'parse_log_karma': parse_log_karma, 'parse_log_tap': parse_log_tap, 'parse_log_immutable_js': parse_log_immutable_js, 'parse_log_redis': parse_log_redis, 'parse_log_jq': parse_log_jq, 'parse_log_doctest': parse_log_doctest, 'parse_log_micropython_test': parse_log_micropython_test, 'parse_log_googletest': parse_log_googletest, 'parse_log_gotest': parse_log_gotest, 'parse_log_maven': parse_log_maven, 'parse_log_ant': parse_log_ant, 'parse_log_gradle_custom': parse_log_gradle_custom, 'parse_log_phpunit': parse_log_phpunit, 'parse_log_minitest': parse_log_minitest, 'parse_log_cucumber': parse_log_cucumber, 'parse_log_ruby_unit': parse_log_ruby_unit, 'parse_log_rspec_transformed_json': parse_log_rspec_transformed_json, 'parse_log_jekyll': parse_log_jekyll, 'parse_log_cargo': parse_log_cargo}
__all__ module-attribute 
__all__ = ['PARSER_REGISTRY']
c
MAP_REPO_TO_PARSER_C module-attribute 
MAP_REPO_TO_PARSER_C = {'redis/redis': parse_log_redis, 'jqlang/jq': parse_log_jq, 'nlohmann/json': parse_log_doctest, 'micropython/micropython': parse_log_micropython_test, 'valkey-io/valkey': parse_log_redis, 'fmtlib/fmt': parse_log_googletest}
parse_log_redis 
parse_log_redis(log: str, test_spec: TestSpec) -> dict[str, str]

Parameters:

Name Type Description Default
log str

log content

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/c.py 
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
def parse_log_redis(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Args:
        log (str): log content
    Returns:
        dict: test case to test status mapping
    """
    test_status_map = {}

    pattern = r"^\[(ok|err|skip|ignore)\]:\s(.+?)(?:\s\((\d+\s*m?s)\))?$"

    for line in log.split("\n"):
        match = re.match(pattern, line.strip())
        if match:
            status, test_name, _duration = match.groups()
            if status == "ok":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif status == "err":
                # Strip out file path information from failed test names
                test_name = re.sub(r"\s+in\s+\S+$", "", test_name)
                test_status_map[test_name] = TestStatus.FAILED.value
            elif status == "skip" or status == "ignore":
                test_status_map[test_name] = TestStatus.SKIPPED.value

    return test_status_map
parse_log_jq 
parse_log_jq(log: str, test_spec: TestSpec) -> dict[str, str]

Parameters:

Name Type Description Default
log str

log content

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/c.py 
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
def parse_log_jq(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Args:
        log (str): log content
    Returns:
        dict: test case to test status mapping
    """
    test_status_map = {}

    pattern = r"^\s*(PASS|FAIL):\s(.+)$"

    for line in log.split("\n"):
        match = re.match(pattern, line.strip())
        if match:
            status, test_name = match.groups()
            if status == "PASS":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif status == "FAIL":
                test_status_map[test_name] = TestStatus.FAILED.value
    return test_status_map
parse_log_doctest 
parse_log_doctest(log: str, test_spec: TestSpec) -> dict[str, str]

Assumes test binary runs with -s -r=xml.

Source code in swebench/harness/log_parsers/c.py 
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
def parse_log_doctest(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Assumes test binary runs with -s -r=xml.
    """
    test_status_map = {}

    # Extract XML content
    start_tag = "<doctest"
    end_tag = "</doctest>"
    start_index = log.find(start_tag)
    end_index = (
        log.find(end_tag, start_index) + len(end_tag) if start_index != -1 else -1
    )

    if start_index != -1 and end_index != -1:
        xml_string = log[start_index:end_index]
        root = ET.fromstring(xml_string)

        for testcase in root.findall(".//TestCase"):
            testcase_name = testcase.get("name")
            for subcase in testcase.findall(".//SubCase"):
                subcase_name = subcase.get("name")
                name = f"{testcase_name} > {subcase_name}"

                expressions = subcase.findall(".//Expression")
                subcase_passed = all(
                    expr.get("success") == "true" for expr in expressions
                )

                if subcase_passed:
                    test_status_map[name] = TestStatus.PASSED.value
                else:
                    test_status_map[name] = TestStatus.FAILED.value

    return test_status_map
parse_log_micropython_test 
parse_log_micropython_test(log: str, test_spec: TestSpec) -> dict[str, str]
Source code in swebench/harness/log_parsers/c.py 
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
def parse_log_micropython_test(log: str, test_spec: TestSpec) -> dict[str, str]:
    test_status_map = {}

    pattern = r"^(pass|FAIL|skip)\s+(.+)$"

    for line in log.split("\n"):
        match = re.match(pattern, line.strip())
        if match:
            status, test_name = match.groups()
            if status == "pass":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif status == "FAIL":
                test_status_map[test_name] = TestStatus.FAILED.value
            elif status == "skip":
                test_status_map[test_name] = TestStatus.SKIPPED.value

    return test_status_map
parse_log_googletest 
parse_log_googletest(log: str, test_spec: TestSpec) -> dict[str, str]
Source code in swebench/harness/log_parsers/c.py 
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
def parse_log_googletest(log: str, test_spec: TestSpec) -> dict[str, str]:
    test_status_map = {}

    pattern = r"^.*\[\s*(OK|FAILED)\s*\]\s(.*)\s\(.*\)$"

    for line in log.split("\n"):
        match = re.match(pattern, line.strip())
        if match:
            status, test_name = match.groups()
            if status == "OK":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif status == "FAILED":
                test_status_map[test_name] = TestStatus.FAILED.value

    return test_status_map
go
MAP_REPO_TO_PARSER_GO module-attribute 
MAP_REPO_TO_PARSER_GO = {'caddyserver/caddy': parse_log_gotest, 'hashicorp/terraform': parse_log_gotest, 'prometheus/prometheus': parse_log_gotest, 'gohugoio/hugo': parse_log_gotest, 'gin-gonic/gin': parse_log_gotest}
parse_log_gotest 
parse_log_gotest(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated with 'go test'

Parameters:

Name Type Description Default
log str

log content

 required
test_spec TestSpec

test spec (unused)

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/go.py 
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
def parse_log_gotest(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated with 'go test'

    Args:
        log (str): log content
        test_spec (TestSpec): test spec (unused)
    Returns:
        dict: test case to test status mapping
    """
    test_status_map = {}

    # Pattern to match test result lines
    pattern = r"^--- (PASS|FAIL|SKIP): (.+) \((.+)\)$"

    for line in log.split("\n"):
        match = re.match(pattern, line.strip())
        if match:
            status, test_name, _duration = match.groups()
            if status == "PASS":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif status == "FAIL":
                test_status_map[test_name] = TestStatus.FAILED.value
            elif status == "SKIP":
                test_status_map[test_name] = TestStatus.SKIPPED.value

    return test_status_map
java
MAP_REPO_TO_PARSER_JAVA module-attribute 
MAP_REPO_TO_PARSER_JAVA = {'google/gson': parse_log_maven, 'apache/druid': parse_log_maven, 'javaparser/javaparser': parse_log_maven, 'projectlombok/lombok': parse_log_ant, 'apache/lucene': parse_log_gradle_custom, 'reactivex/rxjava': parse_log_gradle_custom}
parse_log_maven 
parse_log_maven(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated with 'mvn test'. Annoyingly maven will not print the tests that have succeeded. For this log parser to work, each test must be run individually, and then we look for BUILD (SUCCESS|FAILURE) in the logs.

Handles race conditions where multiple test commands appear before their BUILD results due to concurrent output from shell tracing and Maven.

Parameters:

Name Type Description Default
log str

log content

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/java.py 
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
def parse_log_maven(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated with 'mvn test'.
    Annoyingly maven will not print the tests that have succeeded. For this log
    parser to work, each test must be run individually, and then we look for
    BUILD (SUCCESS|FAILURE) in the logs.

    Handles race conditions where multiple test commands appear before their
    BUILD results due to concurrent output from shell tracing and Maven.

    Args:
        log (str): log content
    Returns:
        dict: test case to test status mapping
    """
    test_status_map = {}
    pending_tests: list[str] = []
    unmatched_results: list[str] = []

    # Get the test name from the command used to execute the test.
    # Assumes we run evaluation with set -x
    test_name_pattern = r"^.*-Dtest=(\S+).*$"
    result_pattern = r"^.*BUILD (SUCCESS|FAILURE)$"

    for line in log.split("\n"):
        test_name_match = re.match(test_name_pattern, line.strip())
        if test_name_match:
            pending_tests.append(test_name_match.groups()[0])

        result_match = re.match(result_pattern, line.strip())
        if result_match:
            status = result_match.groups()[0]
            if pending_tests:
                test_name = pending_tests.pop(0)
                if status == "SUCCESS":
                    test_status_map[test_name] = TestStatus.PASSED.value
                elif status == "FAILURE":
                    test_status_map[test_name] = TestStatus.FAILED.value
            else:
                # Track unmatched results for later matching
                unmatched_results.append(status)

    # Match any remaining pending tests with unmatched results (FIFO order)
    # This handles cases where BUILD results appear after other output
    while pending_tests and unmatched_results:
        test_name = pending_tests.pop(0)
        status = unmatched_results.pop(0)
        if status == "SUCCESS":
            test_status_map[test_name] = TestStatus.PASSED.value
        elif status == "FAILURE":
            test_status_map[test_name] = TestStatus.FAILED.value

    # Warn if there are still pending tests without results
    if pending_tests:
        print(
            f"[WARNING] Maven log parser: {len(pending_tests)} test(s) had no BUILD result: "
            f"{pending_tests}"
        )

    return test_status_map
parse_log_ant 
parse_log_ant(log: str, test_spec: TestSpec) -> dict[str, str]
Source code in swebench/harness/log_parsers/java.py 
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
def parse_log_ant(log: str, test_spec: TestSpec) -> dict[str, str]:
    test_status_map = {}

    pattern = r"^\s*\[junit\]\s+\[(PASS|FAIL|ERR)\]\s+(.*)$"

    for line in log.split("\n"):
        match = re.match(pattern, line.strip())
        if match:
            status, test_name = match.groups()
            if status == "PASS":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif status in ["FAIL", "ERR"]:
                test_status_map[test_name] = TestStatus.FAILED.value

    return test_status_map
parse_log_gradle_custom 
parse_log_gradle_custom(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated with 'gradle test'. Assumes that the pre-install script to update the gradle config has run.

Handles race conditions where test name and status appear on different lines due to interleaved log output from concurrent processes.

Source code in swebench/harness/log_parsers/java.py 
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
def parse_log_gradle_custom(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated with 'gradle test'. Assumes that the
    pre-install script to update the gradle config has run.

    Handles race conditions where test name and status appear on different lines
    due to interleaved log output from concurrent processes.
    """
    test_status_map = {}

    # Pattern for normal case: test name and status on the same line
    # e.g., "com.example.Test > testMethod PASSED"
    # [^>] ensures we don't match lines starting with > (shell prompts, etc.)
    full_pattern = r"^([^>].+)\s+(PASSED|FAILED)$"

    # Pattern for test name without status (race condition case)
    # e.g., "com.example.Test > testMethod" followed by warnings, then "PASSED"
    # Must also start with [^>] for consistency
    test_name_pattern = r"^([^>]\S*\s+>\s+\S+)$"

    # Pattern for standalone status line
    status_only_pattern = r"^(PASSED|FAILED)$"

    pending_test_name = None

    for line in log.split("\n"):
        stripped = line.strip()

        # Check for full match (test name + status on same line)
        match = re.match(full_pattern, stripped)
        if match:
            test_name, status = match.groups()
            if status == "PASSED":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif status == "FAILED":
                test_status_map[test_name] = TestStatus.FAILED.value
            pending_test_name = None
            continue

        # Check for test name without status
        test_name_match = re.match(test_name_pattern, stripped)
        if test_name_match:
            pending_test_name = test_name_match.group(1)
            continue

        # Check for standalone status (applies to pending test name)
        if pending_test_name:
            status_match = re.match(status_only_pattern, stripped)
            if status_match:
                status = status_match.group(1)
                if status == "PASSED":
                    test_status_map[pending_test_name] = TestStatus.PASSED.value
                elif status == "FAILED":
                    test_status_map[pending_test_name] = TestStatus.FAILED.value
                pending_test_name = None

    # Warn if there's a pending test without a result
    if pending_test_name:
        print(
            f"[WARNING] Gradle log parser: test had no status result: {pending_test_name}"
        )

    return test_status_map
javascript
parse_log_scratch_gui module-attribute 
parse_log_scratch_gui = parse_log_carbon
parse_log_lighthouse_jest module-attribute 
parse_log_lighthouse_jest = parse_log_carbon
parse_log_prettier module-attribute 
parse_log_prettier = parse_log_carbon
MAP_REPO_TO_PARSER_JS module-attribute 
MAP_REPO_TO_PARSER_JS = {'Automattic/wp-calypso': parse_log_calypso, 'chartjs/Chart.js': parse_log_chart_js, 'markedjs/marked': parse_log_marked, 'processing/p5.js': parse_log_p5js, 'diegomura/react-pdf': parse_log_react_pdf, 'babel/babel': parse_log_jest, 'vuejs/core': parse_log_vitest, 'facebook/docusaurus': parse_log_jest, 'immutable-js/immutable-js': parse_log_immutable_js, 'mrdoob/three.js': parse_log_tap, 'preactjs/preact': parse_log_karma, 'axios/axios': parse_log_tap, 'alibaba-fusion/next': parse_log_next, 'bpmn-io/bpmn-js': parse_log_bpmn_js, 'carbon-design-system/carbon': parse_log_carbon, 'cypress-io/cypress': parse_log_cypress, 'emotion-js/emotion': parse_log_emotion, 'eslint/eslint': parse_log_eslint, 'GoogleChrome/lighthouse': parse_log_lighthouse, 'grommet/grommet': parse_log_grommet, 'highlightjs/highlight.js': parse_log_highlightjs, 'openlayers/openlayers': parse_log_openlayers, 'plotly/plotly.js': parse_plotly_js, 'prettier/prettier': parse_log_prettier, 'PrismJS/prism': parse_log_prismjs, 'quarto-dev/quarto-cli': parse_log_quarto_cli, 'scratchfoundation/scratch-gui': parse_log_scratch_gui}
parse_log_highlightjs 
parse_log_highlightjs(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by HighlightJS test suite (Mocha test framework)

Parameters:

Name Type Description Default
log str

Log output from running the test suite

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/javascript.py 
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
def parse_log_highlightjs(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by HighlightJS test suite (Mocha test framework)

    Args:
        log: Log output from running the test suite
    Returns:
        dict: test case to test status mapping
    """
    # Strip the mocha epilogue (summary counts + the "failures" detail block
    # containing stack traces). The summary always begins with "<n> passing";
    # the "pending"/"failing" lines are optional (mocha omits them when zero).
    # Anchoring only on "passing" ensures the epilogue is always removed, so
    # failure-detail lines (e.g. "1 failing", "  1) <suite>", "    at /p:1:2")
    # are never mis-parsed as test names.
    summary_pattern = r"\n[ \t]*\d+ passing"
    log = re.split(summary_pattern, log)[0]

    test_status_map = {}
    current_suite = []
    remove_timing_str = lambda test_name: (
        re.sub(r"\(\d+ms\)", "", test_name).strip()
        if bool(re.search(r"\(\d+ms\)", test_name))
        else test_name.strip()
    )

    for line in log.split("\n"):
        if any([line.strip().startswith(x) for x in ["✓", "✔"]]):
            check = "✓" if line.strip().startswith("✓") else "✔"
            test_name = remove_timing_str(line.split(check, 1)[1])
            full_name = ".".join(current_suite + [test_name])
            test_status_map[full_name] = TestStatus.PASSED.value
        elif re.match(r"^\d+\)", line.strip()):
            test_name = remove_timing_str(line.split(")", 1)[1])
            full_name = ".".join(current_suite + [test_name])
            test_status_map[full_name] = TestStatus.FAILED.value
        elif line and line.startswith(" " * 2):
            current_suite = [line.strip()]
        elif line and not line.startswith(" " * 2) and line.startswith(" " * 4):
            current_suite.append(line.strip())

    return test_status_map
parse_log_prismjs 
parse_log_prismjs(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by PrismJS test suite

Source code in swebench/harness/log_parsers/javascript.py 
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def parse_log_prismjs(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by PrismJS test suite
    """
    test_status_map = {}
    # v5 has no constants map; prism uses one test command across all versions,
    # and it appears verbatim in the embedded eval script
    test_cmd = "./node_modules/.bin/mocha tests/run.js --reporter json"
    log_split_by_cmd = log.split(test_cmd)

    for log in log_split_by_cmd[1:]:
        test_results = json.loads("{" + log.split("{", 1)[-1].rsplit("}", 1)[0] + "}")
        for test in test_results["failures"]:
            test_status_map[test["fullTitle"]] = TestStatus.FAILED.value
        for test in test_results["passes"]:
            test_status_map[test["fullTitle"]] = TestStatus.PASSED.value

    return test_status_map
parse_log_eslint 
parse_log_eslint(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by ESLint test suite

Source code in swebench/harness/log_parsers/javascript.py 
73
74
75
76
77
78
79
80
81
82
83
def parse_log_eslint(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by ESLint test suite
    """
    test_status_map = {}
    failure_case_pattern = r"(\s{2}\d+\))([\s\S]*?)\s+(?:(.*)Error)"
    failures = re.findall(failure_case_pattern, log)
    for failure in failures:
        failure = re.sub(r"\s+", " ", failure[1]).strip()
        test_status_map[failure] = TestStatus.FAILED.value
    return test_status_map
parse_log_bpmn_js 
parse_log_bpmn_js(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by BPMN-JS test suite

Source code in swebench/harness/log_parsers/javascript.py 
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
def parse_log_bpmn_js(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by BPMN-JS test suite
    """
    test_status_map = {}
    failure_case_patterns = [
        r"2KPhantomJS\s[\d\.]+\s\(Linux\s[\d\.]+\)\s(.*)FAILED\n",
        r"2KPhantomJS\s[\d\.]+\s\(Linux\sx86\_64\)\s(.*)FAILED\n",
        r"Chrome\sHeadless\s(.*)\s\(Linux\sx86\_64\)\s(.*)FAILED\n",
        r"HeadlessChrome\s[\d\.]+\s\(Linux\s[\d\.]+\)\s(.*)FAILED\n",
    ]
    for failure_case_pattern in failure_case_patterns:
        failures = re.findall(failure_case_pattern, log)
        if len(failures) == 0:
            continue
        for failure in failures:
            if isinstance(failure, tuple):
                # This would only be true for the last failure case pattern
                failure = failure[1]
            test_status_map[failure] = TestStatus.FAILED.value
    return test_status_map
parse_plotly_js 
parse_plotly_js(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by PlotlyJS test suite

Source code in swebench/harness/log_parsers/javascript.py 
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
def parse_plotly_js(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by PlotlyJS test suite
    """
    suite_pattern = re.compile(r"(\s+)-\s+(.*?)\s+:")
    test_pattern = re.compile(r"(\s+)\*\s+(.*?)\s+:\s*(.*)")
    suite_stack = (
        [],
        0,
    )  # (hierarchical list of suite names, number of spaces indicating hierarchy)

    test_status_map = dict()
    for line in log.split("\n"):
        suite_match = suite_pattern.match(line)
        if suite_match:
            spaces, suite_name = suite_match.groups()
            spaces = len(spaces)
            while suite_stack[0] and spaces <= suite_stack[1]:
                suite_stack[0].pop()
            suite_stack = (suite_stack[0] + [suite_name], spaces)
        else:
            test_match = test_pattern.match(line)
            if test_match:
                spaces, test_name, status = test_match.groups()
                spaces = len(spaces)
                while suite_stack[0] and spaces <= suite_stack[1]:
                    suite_stack[0].pop()
                full_test_name = ".".join(suite_stack[0] + [test_name])
                if "ok" not in status.lower() or "failed" in status.lower():
                    test_status_map[full_test_name] = TestStatus.FAILED.value
                else:
                    test_status_map[full_test_name] = TestStatus.PASSED.value
    return test_status_map
parse_log_openlayers 
parse_log_openlayers(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by OpenLayers test suite

Source code in swebench/harness/log_parsers/javascript.py 
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
def parse_log_openlayers(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by OpenLayers test suite
    """
    test_status_map = {}
    failure_case_patterns = [
        r"Chrome\sHeadless\s[\d\.]+\s\(Linux\s[\d\.\_\S]+\)\s(.*)\sFAILED\n",
        r"Chrome\s[\d\.]+\s\(Linux\s[\d\.\_\S]+\)\s(.*)FAILED\n",
        r"^\s{2}\d+\)([\s\S]*?)Error:",
        r"^case\s(.*):\smismatch",
    ]
    for failure_case_pattern in failure_case_patterns:
        failures = re.findall(failure_case_pattern, log, re.MULTILINE)
        if len(failures) == 0:
            continue
        for failure in failures:
            failure = re.sub(r"\s+", " ", failure).strip()
            test_status_map[failure] = TestStatus.FAILED.value
    return test_status_map
parse_log_emotion 
parse_log_emotion(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by Emotion test suite

Source code in swebench/harness/log_parsers/javascript.py 
165
166
167
168
169
170
171
172
173
174
def parse_log_emotion(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by Emotion test suite
    """
    test_status_map = {}
    for failure in re.findall(rf"^FAIL\s(.*)", log, re.MULTILINE):
        test_status_map[failure.strip()] = TestStatus.FAILED.value
    for passing in re.findall(rf"^PASS\s(.*)", log, re.MULTILINE):
        test_status_map[passing.strip()] = TestStatus.PASSED.value
    return test_status_map
parse_log_grommet 
parse_log_grommet(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by Grommet test suite

Source code in swebench/harness/log_parsers/javascript.py 
177
178
179
180
181
182
183
184
185
186
187
188
189
def parse_log_grommet(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by Grommet test suite
    """
    timing_pattern = r"\s\([\d\.]+\s?(?:ms|s)\)$"
    test_status_map = {}
    for failure in re.findall(r"^FAIL\s(.*)", log, re.MULTILINE):
        name = re.sub(timing_pattern, "", failure.strip())
        test_status_map[name] = TestStatus.FAILED.value
    for passing in re.findall(r"^PASS\s(.*)", log, re.MULTILINE):
        name = re.sub(timing_pattern, "", passing.strip())
        test_status_map[name] = TestStatus.PASSED.value
    return test_status_map
parse_log_next_v1 
parse_log_next_v1(log: str, test_spec: TestSpec) -> dict[str, str]
Source code in swebench/harness/log_parsers/javascript.py 
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
def parse_log_next_v1(log: str, test_spec: TestSpec) -> dict[str, str]:
    test_status_map = {}
    suite = []

    # Remove all unicode, error msg blocks
    log = re.sub(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", log)
    log = re.sub(r"AssertionError:\s+#([\s\S]*?)\n\n", "", log, flags=re.DOTALL)

    fail_pattern = r".*?✗\s*(.*)"
    pass_pattern = r".*?✓\s*(.*)"

    for log in log.split(" xvfb-run ")[1:]:
        log = log.split("Starting browser Chrome")[-1]
        for line in log.split("\n"):
            if line.startswith("TOTAL"):
                break
            elif any(
                [
                    line.strip().startswith(x)
                    for x in ["ERROR", "LOG", "WARN", "Check", "in"]
                ]
            ):
                continue
            elif re.match(fail_pattern, line) is not None:
                test = re.match(fail_pattern, line).group(1)
                test_status_map[" - ".join([x[0] for x in suite] + [test])] = (
                    TestStatus.FAILED.value
                )
            elif re.match(pass_pattern, line) is not None:
                test = re.match(pass_pattern, line).group(1)
                test_status_map[" - ".join([x[0] for x in suite] + [test])] = (
                    TestStatus.PASSED.value
                )
            elif len(line) - len(line.lstrip()) > 0:
                # Adjust suite name
                indent = len(line) - len(line.lstrip())
                if len(suite) == 0:
                    # If suite is empty, initialize it
                    suite = [(line.strip(), indent)]
                else:
                    while len(suite) > 0 and suite[-1][-1] >= indent:
                        # Pop until the last element with indent less than current indent
                        suite.pop()
                    suite.append([line.strip(), indent])

    return test_status_map
parse_log_next_v2 
parse_log_next_v2(log: str, test_spec: TestSpec) -> dict[str, str]
Source code in swebench/harness/log_parsers/javascript.py 
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
def parse_log_next_v2(log: str, test_spec: TestSpec) -> dict[str, str]:
    test_status_map = {}
    test_suites_pattern = r"Running\:(.*)\)\n{3}\s{2}(.*)\n"
    log_split_by_suites = re.split(test_suites_pattern, log)
    test_suites = re.findall(test_suites_pattern, log)

    failure_case_pattern = r"^\s{4}\d+\)\s(.*)"
    success_case_pattern = r"^\s{4}✓\s(.*)\("

    for idx, test_suite in enumerate(test_suites):
        test_suite = test_suite[-1]
        log = log_split_by_suites[(idx + 1) * 3]
        failures = re.findall(failure_case_pattern, log, re.MULTILINE)
        for failure in failures:
            print(failure)
            test_status_map[f"{test_suite}: {failure.strip()}"] = (
                TestStatus.FAILED.value
            )
        successes = re.findall(success_case_pattern, log, re.MULTILINE)
        for success in successes:
            test_status_map[f"{test_suite}: {success.strip()}"] = (
                TestStatus.PASSED.value
            )

    return test_status_map
parse_log_next 
parse_log_next(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by Calypso test suite

Source code in swebench/harness/log_parsers/javascript.py 
267
268
269
270
271
272
273
274
def parse_log_next(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by Calypso test suite
    """
    if test_spec.version in ["1.27"]:
        return parse_log_next_v2(log, test_spec)
    else:
        return parse_log_next_v1(log, test_spec)
parse_log_cypress 
parse_log_cypress(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by Cypress test suite

Source code in swebench/harness/log_parsers/javascript.py 
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
def parse_log_cypress(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by Cypress test suite
    """
    test_status_map = {}
    log_split_by_suites = log.split("../..")
    for log in log_split_by_suites[:-1]:
        for test in re.findall(
            r"\"fullTitle\"\:\s\"(.*)\",\n",
            log.split('  "passes": [')[-1].rsplit("  }", 1)[0],
            re.MULTILINE,
        ):
            test_status_map[test] = TestStatus.PASSED.value

        for test in re.findall(
            r"\"fullTitle\"\:\s\"(.*)\",\n",
            log.split('  "failures": [')[-1].rsplit("  ],", 1)[0],
            re.MULTILINE,
        ):
            test_status_map[test] = TestStatus.FAILED.value

    return test_status_map
parse_log_carbon 
parse_log_carbon(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by Carbon test suite

Source code in swebench/harness/log_parsers/javascript.py 
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
def parse_log_carbon(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by Carbon test suite
    """
    test_status_map = {}
    for line in log.split("\n"):
        for pattern in [
            (r"^PASS\s(.*)\s\([\d\.]+ms\)", TestStatus.PASSED.value),
            (r"^PASS\s(.*)\s\([\d\.]+\ss\)", TestStatus.PASSED.value),
            (r"^PASS\s(.*)\s\([\d\.]+s\)", TestStatus.PASSED.value),
            (r"^PASS\s(.*)", TestStatus.PASSED.value),
            (r"^FAIL\s(.*)\s\([\d\.]+ms\)", TestStatus.FAILED.value),
            (r"^FAIL\s(.*)\s\([\d\.]+\ss\)", TestStatus.FAILED.value),
            (r"^FAIL\s(.*)\s\([\d\.]+s\)", TestStatus.FAILED.value),
            (r"^FAIL\s(.*)", TestStatus.FAILED.value),
        ]:
            if re.search(pattern[0], line):
                test_name = re.match(pattern[0], line).group(1)
                test_status_map[test_name] = pattern[1]
                break
    return test_status_map
parse_log_lighthouse_mocha 
parse_log_lighthouse_mocha(log: str, test_spec: TestSpec) -> dict[str, str]
Source code in swebench/harness/log_parsers/javascript.py 
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
def parse_log_lighthouse_mocha(log: str, test_spec: TestSpec) -> dict[str, str]:
    test_status_map = {}

    for log_run in re.split(r"running\s\d+\stest\sfiles", log)[1:]:
        log_run = re.split(r"\d+\spassing", log_run)[0].strip()
        suite = []

        for line in log_run.split("\n"):
            test_case_match_found = False
            for pattern in [
                (r"✔\s(.*)\s\(\d+m+s\)", TestStatus.PASSED.value),
                (r"✔\s(.*)", TestStatus.PASSED.value),
                (r"\d+\)\s(.*)\s\(\d+m+s\)", TestStatus.FAILED.value),
                (r"\d+\)\s(.*)", TestStatus.FAILED.value),
            ]:
                if re.match(pattern[0], line.strip()):
                    test = re.match(pattern[0], line.strip()).group(1)
                    test_status_map[" - ".join([x[0] for x in suite] + [test])] = (
                        pattern[1]
                    )
                    test_case_match_found = True
                    break

            if not test_case_match_found and len(line) - len(line.lstrip()) > 0:
                # Adjust suite name
                indent = len(line) - len(line.lstrip())
                if len(suite) == 0:
                    # If suite is empty, initialize it
                    suite = [(line.strip(), indent)]
                else:
                    while len(suite) > 0 and suite[-1][-1] >= indent:
                        # Pop until the last element with indent less than current indent
                        suite.pop()
                    suite.append([line.strip(), indent])

    return test_status_map
parse_log_lighthouse 
parse_log_lighthouse(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by Lighthouse test suite

Source code in swebench/harness/log_parsers/javascript.py 
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
def parse_log_lighthouse(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by Lighthouse test suite
    """
    if test_spec.version in ["9.5", "10.0", "10.2"]:
        return parse_log_lighthouse_mocha(log, test_spec)
    elif float(test_spec.version) >= 3:
        return parse_log_lighthouse_jest(log, test_spec)

    test_status_map = {}
    log_split_by_cmd = log.split("./node_modules/.bin/mocha --reporter json")
    for log in log_split_by_cmd[1:]:
        for test in re.findall(
            r"\"fullTitle\"\:\s\"(.*)\",\n",
            log.split('"passes": [')[-1].rsplit("]", 1)[0],
            re.MULTILINE,
        ):
            test_status_map[test] = TestStatus.PASSED.value

        for test in re.findall(
            r"\"fullTitle\"\:\s\"(.*)\",\n",
            log.split('"failures": [')[-1].rsplit("],", 1)[0],
            re.MULTILINE,
        ):
            test_status_map[test] = TestStatus.FAILED.value
    return test_status_map
parse_log_quarto_cli 
parse_log_quarto_cli(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by Quarto test suite

Source code in swebench/harness/log_parsers/javascript.py 
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
def parse_log_quarto_cli(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by Quarto test suite
    """
    test_case_prefix = r"\[(smoke|unit)\]"
    quarto_val_divider = "..."
    exclude_pattern = r"test result:"
    tmp_file_patterns = [r"/tmp/(\w+)\.qmd", r"/tmp/(\w+)/"]

    finished_pattern = r"ERRORS"
    finished_pattern2 = r"failures:"
    tmp_key_map = {}
    cnt = 0  # For handling /tmp/.../ files
    test_status_map = {}

    for line in log.split("\n"):
        # Stop checking after ERROR is found
        if re.search(finished_pattern, line) or re.search(finished_pattern2, line):
            break

        if re.search(test_case_prefix, line):
            # Replace tmp files properly
            for pattern in tmp_file_patterns:
                match = re.search(pattern, line)
                if match:
                    if match.group(0) not in tmp_key_map.keys():
                        tmp_key_map[match.group(0)] = cnt
                        cnt += 1
                    line = re.sub(pattern, f"/tmp/{tmp_key_map[match.group(0)]}", line)

            # Handle parsing test case passing / failing
            test_name = line.split(quarto_val_divider)[0].strip()
            for pattern in [
                (r"32mok", TestStatus.PASSED.value),
                (r"31mFAILED", TestStatus.FAILED.value),
            ]:
                reject_condition = (
                    re.match(exclude_pattern, line)
                    or (
                        re.search("passed", line)
                        and re.search("failed", line)
                        and re.search("steps", line)
                    )
                    or (re.search("34mINFO", line))
                )

                if re.search(pattern[0], line) and not reject_condition:
                    # All tests are of the form {name ... status}
                    test_status_map[test_name] = pattern[1]
                    break

            # Assume if test name had no associated status, it FAILED
            if test_name not in test_status_map and not reject_condition:
                test_status_map[test_name] = TestStatus.FAILED.value

    return test_status_map
parse_log_calypso 
parse_log_calypso(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by Calypso test suite

Source code in swebench/harness/log_parsers/javascript.py 
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
def parse_log_calypso(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by Calypso test suite
    """
    test_status_map = {}
    suite = []

    get_test_name = lambda suite, match_pattern, line: " - ".join(
        [" - ".join([x[0] for x in suite]), re.match(match_pattern, line).group(1)]
    ).strip()

    for log in log.split(" ./node_modules/.bin/jest ")[1:]:
        for line in log.split("\n"):
            if any([line.startswith(x) for x in ["Test Suites", "  ● "]]):
                break
            elif line.strip().startswith("✓"):
                # Test passed
                match_pattern = (
                    r"^\s+✓\s(.*)\(\d+ms\)$"
                    if re.search(r"\(\d+ms\)", line) is not None
                    else r"^\s+✓\s(.*)"
                )
                test_status_map[get_test_name(suite, match_pattern, line)] = (
                    TestStatus.PASSED.value
                )
            elif line.strip().startswith("✕"):
                # Test failed
                match_pattern = (
                    r"^\s+✕\s(.*)\(\d+ms\)$"
                    if re.search(r"\(\d+ms\)", line) is not None
                    else r"^\s+✕\s(.*)"
                )
                test_status_map[get_test_name(suite, match_pattern, line)] = (
                    TestStatus.FAILED.value
                )
            elif len(line) - len(line.lstrip()) > 0:
                # Adjust suite name
                indent = len(line) - len(line.lstrip())
                if len(suite) == 0:
                    # If suite is empty, initialize it
                    suite = [(line.strip(), indent)]
                else:
                    while len(suite) > 0 and suite[-1][-1] >= indent:
                        # Pop until the last element with indent less than current indent
                        suite.pop()
                    suite.append([line.strip(), indent])

    return test_status_map
parse_log_chart_js 
parse_log_chart_js(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by ChartJS test suite

Source code in swebench/harness/log_parsers/javascript.py 
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
def parse_log_chart_js(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by ChartJS test suite
    """
    log = ansi_escape(log)
    test_status_map = {}
    failure_case_patterns = [
        # use [^\S\r\n] to avoid overlapping Chrome groups on separate lines
        (r"Chrome\s[\d\.]+[^\S\r\n]\(.+?\)[^\S\r\n](.*)FAILED$", re.MULTILINE),
    ]
    for failure_case_pattern, flags in failure_case_patterns:
        failures = re.findall(failure_case_pattern, log, flags)
        if len(failures) == 0:
            continue
        for failure in failures:
            test_status_map[failure] = TestStatus.FAILED.value
    return test_status_map
parse_log_marked 
parse_log_marked(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by Marked test suite

Source code in swebench/harness/log_parsers/javascript.py 
522
523
524
525
526
527
528
529
530
531
def parse_log_marked(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by Marked test suite
    """
    test_status_map = {}
    for line in log.split("\n"):
        if re.search(r"^\d+\)\s(.*)", line):
            test = re.search(r"^\d+\)\s(.*)", line).group(1)
            test_status_map[test.strip()] = TestStatus.FAILED.value
    return test_status_map
parse_log_p5js 
parse_log_p5js(log: str, test_spec: TestSpec) -> dict[str, str]
Source code in swebench/harness/log_parsers/javascript.py 
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
def parse_log_p5js(log: str, test_spec: TestSpec) -> dict[str, str]:
    def remove_json_blocks(log_content):
        filtered_lines = []
        in_json_block = False
        in_json_list_block = False
        for line in log_content.split("\n"):
            stripped_line = line.rstrip()  # Remove trailing whitespace
            if stripped_line.endswith("{"):
                in_json_block = True
                continue
            if stripped_line.endswith("["):
                in_json_list_block = True
                continue
            if stripped_line == "}" and in_json_block:
                in_json_block = False
                continue
            if stripped_line == "]" and in_json_list_block:
                in_json_list_block = False
                continue
            if in_json_block or in_json_list_block:
                continue
            if stripped_line.startswith("{") and stripped_line.endswith("}"):
                continue
            if stripped_line.startswith("[") and stripped_line.endswith("]"):
                continue
            filtered_lines.append(line)
        return "\n".join(filtered_lines)

    def remove_xml_blocks(log_content):
        xml_pat = re.compile(r"<(\w+)>[\s\S]*?<\/\1>", re.MULTILINE)
        match = xml_pat.search(log_content)
        while match:
            # count the number of opening tags in the match
            opening_tags = match.group().count(rf"<{match.group(1)}>") - 1
            opening_tags = max(opening_tags, 0)
            start = match.start()
            end = match.end()
            log_content = (
                log_content[:start]
                + f"<{match.group(1)}>" * opening_tags
                + log_content[end:]
            )
            match = xml_pat.search(log_content)
        return log_content

    def is_valid_fail(match):
        last_line_indent = 0
        for line in match.group(2).split("\n"):
            line_indent = len(line) - len(line.lstrip())
            if line_indent <= last_line_indent:
                return False
            last_line_indent = line_indent
        return True

    log = ansi_escape(log)
    log = remove_json_blocks(log)
    log = remove_xml_blocks(log)
    test_results = {}

    # Parse failing tests
    fail_pattern = re.compile(r"^\s*(\d+)\)(.{0,1000}?):", re.MULTILINE | re.DOTALL)
    for match in fail_pattern.finditer(log):
        if is_valid_fail(match):
            test_names = list(map(str.strip, match.group(2).split("\n")))
            full_name = ":".join(test_names)
            test_results[full_name] = TestStatus.FAILED.value

    return test_results
parse_log_react_pdf 
parse_log_react_pdf(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated by Carbon test suite

Source code in swebench/harness/log_parsers/javascript.py 
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
def parse_log_react_pdf(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated by Carbon test suite
    """
    test_status_map = {}
    for line in log.split("\n"):
        for pattern in [
            (r"^PASS\s(.*)\s\([\d\.]+ms\)", TestStatus.PASSED.value),
            (r"^PASS\s(.*)\s\([\d\.]+\ss\)", TestStatus.PASSED.value),
            (r"^PASS\s(.*)\s\([\d\.]+s\)", TestStatus.PASSED.value),
            (r"^PASS\s(.*)", TestStatus.PASSED.value),
            (r"^FAIL\s(.*)\s\([\d\.]+ms\)", TestStatus.FAILED.value),
            (r"^FAIL\s(.*)\s\([\d\.]+\ss\)", TestStatus.FAILED.value),
            (r"^FAIL\s(.*)\s\([\d\.]+s\)", TestStatus.FAILED.value),
            (r"^FAIL\s(.*)", TestStatus.FAILED.value),
        ]:
            if re.search(pattern[0], line):
                test_name = re.match(pattern[0], line).group(1)
                test_status_map[test_name] = pattern[1]
                break
    return test_status_map
parse_log_jest 
parse_log_jest(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated with Jest. Assumes --verbose flag.

Parameters:

Name Type Description Default
log str

log content

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/javascript.py 
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
def parse_log_jest(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated with Jest. Assumes --verbose flag.

    Args:
        log (str): log content
    Returns:
        dict: test case to test status mapping
    """
    test_status_map = {}

    pattern = r"^\s*(✓|✕|○)\s(.+?)(?:\s\((\d+\s*m?s)\))?$"

    for line in log.split("\n"):
        match = re.match(pattern, line.strip())
        if match:
            status_symbol, test_name, _duration = match.groups()
            test_name = test_name.strip()
            if status_symbol == "✓":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif status_symbol == "✕":
                test_status_map[test_name] = TestStatus.FAILED.value
            elif status_symbol == "○":
                test_status_map[test_name] = TestStatus.SKIPPED.value
    return test_status_map
parse_log_jest_json 
parse_log_jest_json(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated with Jest. Assumes the --json flag has been piped into JEST_JSON_JQ_TRANSFORM. Unlike --verbose, tests with the same name in different describe blocks print with different names.

Source code in swebench/harness/log_parsers/javascript.py 
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
def parse_log_jest_json(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated with Jest. Assumes the --json flag has been
    piped into JEST_JSON_JQ_TRANSFORM. Unlike --verbose, tests with the same name
    in different describe blocks print with different names.
    """
    test_status_map = {}

    pattern = r"^\[(PASSED|FAILED)\]\s(.+)$"

    for line in log.split("\n"):
        match = re.match(pattern, line.strip())
        if match:
            status, test_name = match.groups()
            if status == "PASSED":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif status == "FAILED":
                test_status_map[test_name] = TestStatus.FAILED.value
    return test_status_map
parse_log_vitest 
parse_log_vitest(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated with vitest. Assumes --reporter=verbose flag.

Source code in swebench/harness/log_parsers/javascript.py 
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
def parse_log_vitest(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated with vitest. Assumes --reporter=verbose flag.
    """
    test_status_map = {}

    pattern = r"^\s*(✓|×|↓)\s(.+?)(?:\s(\d+\s*m?s?|\[skipped\]))?$"

    for line in log.split("\n"):
        match = re.match(pattern, line.strip())
        if match:
            status_symbol, test_name, _duration_or_skipped = match.groups()
            if status_symbol == "✓":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif status_symbol == "×":
                test_status_map[test_name] = TestStatus.FAILED.value
            elif status_symbol == "↓":
                test_status_map[test_name] = TestStatus.SKIPPED.value
    return test_status_map
parse_log_karma 
parse_log_karma(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated with Karma. Handles duplicate test names in different describe blocks. Logic is brittle.

Source code in swebench/harness/log_parsers/javascript.py 
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
def parse_log_karma(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated with Karma. Handles duplicate test names in
    different describe blocks. Logic is brittle.
    """
    test_status_map = {}
    current_indent = -1
    current_suite = []
    started = False

    pattern = r"^(\s*)?([✔✖])?\s(.*)$"

    for line in log.split("\n"):
        if line.startswith("SUMMARY:"):
            # Individual test logs end here
            return test_status_map

        if "Starting browser" in line:
            started = True
            continue

        if not started:
            continue

        match = re.match(pattern, line)
        if match:
            indent, status, name = match.groups()

            if indent and not status:
                new_indent = len(indent)
                if new_indent > current_indent:
                    current_indent = new_indent
                    current_suite.append(name)
                elif new_indent < current_indent:
                    current_indent = new_indent
                    current_suite.pop()
                    continue

            if status in ("✔", "✖"):
                full_test_name = " > ".join(current_suite + [name])
                test_status_map[full_test_name] = (
                    TestStatus.PASSED.value
                    if status == "✔"
                    else TestStatus.FAILED.value
                )

    return test_status_map
parse_log_tap 
parse_log_tap(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated with TAP

Parameters:

Name Type Description Default
log str

log content

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/javascript.py 
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
def parse_log_tap(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated with TAP

    Args:
        log (str): log content
    Returns:
        dict: test case to test status mapping
    """
    test_status_map = {}

    # Pattern to match TAP result lines
    pattern = r"^(ok|not ok) (\d+) (.+)$"

    for line in log.split("\n"):
        match = re.match(pattern, line.strip())
        if match:
            status, _test_number, test_name = match.groups()
            if status == "ok":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif status == "not ok":
                test_status_map[test_name] = TestStatus.FAILED.value

    return test_status_map
parse_log_immutable_js 
parse_log_immutable_js(log: str, test_spec: TestSpec) -> dict[str, str]

Different immutable.js instances use different test runners and log formats. This function selects the appropriate log parser based on the instance id.

Source code in swebench/harness/log_parsers/javascript.py 
771
772
773
774
775
776
777
778
779
780
781
782
783
def parse_log_immutable_js(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Different immutable.js instances use different test runners and log formats.
    This function selects the appropriate log parser based on the instance id.
    """
    pr_number = test_spec.instance_id.split("-")[-1]

    if pr_number in ["2006"]:
        return parse_log_jest(log, test_spec)
    elif pr_number in ["2005"]:
        return parse_log_jest_json(log, test_spec)
    else:
        raise ValueError(f"Unknown instance id: {test_spec.instance_id}")
php
MAP_REPO_TO_PARSER_PHP module-attribute 
MAP_REPO_TO_PARSER_PHP = {'phpoffice/phpspreadsheet': parse_log_phpunit, 'laravel/framework': parse_log_phpunit, 'php-cs-fixer/php-cs-fixer': parse_log_phpunit, 'briannesbitt/carbon': parse_log_phpunit}
parse_log_phpunit 
parse_log_phpunit(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for phpunit logs with the --testdox option. Args: log (str): log content test_spec (TestSpec): test spec (unused) Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/php.py 
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
def parse_log_phpunit(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for phpunit logs with the --testdox option.
    Args:
        log (str): log content
        test_spec (TestSpec): test spec (unused)
    Returns:
        dict: test case to test status mapping
    """
    test_status_map = {}
    suite = None

    suite_pattern = r"^(\w.+) \(.+\)$"
    test_pattern = r"^\s*([✔✘↩])\s*(.*)$"

    for line in log.split("\n"):
        suite_match = re.match(suite_pattern, line)
        if suite_match:
            suite = suite_match.groups()[0]
            continue

        test_match = re.match(test_pattern, line)
        if test_match:
            status, test_name = test_match.groups()
            full_test_name = f"{suite} > {test_name}"

            if status == "✔":
                test_status_map[full_test_name] = TestStatus.PASSED.value
            elif status == "✘":
                test_status_map[full_test_name] = TestStatus.FAILED.value
            elif status == "↩":
                test_status_map[full_test_name] = TestStatus.SKIPPED.value

    return test_status_map
python
parse_log_astroid module-attribute 
parse_log_astroid = parse_log_pytest
parse_log_flask module-attribute 
parse_log_flask = parse_log_pytest
parse_log_marshmallow module-attribute 
parse_log_marshmallow = parse_log_pytest
parse_log_pvlib module-attribute 
parse_log_pvlib = parse_log_pytest
parse_log_pyvista module-attribute 
parse_log_pyvista = parse_log_pytest
parse_log_sqlfluff module-attribute 
parse_log_sqlfluff = parse_log_pytest
parse_log_xarray module-attribute 
parse_log_xarray = parse_log_pytest
parse_log_pydicom module-attribute 
parse_log_pydicom = parse_log_pytest_options
parse_log_requests module-attribute 
parse_log_requests = parse_log_pytest_options
parse_log_pylint module-attribute 
parse_log_pylint = parse_log_pytest_options
parse_log_astropy module-attribute 
parse_log_astropy = parse_log_pytest_v2
parse_log_scikit module-attribute 
parse_log_scikit = parse_log_pytest_v2
parse_log_sphinx module-attribute 
parse_log_sphinx = parse_log_pytest_v2
MAP_REPO_TO_PARSER_PY module-attribute 
MAP_REPO_TO_PARSER_PY = {'astropy/astropy': parse_log_astropy, 'django/django': parse_log_django, 'marshmallow-code/marshmallow': parse_log_marshmallow, 'matplotlib/matplotlib': parse_log_matplotlib, 'mwaskom/seaborn': parse_log_seaborn, 'pallets/flask': parse_log_flask, 'psf/requests': parse_log_requests, 'pvlib/pvlib-python': parse_log_pvlib, 'pydata/xarray': parse_log_xarray, 'pydicom/pydicom': parse_log_pydicom, 'pylint-dev/astroid': parse_log_astroid, 'pylint-dev/pylint': parse_log_pylint, 'pytest-dev/pytest': parse_log_pytest, 'pyvista/pyvista': parse_log_pyvista, 'scikit-learn/scikit-learn': parse_log_scikit, 'sqlfluff/sqlfluff': parse_log_sqlfluff, 'sphinx-doc/sphinx': parse_log_sphinx, 'sympy/sympy': parse_log_sympy}
parse_log_pytest 
parse_log_pytest(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated with PyTest framework

Parameters:

Name Type Description Default
log str

log content

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/python.py 
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
def parse_log_pytest(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated with PyTest framework

    Args:
        log (str): log content
    Returns:
        dict: test case to test status mapping
    """
    test_status_map = {}
    for line in log.split("\n"):
        if any([line.startswith(x.value) for x in TestStatus]):
            # Additional parsing for FAILED status
            if line.startswith(TestStatus.FAILED.value):
                line = line.replace(" - ", " ")
            test_case = line.split()
            if len(test_case) <= 1:
                continue
            if _is_skip_summary(test_case[0], test_case[1]):
                continue
            test_status_map[test_case[1]] = test_case[0]
    return test_status_map
parse_log_pytest_options 
parse_log_pytest_options(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated with PyTest framework with options

Parameters:

Name Type Description Default
log str

log content

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/python.py 
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
def parse_log_pytest_options(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated with PyTest framework with options

    Args:
        log (str): log content
    Returns:
        dict: test case to test status mapping
    """
    option_pattern = re.compile(r"(.*?)\[(.*)\]")
    test_status_map = {}
    for line in log.split("\n"):
        if any([line.startswith(x.value) for x in TestStatus]):
            # Additional parsing for FAILED status
            if line.startswith(TestStatus.FAILED.value):
                line = line.replace(" - ", " ")
            test_case = line.split()
            if len(test_case) <= 1:
                continue
            if _is_skip_summary(test_case[0], test_case[1]):
                continue
            has_option = option_pattern.search(test_case[1])
            if has_option:
                main, option = has_option.groups()
                if (
                    option.startswith("/")
                    and not option.startswith("//")
                    and "*" not in option
                ):
                    option = "/" + option.split("/")[-1]
                test_name = f"{main}[{option}]"
            else:
                test_name = test_case[1]
            test_status_map[test_name] = test_case[0]
    return test_status_map
parse_log_django 
parse_log_django(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated with Django tester framework

Parameters:

Name Type Description Default
log str

log content

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/python.py 
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
def parse_log_django(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated with Django tester framework

    Args:
        log (str): log content
    Returns:
        dict: test case to test status mapping
    """
    test_status_map = {}
    lines = log.split("\n")

    prev_test = None
    for line in lines:
        line = line.strip()

        # This isn't ideal but the test output spans multiple lines
        if "--version is equivalent to version" in line:
            test_status_map["--version is equivalent to version"] = (
                TestStatus.PASSED.value
            )

        # Log it in case of error
        if " ... " in line:
            prev_test = line.split(" ... ")[0]

        pass_suffixes = (" ... ok", " ... OK", " ...  OK")
        for suffix in pass_suffixes:
            if line.endswith(suffix):
                # TODO: Temporary, exclusive fix for django__django-7188
                # The proper fix should involve somehow getting the test results to
                # print on a separate line, rather than the same line
                if line.strip().startswith(
                    "Applying sites.0002_alter_domain_unique...test_no_migrations"
                ):
                    line = line.split("...", 1)[-1].strip()
                test = line.rsplit(suffix, 1)[0]
                test_status_map[test] = TestStatus.PASSED.value
                break
        if " ... skipped" in line:
            test = line.split(" ... skipped")[0]
            test_status_map[test] = TestStatus.SKIPPED.value
        if line.endswith(" ... FAIL"):
            test = line.split(" ... FAIL")[0]
            test_status_map[test] = TestStatus.FAILED.value
        if line.startswith("FAIL:"):
            test = line.split()[1].strip()
            test_status_map[test] = TestStatus.FAILED.value
        if line.endswith(" ... ERROR"):
            test = line.split(" ... ERROR")[0]
            test_status_map[test] = TestStatus.ERROR.value
        if line.startswith("ERROR:"):
            test = line.split()[1].strip()
            test_status_map[test] = TestStatus.ERROR.value

        if line.lstrip().startswith("ok") and prev_test is not None:
            # It means the test passed, but there's some additional output (including new lines)
            # between "..." and "ok" message
            test = prev_test
            test_status_map[test] = TestStatus.PASSED.value

    # TODO: This is very brittle, we should do better
    # There's a bug in the django logger, such that sometimes a test output near the end gets
    # interrupted by a particular long multiline print statement.
    # We have observed this in one of 3 forms:
    # - "{test_name} ... Testing against Django installed in {*} silenced.\nok"
    # - "{test_name} ... Internal Server Error: \/(.*)\/\nok"
    # - "{test_name} ... System check identified no issues (0 silenced).\nok"
    patterns = [
        r"^(.*?)\s\.\.\.\sTesting\ against\ Django\ installed\ in\ ((?s:.*?))\ silenced\)\.\nok$",
        r"^(.*?)\s\.\.\.\sInternal\ Server\ Error:\ \/(.*)\/\nok$",
        r"^(.*?)\s\.\.\.\sSystem check identified no issues \(0 silenced\)\nok$",
    ]
    for pattern in patterns:
        for match in re.finditer(pattern, log, re.MULTILINE):
            test_name = match.group(1)
            test_status_map[test_name] = TestStatus.PASSED.value
    return test_status_map
parse_log_pytest_v2 
parse_log_pytest_v2(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated with PyTest framework (Later Version)

Parameters:

Name Type Description Default
log str

log content

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/python.py 
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
def parse_log_pytest_v2(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated with PyTest framework (Later Version)

    Args:
        log (str): log content
    Returns:
        dict: test case to test status mapping
    """
    test_status_map = {}
    escapes = "".join([chr(char) for char in range(1, 32)])
    for line in log.split("\n"):
        line = re.sub(r"\[(\d+)m", "", line)
        translator = str.maketrans("", "", escapes)
        line = line.translate(translator)
        if any([line.startswith(x.value) for x in TestStatus]):
            if line.startswith(TestStatus.FAILED.value):
                # drop the trailing " - <assertion message>" so it can't enter the id
                line = line.split(" - ", 1)[0]
            test_case = line.split()
            if len(test_case) >= 2 and not _is_skip_summary(test_case[0], test_case[1]):
                test_status_map[" ".join(test_case[1:])] = test_case[0]
        # Support older pytest versions by checking if the line ends with the test status
        elif any([line.endswith(x.value) for x in TestStatus]):
            test_case = line.split()
            if len(test_case) >= 2:
                test_status_map[" ".join(test_case[:-1])] = test_case[-1]
    return test_status_map
parse_log_seaborn 
parse_log_seaborn(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated with seaborn testing framework

Parameters:

Name Type Description Default
log str

log content

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/python.py 
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
def parse_log_seaborn(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated with seaborn testing framework

    Args:
        log (str): log content
    Returns:
        dict: test case to test status mapping
    """
    test_status_map = {}
    for line in log.split("\n"):
        parts = line.split()
        if len(parts) < 2:
            continue
        if line.startswith(TestStatus.FAILED.value):
            test_case = parts[1]
            test_status_map[test_case] = TestStatus.FAILED.value
        elif f" {TestStatus.PASSED.value} " in line:
            if parts[1] == TestStatus.PASSED.value:
                test_case = parts[0]
                test_status_map[test_case] = TestStatus.PASSED.value
        elif line.startswith(TestStatus.PASSED.value):
            test_case = parts[1]
            test_status_map[test_case] = TestStatus.PASSED.value
    return test_status_map
parse_log_sympy 
parse_log_sympy(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated with Sympy framework

Parameters:

Name Type Description Default
log str

log content

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/python.py 
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
def parse_log_sympy(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated with Sympy framework

    Args:
        log (str): log content
    Returns:
        dict: test case to test status mapping
    """
    test_status_map = {}
    pattern = r"(_*) (.*)\.py:(.*) (_*)"
    matches = re.findall(pattern, log)
    for match in matches:
        test_case = f"{match[1]}.py:{match[2]}"
        test_status_map[test_case] = TestStatus.FAILED.value
    for line in log.split("\n"):
        line = line.strip()
        if line.startswith("test_"):
            if line.endswith(" E"):
                test = line.split()[0]
                test_status_map[test] = TestStatus.ERROR.value
            if line.endswith(" F"):
                test = line.split()[0]
                test_status_map[test] = TestStatus.FAILED.value
            if line.endswith(" ok"):
                test = line.split()[0]
                test_status_map[test] = TestStatus.PASSED.value
    return test_status_map
parse_log_matplotlib 
parse_log_matplotlib(log: str, test_spec: TestSpec) -> dict[str, str]

Parser for test logs generated with PyTest framework

Parameters:

Name Type Description Default
log str

log content

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/python.py 
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
def parse_log_matplotlib(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Parser for test logs generated with PyTest framework

    Args:
        log (str): log content
    Returns:
        dict: test case to test status mapping
    """
    test_status_map = {}
    for line in log.split("\n"):
        line = line.replace("MouseButton.LEFT", "1")
        line = line.replace("MouseButton.RIGHT", "3")
        if any([line.startswith(x.value) for x in TestStatus]):
            # Additional parsing for FAILED status
            if line.startswith(TestStatus.FAILED.value):
                line = line.replace(" - ", " ")
            test_case = line.split()
            if len(test_case) <= 1:
                continue
            if _is_skip_summary(test_case[0], test_case[1]):
                continue
            test_status_map[test_case[1]] = test_case[0]
    return test_status_map
ruby
MAP_REPO_TO_PARSER_RUBY module-attribute 
MAP_REPO_TO_PARSER_RUBY = {'jekyll/jekyll': parse_log_jekyll, 'fluent/fluentd': parse_log_ruby_unit, 'fastlane/fastlane': parse_log_rspec_transformed_json, 'jordansissel/fpm': parse_log_rspec_transformed_json, 'faker-ruby/faker': parse_log_ruby_unit, 'rubocop/rubocop': parse_log_rspec_transformed_json}
parse_log_minitest 
parse_log_minitest(log: str, test_spec: TestSpec) -> dict[str, str]

Parameters:

Name Type Description Default
log str

log content

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/ruby.py 
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
def parse_log_minitest(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Args:
        log (str): log content
    Returns:
        dict: test case to test status mapping
    """
    test_status_map = {}

    pattern = r"^(.+)\. .*=.*(\.|F|E).*$"

    for line in log.split("\n"):
        match = re.match(pattern, line.strip())
        if match:
            test_name, outcome = match.groups()
            if outcome == ".":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif outcome in ["F", "E"]:
                test_status_map[test_name] = TestStatus.FAILED.value

    return test_status_map
parse_log_cucumber 
parse_log_cucumber(log: str, test_spec: TestSpec) -> dict[str, str]

Assumes --format progress is used.

Source code in swebench/harness/log_parsers/ruby.py 
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
def parse_log_cucumber(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Assumes --format progress is used.
    """
    test_status_map = {}

    pattern = r"^(.*) \.+(\.|F)"

    for line in log.split("\n"):
        match = re.match(pattern, line.strip())
        if match:
            test_name, outcome = match.groups()
            if outcome == ".":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif outcome == "F":
                test_status_map[test_name] = TestStatus.FAILED.value

    return test_status_map
parse_log_ruby_unit 
parse_log_ruby_unit(log: str, test_spec: TestSpec) -> dict[str, str]
Source code in swebench/harness/log_parsers/ruby.py 
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
def parse_log_ruby_unit(log: str, test_spec: TestSpec) -> dict[str, str]:
    test_status_map = {}

    pattern = r"^\s*(?:test: )?(.+):\s+(\.|E\b|F\b|O\b)"

    for line in log.split("\n"):
        match = re.match(pattern, line.strip())
        if match:
            test_name, outcome = match.groups()
            if outcome == ".":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif outcome in ["E", "F"]:
                test_status_map[test_name] = TestStatus.FAILED.value
            elif outcome == "O":
                test_status_map[test_name] = TestStatus.SKIPPED.value

    return test_status_map
parse_log_rspec_transformed_json 
parse_log_rspec_transformed_json(log: str, test_spec: TestSpec) -> dict[str, str]
Source code in swebench/harness/log_parsers/ruby.py 
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
def parse_log_rspec_transformed_json(log: str, test_spec: TestSpec) -> dict[str, str]:
    test_status_map = {}

    pattern = r"(.+) - (passed|failed)"

    for line in log.split("\n"):
        match = re.match(pattern, line.strip())
        if match:
            test_name, outcome = match.groups()
            if outcome == "passed":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif outcome == "failed":
                test_status_map[test_name] = TestStatus.FAILED.value
            elif outcome == "pending":
                test_status_map[test_name] = TestStatus.SKIPPED.value
            else:
                raise ValueError(f"Unknown outcome: {outcome}")

    return test_status_map
parse_log_jekyll 
parse_log_jekyll(log: str, test_spec: TestSpec) -> dict[str, str]

Different jekyll instances use different test runners and log formats. This function selects the appropriate log parser based on the instance id.

Source code in swebench/harness/log_parsers/ruby.py 
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
def parse_log_jekyll(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Different jekyll instances use different test runners and log formats.
    This function selects the appropriate log parser based on the instance id.
    """
    pr_number = test_spec.instance_id.split("-")[1]

    if pr_number in ["9141", "8047", "8167"]:
        return parse_log_minitest(log, test_spec)
    elif pr_number in ["8761", "8771"]:
        return parse_log_cucumber(log, test_spec)
    else:
        raise ValueError(f"Unknown instance id: {test_spec.instance_id}")
rust
MAP_REPO_TO_PARSER_RUST module-attribute 
MAP_REPO_TO_PARSER_RUST = {'burntsushi/ripgrep': parse_log_cargo, 'sharkdp/bat': parse_log_cargo, 'astral-sh/ruff': parse_log_cargo, 'tokio-rs/tokio': parse_log_cargo, 'uutils/coreutils': parse_log_cargo, 'nushell/nushell': parse_log_cargo, 'tokio-rs/axum': parse_log_cargo}
parse_log_cargo 
parse_log_cargo(log: str, test_spec: TestSpec) -> dict[str, str]

Parameters:

Name Type Description Default
log str

log content

 required

Returns: dict: test case to test status mapping

Source code in swebench/harness/log_parsers/rust.py 
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
def parse_log_cargo(log: str, test_spec: TestSpec) -> dict[str, str]:
    """
    Args:
        log (str): log content
    Returns:
        dict: test case to test status mapping
    """
    test_status_map = {}

    pattern = r"^test\s+(\S+)\s+\.\.\.\s+(\w+)$"

    for line in log.split("\n"):
        match = re.match(pattern, line.strip())
        if match:
            test_name, outcome = match.groups()
            if outcome == "ok":
                test_status_map[test_name] = TestStatus.PASSED.value
            elif outcome == "FAILED":
                test_status_map[test_name] = TestStatus.FAILED.value

    return test_status_map

modal_eval

__all__ module-attribute 
__all__ = ['run_instances_modal', 'validate_modal_credentials']
run_instances_modal 
run_instances_modal(predictions: dict, instances: list, full_dataset: list, run_id: str, timeout: int)

Run all instances for the given predictions on Modal.

Parameters:

Name Type Description Default
predictions dict

Predictions dict generated by the model

 required
instances list

List of instances

 required
run_id str

Run ID

 required
timeout int

Timeout for running tests

 required
Source code in swebench/harness/modal_eval/run_evaluation_modal.py 
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
def run_instances_modal(
    predictions: dict,
    instances: list,
    full_dataset: list,
    run_id: str,
    timeout: int,
):
    """
    Run all instances for the given predictions on Modal.

    Args:
        predictions (dict): Predictions dict generated by the model
        instances (list): List of instances
        run_id (str): Run ID
        timeout (int): Timeout for running tests
    """
    test_specs = [make_test_spec(inst) for inst in instances]

    with modal.enable_output():
        with app.run():
            run_test_specs = []

            # Check for instances that have already been run
            for test_spec in test_specs:
                log_dir = get_log_dir(
                    predictions[test_spec.instance_id], run_id, test_spec.instance_id
                )
                if log_dir.exists():
                    continue
                run_test_specs.append(test_spec)

            if run_test_specs:
                # Run instances that haven't been run yet
                results = run_instance_modal.starmap(
                    [
                        (
                            test_spec,
                            predictions[test_spec.instance_id],
                            run_id,
                            timeout,
                        )
                        for test_spec in run_test_specs
                    ],
                    return_exceptions=True,
                )

                for result in results:
                    if not isinstance(result, TestOutput):
                        print(f"Result failed with error: {result}")
                        continue

                    # Save logs locally
                    log_dir = result.log_dir
                    log_dir.mkdir(parents=True, exist_ok=True)
                    with open(log_dir / "run_instance.log", "w") as f:
                        f.write(result.run_instance_log)
                    with open(log_dir / "test_output.txt", "w") as f:
                        f.write(result.test_output)
                    with open(log_dir / "patch.diff", "w") as f:
                        f.write(result.patch_diff)
                    with open(log_dir / "report.json", "w") as f:
                        try:
                            report_json = json.loads(result.report_json_str)
                            json.dump(report_json, f, indent=4)
                        except Exception:
                            # This happens if the test fails with any exception
                            print(f"{result.instance_id}: no report.json")

            make_run_report(predictions, full_dataset, run_id)
validate_modal_credentials 
validate_modal_credentials()

Validate that Modal credentials exist by checking for ~/.modal.toml file. Raises an exception if credentials are not configured.

Source code in swebench/harness/modal_eval/utils.py 
 4
 5
 6
 7
 8
 9
10
11
12
13
14
def validate_modal_credentials():
    """
    Validate that Modal credentials exist by checking for ~/.modal.toml file.
    Raises an exception if credentials are not configured.
    """
    modal_config_path = Path.home() / ".modal.toml"
    if not modal_config_path.exists():
        raise RuntimeError(
            "~/.modal.toml not found - it looks like you haven't configured credentials for Modal.\n"
            "Run 'modal token new' in your terminal to configure credentials."
        )
run_evaluation_modal
SANDBOX_ENTRYPOINT module-attribute 
SANDBOX_ENTRYPOINT = 'run_evaluation_modal_entrypoint'
LOCAL_SANDBOX_ENTRYPOINT_PATH module-attribute 
LOCAL_SANDBOX_ENTRYPOINT_PATH = (Path(__file__).parent / f'{SANDBOX_ENTRYPOINT}.py').resolve()
REMOTE_SANDBOX_ENTRYPOINT_PATH module-attribute 
REMOTE_SANDBOX_ENTRYPOINT_PATH = f'/root/{SANDBOX_ENTRYPOINT}.py'
app module-attribute 
app = modal.App('swebench-evaluation')
swebench_image module-attribute 
swebench_image = modal.Image.debian_slim().pip_install('swebench', 'tenacity')
TestOutput dataclass 
TestOutput(instance_id: str, test_output: str, report_json_str: str, run_instance_log: str, patch_diff: str, log_dir: Path, errored: bool)
instance_id instance-attribute 
instance_id: str
test_output instance-attribute 
test_output: str
report_json_str instance-attribute 
report_json_str: str
run_instance_log instance-attribute 
run_instance_log: str
patch_diff instance-attribute 
patch_diff: str
log_dir instance-attribute 
log_dir: Path
errored instance-attribute 
errored: bool
ModalSandboxRuntime 
ModalSandboxRuntime(test_spec: TestSpec, timeout: int | None = None, verbose: bool = True)

Runtime for running instances in a Modal Sandbox.

Source code in swebench/harness/modal_eval/run_evaluation_modal.py 
59
60
61
62
63
64
65
66
67
68
69
def __init__(
    self, test_spec: TestSpec, timeout: int | None = None, verbose: bool = True
):
    self.test_spec = test_spec
    self.image = ModalSandboxRuntime.get_instance_image(test_spec)
    self.sandbox = self._get_sandbox(timeout)
    self.verbose = verbose
    self._stream_tasks = []

    # Hack for pylint
    self.write_file("/sys/fs/cgroup/cpu/cpu.shares", "2048")
test_spec instance-attribute 
test_spec = test_spec
image instance-attribute 
image = ModalSandboxRuntime.get_instance_image(test_spec)
sandbox instance-attribute 
sandbox = self._get_sandbox(timeout)
verbose instance-attribute 
verbose = verbose
write_file 
write_file(file_path: str, content: str)
Source code in swebench/harness/modal_eval/run_evaluation_modal.py 
119
120
def write_file(self, file_path: str, content: str):
    self.sandbox.open(file_path, "w").write(content)
exec 
exec(command: str) -> tuple[str, int]

Execute a command in the sandbox.

Returns:

Type Description
tuple[str, int]

tuple[str, int]: Sandbox output and return code.
Source code in swebench/harness/modal_eval/run_evaluation_modal.py 
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
def exec(self, command: str) -> tuple[str, int]:
    """
    Execute a command in the sandbox.

    Returns:
        tuple[str, int]: Sandbox output and return code.
    """
    p = self.sandbox.exec("python", "-m", SANDBOX_ENTRYPOINT, command)
    stdout = []
    stderr = []
    try:
        # We separate stdout/stderr because some tests rely on them being separate.
        # We still read stdout/stderr simultaneously to continuously
        # flush both streams and avoid blocking.
        asyncio.run(self._read_output(p, stdout, stderr))
    except Exception as e:
        print(f"Error during command execution: {e}")
    p.wait()
    return "".join(stdout + stderr), p.returncode
__exit__ 
__exit__(exc_type, exc_val, exc_tb)
Source code in swebench/harness/modal_eval/run_evaluation_modal.py 
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
def __exit__(self, exc_type, exc_val, exc_tb):
    if self._stream_tasks:
        try:
            # Forcefully kill remaining streams
            for task in self._stream_tasks:
                if not task.done():
                    task.cancel()
                    try:
                        asyncio.wait_for(task, timeout=0.1)
                    except asyncio.TimeoutError:
                        pass
                    except Exception:
                        pass

            self.sandbox.terminate()
        except Exception:
            pass
        finally:
            self._stream_tasks = []
get_instance_image staticmethod 
get_instance_image(test_spec: TestSpec) -> Image
Source code in swebench/harness/modal_eval/run_evaluation_modal.py 
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
@staticmethod
def get_instance_image(test_spec: TestSpec) -> modal.Image:
    # TODO: setup_env_script and install_repo_script are not part of the
    # current TestSpec dataclass.  This method needs to be updated to work
    # with pre-built images or to source these scripts from elsewhere.
    env_script = test_spec.setup_env_script
    # add trusted host flag for Modal's PyPI mirror
    env_script = env_script.replace(
        "conda activate testbed && python -m pip install -r $HOME/requirements.txt",
        "conda activate testbed && python -m pip install --trusted-host pypi-mirror.modal.local -r $HOME/requirements.txt",
    )
    repo_script = test_spec.install_repo_script

    remote_env_script_path = "/root/setup_env.sh"
    remote_repo_script_path = "/root/setup_repo.sh"

    Path(remote_env_script_path).write_text(env_script)
    Path(remote_repo_script_path).write_text(repo_script)

    # Modal automatically caches images
    # https://modal.com/docs/guide/custom-container#image-caching-and-rebuilds
    return (
        modal.Image.from_registry("ubuntu:22.04", add_python="3.11")
        .run_commands("apt update")
        .env({"DEBIAN_FRONTEND": "noninteractive", "TZ": "Etc/UTC"})
        .apt_install(
            "wget",
            "git",
            "build-essential",
            "libffi-dev",
            "libtiff-dev",
            "jq",
            "curl",
            "locales",
            "locales-all",
            "tzdata",
        )
        .run_commands(
            "wget 'https://repo.anaconda.com/miniconda/Miniconda3-py311_23.11.0-2-Linux-x86_64.sh' -O miniconda.sh",
            "bash miniconda.sh -b -p /opt/miniconda3",
            "echo 'export PATH=/opt/miniconda3/bin:$PATH' >> ~/.bashrc",
            "/opt/miniconda3/bin/conda init --all",
            "/opt/miniconda3/bin/conda config --append channels conda-forge",
            "adduser --disabled-password --gecos 'dog' nonroot",
        )
        .add_local_file(
            Path(remote_env_script_path), remote_env_script_path, copy=True
        )
        .add_local_file(
            Path(remote_repo_script_path), remote_repo_script_path, copy=True
        )
        .run_commands(
            f"chmod +x {remote_env_script_path}",
            f"/bin/bash -c 'source ~/.bashrc && {remote_env_script_path}'",
            "echo 'source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbed' >> /root/.bashrc",
            f"/bin/bash {remote_repo_script_path}",
        )
        .workdir("/testbed/")
    )
get_log_dir 
get_log_dir(pred: dict, run_id: str, instance_id: str) -> Path
Source code in swebench/harness/modal_eval/run_evaluation_modal.py 
223
224
225
226
227
def get_log_dir(pred: dict, run_id: str, instance_id: str) -> Path:
    model_name_or_path = cast(
        str, pred.get("model_name_or_path", "None").replace("/", "__")
    )
    return RUN_EVALUATION_LOG_DIR / run_id / model_name_or_path / instance_id
run_instance_modal 
run_instance_modal(test_spec: TestSpec, pred: dict, run_id: str, timeout: int | None = None) -> TestOutput

Run a single instance with the given prediction.

Parameters:

Name Type Description Default
test_spec TestSpec

TestSpec instance

 required
pred dict

Prediction w/ model_name_or_path, model_patch, instance_id

 required
run_id str

Run ID

 required
timeout int

Timeout for running tests

 None
Source code in swebench/harness/modal_eval/run_evaluation_modal.py 
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
@app.function(
    image=swebench_image.add_local_file(
        LOCAL_SANDBOX_ENTRYPOINT_PATH,
        REMOTE_SANDBOX_ENTRYPOINT_PATH,
    ),
    timeout=120
    * 60,  # Much larger than default timeout to account for image build time
    include_source=True,
)
def run_instance_modal(
    test_spec: TestSpec,
    pred: dict,
    run_id: str,
    timeout: int | None = None,
) -> TestOutput:
    """
    Run a single instance with the given prediction.

    Args:
        test_spec (TestSpec): TestSpec instance
        pred (dict): Prediction w/ model_name_or_path, model_patch, instance_id
        run_id (str): Run ID
        timeout (int): Timeout for running tests
    """
    instance_id = test_spec.instance_id
    log_dir = get_log_dir(pred, run_id, instance_id)
    log_dir.mkdir(parents=True, exist_ok=True)

    log_file = log_dir / "run_instance.log"

    logger = setup_logger(instance_id, log_file, add_stdout=True)

    try:
        runner = ModalSandboxRuntime(test_spec, timeout)
    except Exception as e:
        print(f"Error creating sandbox: {e}")
        raise EvaluationError(
            instance_id,
            f"Error creating sandbox: {e}",
            logger,
        ) from e

    patch_diff = pred.get("model_patch", "")

    try:
        patch_file = "/tmp/patch.diff"
        runner.write_file(patch_file, patch_diff)

        apply_patch_output, returncode = runner.exec(
            "cd /testbed && git apply -v /tmp/patch.diff",
        )

        if returncode != 0:
            logger.info("Failed to apply patch to container, trying again...")

            apply_patch_output, returncode = runner.exec(
                "cd /testbed && patch --batch --fuzz=5 -p1 -i /tmp/patch.diff",
            )

            if returncode != 0:
                logger.info(f"{APPLY_PATCH_FAIL}:\n{apply_patch_output}")
                raise EvaluationError(
                    instance_id,
                    f"{APPLY_PATCH_FAIL}:\n{apply_patch_output}",
                    logger,
                )
            else:
                logger.info(f"{APPLY_PATCH_PASS}:\n{apply_patch_output}")
        else:
            logger.info(f"{APPLY_PATCH_PASS}:\n{apply_patch_output}")

        # Get git diff before running eval script
        git_diff_output_before, returncode = runner.exec(
            "cd /testbed && git diff",
        )
        logger.info(f"Git diff before:\n{git_diff_output_before}")

        eval_file = "/root/eval.sh"
        eval_script = test_spec.eval_script
        # django hack
        eval_script = eval_script.replace("locale-gen", "locale-gen en_US.UTF-8")
        runner.write_file(eval_file, eval_script)

        start_time = time.time()

        run_command = "cd /testbed"
        # pylint hack
        if "pylint" in test_spec.instance_id:
            run_command += " && PYTHONPATH="
        # a `python3 -c 'sys.setrecursionlimit(...)'` step used to sit here, meaning to
        # give the tests more stack. It set the limit in its own process and exited, so
        # the eval script below never saw it -- it only looked like the local runs
        # differed from Modal. Removed rather than left as decoration.
        # run eval script
        run_command += " && /bin/bash /root/eval.sh"
        test_output, returncode = runner.exec(run_command)

        total_runtime = time.time() - start_time

        test_output_path = log_dir / "test_output.txt"
        logger.info(f"Test runtime: {total_runtime:_.2f} seconds")
        with open(test_output_path, "w") as f:
            f.write(test_output)
            logger.info(f"Test output for {instance_id} written to {test_output_path}")
            print(f"Test output for {instance_id} written to {test_output_path}")

        # Get git diff after running eval script
        git_diff_output_after, returncode = runner.exec("cd /testbed && git diff")

        # Check if git diff changed after running eval script
        logger.info(f"Git diff after:\n{git_diff_output_after}")
        if git_diff_output_after != git_diff_output_before:
            logger.info("Git diff changed after running eval script")

        # Get report from test output
        logger.info(f"Grading answer for {instance_id}...")
        report = get_eval_report(
            test_spec=test_spec,
            prediction=pred,
            test_log_path=test_output_path,
            include_tests_status=True,
        )
        logger.info(
            f"report: {report}\n"
            f"Result for {instance_id}: resolved: {report[instance_id]['resolved']}"
        )

        return TestOutput(
            instance_id=instance_id,
            test_output=test_output,
            report_json_str=json.dumps(report, indent=4),
            run_instance_log=log_file.read_text(),
            patch_diff=patch_diff,
            log_dir=log_dir,
            errored=False,
        )
    except modal.exception.SandboxTimeoutError as e:
        raise EvaluationError(
            instance_id,
            f"Test timed out after {timeout} seconds.",
            logger,
        ) from e
    except EvaluationError:
        error_msg = traceback.format_exc()
        logger.info(error_msg)
        return TestOutput(
            instance_id=instance_id,
            test_output="",
            report_json_str="",
            run_instance_log=log_file.read_text(),
            patch_diff=patch_diff,
            log_dir=log_dir,
            errored=True,
        )
    except Exception as e:
        error_msg = (
            f"Error in evaluating model for {instance_id}: {e}\n"
            f"{traceback.format_exc()}\n"
            f"Check ({logger.log_file}) for more information."
        )
        logger.error(error_msg)
        return TestOutput(
            instance_id=instance_id,
            test_output="",
            report_json_str="",
            run_instance_log=log_file.read_text(),
            patch_diff=patch_diff,
            log_dir=log_dir,
            errored=True,
        )
run_instances_modal 
run_instances_modal(predictions: dict, instances: list, full_dataset: list, run_id: str, timeout: int)

Run all instances for the given predictions on Modal.

Parameters:

Name Type Description Default
predictions dict

Predictions dict generated by the model

 required
instances list

List of instances

 required
run_id str

Run ID

 required
timeout int

Timeout for running tests

 required
Source code in swebench/harness/modal_eval/run_evaluation_modal.py 
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
def run_instances_modal(
    predictions: dict,
    instances: list,
    full_dataset: list,
    run_id: str,
    timeout: int,
):
    """
    Run all instances for the given predictions on Modal.

    Args:
        predictions (dict): Predictions dict generated by the model
        instances (list): List of instances
        run_id (str): Run ID
        timeout (int): Timeout for running tests
    """
    test_specs = [make_test_spec(inst) for inst in instances]

    with modal.enable_output():
        with app.run():
            run_test_specs = []

            # Check for instances that have already been run
            for test_spec in test_specs:
                log_dir = get_log_dir(
                    predictions[test_spec.instance_id], run_id, test_spec.instance_id
                )
                if log_dir.exists():
                    continue
                run_test_specs.append(test_spec)

            if run_test_specs:
                # Run instances that haven't been run yet
                results = run_instance_modal.starmap(
                    [
                        (
                            test_spec,
                            predictions[test_spec.instance_id],
                            run_id,
                            timeout,
                        )
                        for test_spec in run_test_specs
                    ],
                    return_exceptions=True,
                )

                for result in results:
                    if not isinstance(result, TestOutput):
                        print(f"Result failed with error: {result}")
                        continue

                    # Save logs locally
                    log_dir = result.log_dir
                    log_dir.mkdir(parents=True, exist_ok=True)
                    with open(log_dir / "run_instance.log", "w") as f:
                        f.write(result.run_instance_log)
                    with open(log_dir / "test_output.txt", "w") as f:
                        f.write(result.test_output)
                    with open(log_dir / "patch.diff", "w") as f:
                        f.write(result.patch_diff)
                    with open(log_dir / "report.json", "w") as f:
                        try:
                            report_json = json.loads(result.report_json_str)
                            json.dump(report_json, f, indent=4)
                        except Exception:
                            # This happens if the test fails with any exception
                            print(f"{result.instance_id}: no report.json")

            make_run_report(predictions, full_dataset, run_id)
run_evaluation_modal_entrypoint
STDIO_RATE_LIMIT_BYTES_PER_SEC module-attribute 
STDIO_RATE_LIMIT_BYTES_PER_SEC = 64 * 1024 // 2
parser module-attribute 
parser = argparse.ArgumentParser(description='Execute a shell command and stream output')
args module-attribute 
args = parser.parse_args()
exec async 
exec(command: str) -> int
Source code in swebench/harness/modal_eval/run_evaluation_modal_entrypoint.py 
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
async def exec(command: str) -> int:
    p = await asyncio.create_subprocess_shell(
        command,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
        limit=1024 * 1024,
    )

    stdout_lines = []
    stderr_lines = []

    async def read_stream(stream, lines, fd):
        tokens = STDIO_RATE_LIMIT_BYTES_PER_SEC
        last_refill = asyncio.get_event_loop().time()

        while True:
            try:
                line = await stream.readline()
                if not line:
                    break
            except (asyncio.LimitOverrunError, ValueError):
                # buffer exceeded asyncio stream limit
                fallback_chunk_size = 8192
                line = await stream.read(fallback_chunk_size)
                if not line:
                    break

            remaining_data = line
            buffer = bytearray()

            while remaining_data:
                current_time = asyncio.get_event_loop().time()
                time_passed = current_time - last_refill

                tokens = min(
                    STDIO_RATE_LIMIT_BYTES_PER_SEC,
                    tokens + (time_passed * STDIO_RATE_LIMIT_BYTES_PER_SEC),
                )
                last_refill = current_time

                chunk_size = min(
                    len(remaining_data), STDIO_RATE_LIMIT_BYTES_PER_SEC, int(tokens)
                )

                if chunk_size == 0:
                    sleep_time = max(
                        0.01,
                        (0.01 * STDIO_RATE_LIMIT_BYTES_PER_SEC - tokens)
                        / STDIO_RATE_LIMIT_BYTES_PER_SEC,
                    )
                    await asyncio.sleep(sleep_time)
                    continue

                buffer.extend(remaining_data[:chunk_size])

                # Find last valid UTF-8 character boundary.
                # This is to avoid partial characters being written to
                # container stdout/stderr, which results in a very small
                # chance of errors of the form: "Error reading stream: 'utf-8' codec can't decode bytes in position ..."
                valid_bytes = len(
                    buffer.decode("utf-8", errors="ignore").encode("utf-8")
                )

                if valid_bytes > 0:
                    chunk = buffer[:valid_bytes]
                    if fd == "stdout":
                        sys.stdout.buffer.write(chunk)
                        sys.stdout.buffer.flush()
                    else:
                        sys.stderr.buffer.write(chunk)
                        sys.stderr.buffer.flush()

                    buffer = buffer[valid_bytes:]
                    tokens -= valid_bytes

                remaining_data = remaining_data[chunk_size:]

            if buffer:
                if fd == "stdout":
                    sys.stdout.buffer.write(buffer)
                    sys.stdout.buffer.flush()
                else:
                    sys.stderr.buffer.write(buffer)
                    sys.stderr.buffer.flush()

            lines.append(line)

    await asyncio.gather(
        read_stream(p.stdout, stdout_lines, "stdout"),
        read_stream(p.stderr, stderr_lines, "stderr"),
    )

    return await p.wait()
main async 
main(command: str)
Source code in swebench/harness/modal_eval/run_evaluation_modal_entrypoint.py 
111
112
113
async def main(command: str):
    returncode = await exec(command)
    exit(returncode)
utils
validate_modal_credentials 
validate_modal_credentials()

Validate that Modal credentials exist by checking for ~/.modal.toml file. Raises an exception if credentials are not configured.

Source code in swebench/harness/modal_eval/utils.py 
 4
 5
 6
 7
 8
 9
10
11
12
13
14
def validate_modal_credentials():
    """
    Validate that Modal credentials exist by checking for ~/.modal.toml file.
    Raises an exception if credentials are not configured.
    """
    modal_config_path = Path.home() / ".modal.toml"
    if not modal_config_path.exists():
        raise RuntimeError(
            "~/.modal.toml not found - it looks like you haven't configured credentials for Modal.\n"
            "Run 'modal token new' in your terminal to configure credentials."
        )

remove_containers

parser module-attribute 
parser = ArgumentParser(description=__doc__)
args module-attribute 
args = parser.parse_args()
main 
main(instance_ids=None, predictions_path=None, run_id=None)
Source code in swebench/harness/remove_containers.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
def main(instance_ids=None, predictions_path=None, run_id=None):
    all_ids = set()
    if predictions_path:
        with open(predictions_path, "r") as f:
            predictions = json.loads(f.read())
            for pred in predictions:
                all_ids.add(pred["instance_id"])

    if instance_ids:
        all_ids |= set(instance_ids)

    if not all_ids and not run_id:
        print("Provide --instance_ids, --predictions_path or --run_id, exiting.")
        return

    client = _docker_client()
    # containers are named sweb.eval.<instance_id>.<run_id>, so match on the
    # prefix rather than an exact name
    removed = 0
    for container in client.containers.list(all=True):
        name = container.name
        if not name.startswith("sweb.eval."):
            continue
        rest = name[len("sweb.eval.") :]
        instance_id, _, container_run = rest.rpartition(".")
        if not instance_id:  # no run id in the name
            instance_id, container_run = rest, ""
        if all_ids and instance_id not in all_ids:
            continue
        if run_id and container_run != run_id:
            continue
        try:
            container.remove(force=True)
            print(f"Removed container {name}")
            removed += 1
        except Exception as e:
            print(f"Error removing container {name}: {e}")
    print(f"Removed {removed} container(s).")

reporting

make_run_report 
make_run_report(predictions: dict, full_dataset: list, run_id: str, client: Optional[DockerClient] = None) -> Path

Make a final evaluation and run report of the instances that have been run. Also reports on images and containers that may still running if client is provided.

Parameters:

Name Type Description Default
predictions dict

Predictions dict generated by the model

 required
full_dataset list

List of all instances

 required
run_id str

Run ID

 required
client DockerClient

Docker client (optional)

 None

Returns:

Type Description
Path

Path to report file
Source code in swebench/harness/reporting.py 
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
def make_run_report(
    predictions: dict,
    full_dataset: list,
    run_id: str,
    client: Optional[docker.DockerClient] = None,
) -> Path:
    """
    Make a final evaluation and run report of the instances that have been run.
    Also reports on images and containers that may still running if client is provided.

    Args:
        predictions (dict): Predictions dict generated by the model
        full_dataset (list): List of all instances
        run_id (str): Run ID
        client (docker.DockerClient): Docker client (optional)

    Returns:
        Path to report file
    """
    # instantiate sets to store IDs of different outcomes
    completed_ids = set()
    resolved_ids = set()
    error_ids = set()
    unstopped_containers = set()
    unremoved_images = set()
    unresolved_ids = set()
    incomplete_ids = set()
    infra_failure_ids = set()
    ambiguous_failure_ids = set()
    # get instances with empty patches
    empty_patch_ids = set()

    # iterate through dataset and check if the instance has been run
    for instance in full_dataset:
        instance_id = instance["instance_id"]
        if instance_id not in predictions:
            # skip instances without predictions
            incomplete_ids.add(instance_id)
            continue
        prediction = predictions[instance_id]
        if prediction.get("model_patch", None) in ["", None]:
            empty_patch_ids.add(instance_id)
            continue
        report_file = (
            RUN_EVALUATION_LOG_DIR
            / run_id
            / prediction["model_name_or_path"].replace("/", "__")
            / prediction["instance_id"]
            / LOG_REPORT
        )
        if report_file.exists():
            # If report file exists, then the instance has been run
            completed_ids.add(instance_id)
            try:
                content = report_file.read_text().strip()
                if not content:  # Empty file
                    error_ids.add(instance_id)
                    continue

                report = json.loads(content)
                if report[instance_id]["resolved"]:
                    # Record if the instance was resolved
                    resolved_ids.add(instance_id)
                else:
                    unresolved_ids.add(instance_id)
            except (json.JSONDecodeError, KeyError):
                # If the report file is not valid JSON or missing keys, treat as error
                error_ids.add(instance_id)
        else:
            # Otherwise, the instance was not run successfully
            error_ids.add(instance_id)

    # Classify why the non-resolved instances failed (#586). Purely additive:
    # these ids stay in unresolved_ids/error_ids, so the denominator is unchanged.
    infra_failure_reasons = {}
    for instance_id in unresolved_ids | error_ids:
        prediction = predictions[instance_id]
        instance_log_dir = (
            RUN_EVALUATION_LOG_DIR
            / run_id
            / prediction["model_name_or_path"].replace("/", "__")
            / instance_id
        )
        classification = classify_logs(
            instance_log_dir / LOG_TEST_OUTPUT, instance_log_dir / LOG_INSTANCE
        )
        if classification:
            reason, tier = classification
            infra_failure_reasons[instance_id] = reason
            if tier == TIER_ENVIRONMENT:
                infra_failure_ids.add(instance_id)
            else:
                ambiguous_failure_ids.add(instance_id)

    if client:
        # get remaining images and containers
        images = list_images(client)
        for instance in full_dataset:
            image_name = instance.get("image", "")
            if image_name in images:
                unremoved_images.add(image_name)
        containers = client.containers.list(all=True)
        for container in containers:
            if run_id in container.name:
                unstopped_containers.add(container.name)

    # print final report
    dataset_ids = {i["instance_id"] for i in full_dataset}
    print(f"Total instances: {len(full_dataset)}")
    print(f"Instances submitted: {len(set(predictions.keys()) & dataset_ids)}")
    print(f"Instances completed: {len(completed_ids)}")
    print(f"Instances incomplete: {len(incomplete_ids)}")
    print(f"Instances resolved: {len(resolved_ids)}")
    print(f"Instances unresolved: {len(unresolved_ids)}")
    print(f"Instances with likely infrastructure failures: {len(infra_failure_ids)}")
    print(f"Instances with ambiguous failures: {len(ambiguous_failure_ids)}")
    print(f"Instances with empty patches: {len(empty_patch_ids)}")
    print(f"Instances with errors: {len(error_ids)}")
    if client:
        print(f"Unstopped containers: {len(unstopped_containers)}")
        print(f"Unremoved images: {len(unremoved_images)}")

    # write report to file
    report = {
        "total_instances": len(full_dataset),
        "submitted_instances": len(predictions),
        "completed_instances": len(completed_ids),
        "resolved_instances": len(resolved_ids),
        "unresolved_instances": len(unresolved_ids),
        "infra_failure_instances": len(infra_failure_ids),
        "ambiguous_failure_instances": len(ambiguous_failure_ids),
        "empty_patch_instances": len(empty_patch_ids),
        "error_instances": len(error_ids),
        "completed_ids": list(sorted(completed_ids)),
        "incomplete_ids": list(sorted(incomplete_ids)),
        "empty_patch_ids": list(sorted(empty_patch_ids)),
        "submitted_ids": list(sorted(predictions.keys())),
        "resolved_ids": list(sorted(resolved_ids)),
        "unresolved_ids": list(sorted(unresolved_ids)),
        "infra_failure_ids": list(sorted(infra_failure_ids)),
        "ambiguous_failure_ids": list(sorted(ambiguous_failure_ids)),
        "failure_reasons": dict(sorted(infra_failure_reasons.items())),
        "error_ids": list(sorted(error_ids)),
        "schema_version": 2,
    }
    if client:
        report.update(
            {
                "unstopped_instances": len(unstopped_containers),
                "unstopped_containers": list(sorted(unstopped_containers)),
                "unremoved_images": list(sorted(unremoved_images)),
            }
        )
    # The report belongs with the run's other artifacts, and always goes there.
    report_dir = RUN_EVALUATION_LOG_DIR / run_id
    report_dir.mkdir(parents=True, exist_ok=True)
    report_file = report_dir / "results.json"
    with open(report_file, "w") as f:
        print(json.dumps(report, indent=4), file=f)
    print(f"Report written to {report_file}")
    return report_file

run_evaluation

GIT_APPLY_CMDS module-attribute 
GIT_APPLY_CMDS = ['git apply --verbose', 'git apply --verbose --3way', 'git apply --verbose --reject', 'patch --batch --forward --fuzz=5 -p1 -i']
DOCKER_CLIENT_TIMEOUT module-attribute 
DOCKER_CLIENT_TIMEOUT = int(os.environ.get('SWEBENCH_DOCKER_TIMEOUT', '1800'))
DOCKER_CLIENT_POOL_SIZE module-attribute 
DOCKER_CLIENT_POOL_SIZE = int(os.environ.get('SWEBENCH_DOCKER_POOL_SIZE', '128'))
parser module-attribute 
parser = ArgumentParser(description='Run evaluation harness for the given dataset and predictions.', formatter_class=ArgumentDefaultsHelpFormatter)
args module-attribute 
args = parser.parse_args()
create_container 
create_container(test_spec: TestSpec, client: DockerClient, run_id: str, logger: Logger)

Creates a container from an instance image for running evaluation.

Parameters:

Name Type Description Default
test_spec TestSpec

Test spec with evaluation details

 required
client DockerClient

Docker client for creating the container

 required
run_id str

Run ID identifying process, used for the container name

 required
logger Logger

Logger to use for logging the creation process

 required
Source code in swebench/harness/run_evaluation.py 
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
def create_container(
    test_spec: TestSpec,
    client: docker.DockerClient,
    run_id: str,
    logger: logging.Logger,
):
    """
    Creates a container from an instance image for running evaluation.

    Args:
        test_spec (TestSpec): Test spec with evaluation details
        client (docker.DockerClient): Docker client for creating the container
        run_id (str): Run ID identifying process, used for the container name
        logger (logging.Logger): Logger to use for logging the creation process
    """
    container = None
    try:
        # Check if the image exists
        try:
            client.images.get(test_spec.image)
        except docker.errors.ImageNotFound:
            try:
                logger.info("Image not found locally, attempting to pull...")
                client.images.pull(test_spec.image)
            except docker.errors.ImageNotFound:
                raise EvaluationError(
                    test_spec.instance_id,
                    f"Image {test_spec.image} not found for {test_spec.instance_id}",
                    logger,
                )

        logger.info(f"Creating container for {test_spec.instance_id}...")

        container_name = f"sweb.eval.{test_spec.instance_id.lower()}.{run_id}"
        # Remove any existing container with this name (handles ghost containers)
        try:
            old = client.containers.get(container_name)
            old.remove(force=True)
            logger.info(f"Removed existing container {container_name}")
        except docker.errors.NotFound:
            pass
        except Exception:
            pass
        try:
            container = client.containers.create(
                image=test_spec.image,
                name=container_name,
                user=CONTAINER_USER,
                detach=True,
                command="tail -f /dev/null",
                # Docker's default seccomp profile only permits CLONE_NEWUSER with
                # CAP_SYS_ADMIN, which browser sandboxes need (e.g. openlayers karma)
                cap_add=["SYS_ADMIN"],
            )
        except docker.errors.APIError as e:
            if "409" in str(e) or "Conflict" in str(e):
                # Ghost container — use a unique suffix
                import time

                container_name = f"{container_name}.{int(time.time())}"
                logger.info(f"Retrying with unique name: {container_name}")
                container = client.containers.create(
                    image=test_spec.image,
                    name=container_name,
                    user=CONTAINER_USER,
                    detach=True,
                    command="tail -f /dev/null",
                    # Docker's default seccomp profile only permits CLONE_NEWUSER with
                    # CAP_SYS_ADMIN, which browser sandboxes need (e.g. openlayers karma)
                    cap_add=["SYS_ADMIN"],
                )
            else:
                raise
        logger.info(f"Container for {test_spec.instance_id} created: {container.id}")
        return container
    except Exception as e:
        logger.error(f"Error creating container for {test_spec.instance_id}: {e}")
        logger.info(traceback.format_exc())
        cleanup_container(client, container, logger)
        raise EvaluationError(test_spec.instance_id, str(e), logger) from e
run_instance 
run_instance(test_spec: TestSpec, pred: dict, client: DockerClient, run_id: str, timeout: int | None = None, rewrite_reports: bool = False, skip_patch: bool = False, task_repo: str | None = None)

Run a single instance with the given prediction.

Parameters:

Name Type Description Default
test_spec TestSpec

TestSpec instance with pre-built image

 required
pred dict

Prediction w/ model_name_or_path, model_patch, instance_id

 required
client DockerClient

Docker client

 required
run_id str

Run ID

 required
timeout int

Timeout for running tests

 None
rewrite_reports bool

True if eval run is just to reformat existing report

 False
skip_patch bool

True to skip applying model patch (negative test mode)

 False
Source code in swebench/harness/run_evaluation.py 
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
def run_instance(
    test_spec: TestSpec,
    pred: dict,
    client: docker.DockerClient,
    run_id: str,
    timeout: int | None = None,
    rewrite_reports: bool = False,
    skip_patch: bool = False,
    task_repo: str | None = None,
):
    """
    Run a single instance with the given prediction.

    Args:
        test_spec (TestSpec): TestSpec instance with pre-built image
        pred (dict): Prediction w/ model_name_or_path, model_patch, instance_id
        client (docker.DockerClient): Docker client
        run_id (str): Run ID
        timeout (int): Timeout for running tests
        rewrite_reports (bool): True if eval run is just to reformat existing report
        skip_patch (bool): True to skip applying model patch (negative test mode)
    """
    # Set up logging directory
    instance_id = test_spec.instance_id
    model_name_or_path = pred.get("model_name_or_path", "None").replace("/", "__")
    log_dir = RUN_EVALUATION_LOG_DIR / run_id / model_name_or_path / instance_id

    # Set up report file
    report_path = log_dir / LOG_REPORT
    if rewrite_reports:
        test_output_path = log_dir / LOG_TEST_OUTPUT
        if not test_output_path.exists():
            raise ValueError(f"Test output file {test_output_path} does not exist")
        report = get_eval_report(
            test_spec=test_spec,
            prediction=pred,
            test_log_path=test_output_path,
            include_tests_status=True,
        )
        # Write report to report.json
        with open(report_path, "w") as f:
            f.write(json.dumps(report, indent=4))
        return instance_id, report
    if report_path.exists():
        return instance_id, json.loads(report_path.read_text())

    # Set up logger
    log_dir.mkdir(parents=True, exist_ok=True)
    log_file = log_dir / LOG_INSTANCE
    logger = setup_logger(instance_id, log_file)

    # Run the instance
    container = None
    try:
        # Create container from image
        container = create_container(test_spec, client, run_id, logger)
        container.start()
        logger.info(f"Container for {instance_id} started: {container.id}")

        if not skip_patch:
            # Copy model prediction as patch file to container
            patch_file = Path(log_dir / "patch.diff")
            patch_file.write_text(pred["model_patch"] or "")
            logger.info(
                f"Intermediate patch for {instance_id} written to {patch_file}, now applying to container..."
            )
            copy_to_container(
                container, patch_file, PurePosixPath(CONTAINER_PATCH_FILE)
            )

            # Attempt to apply patch to container
            applied_patch = False
            for attempt, git_apply_cmd in enumerate(GIT_APPLY_CMDS):
                if attempt:
                    # a failed attempt (notably --reject) leaves partial state behind,
                    # which makes every later command fail; restart from a pristine tree
                    container.exec_run(
                        ["/bin/bash", "-c", "git checkout -- . ; git clean -fd"],
                        workdir=CONTAINER_WORKDIR,
                        user=CONTAINER_USER,
                    )
                val = container.exec_run(
                    f"{git_apply_cmd} {CONTAINER_PATCH_FILE}",
                    workdir=CONTAINER_WORKDIR,
                    user=CONTAINER_USER,
                )
                if val.exit_code == 0:
                    logger.info(f"{APPLY_PATCH_PASS}:\n{val.output.decode('utf-8')}")
                    applied_patch = True
                    break
                else:
                    logger.info(f"Failed to apply patch to container: {git_apply_cmd}")
            if not applied_patch:
                # the chain can leave the patch fully applied while each command still exited non-zero
                reverse_check = container.exec_run(
                    f"git apply --check --reverse {CONTAINER_PATCH_FILE}",
                    workdir=CONTAINER_WORKDIR,
                    user=CONTAINER_USER,
                )
                if reverse_check.exit_code == 0:
                    logger.info(f"{APPLY_PATCH_PASS}: verified already applied")
                    applied_patch = True
            if not applied_patch:
                logger.info(f"{APPLY_PATCH_FAIL}:\n{val.output.decode('utf-8')}")
                raise EvaluationError(
                    instance_id,
                    f"{APPLY_PATCH_FAIL}:\n{val.output.decode('utf-8')}",
                    logger,
                )
        else:
            logger.info(f"Skipping model patch for {instance_id} (--no-patch mode)")

        # Get git diff before running eval script
        git_diff_output_before = (
            container.exec_run(
                "git -c core.fileMode=false diff", workdir=CONTAINER_WORKDIR
            )
            .output.decode("utf-8")
            .strip()
        )
        logger.info(f"Git diff before:\n{git_diff_output_before}")

        # Materialize multimodal binary assets (e.g. expected.png rendering
        # baselines). A text test_patch cannot carry them, so the dataset ships
        # them as urls in image_assets; without this the tests run against a
        # missing baseline and error out.
        restore_cmds = _stage_image_assets(
            container, test_spec, log_dir, logger, task_repo
        )

        eval_file = Path(log_dir / "eval.sh")
        eval_file.write_text(_inject_asset_restore(test_spec.eval_script, restore_cmds))
        logger.info(
            f"Eval script for {instance_id} written to {eval_file}; copying to container..."
        )
        copy_to_container(container, eval_file, PurePosixPath("/eval.sh"))

        # Run eval script, write output to logs
        test_output, timed_out, total_runtime = exec_run_with_timeout(
            container, "/bin/bash /eval.sh", timeout
        )
        test_output_path = log_dir / LOG_TEST_OUTPUT
        logger.info(f"Test runtime: {total_runtime:_.2f} seconds")
        with open(test_output_path, "w") as f:
            f.write(test_output)
            logger.info(f"Test output for {instance_id} written to {test_output_path}")
            if timed_out:
                f.write(f"\n\nTimeout error: {timeout} seconds exceeded.")
                raise EvaluationError(
                    instance_id,
                    f"Test timed out after {timeout} seconds.",
                    logger,
                )

        # Get git diff after running eval script (ignore permission changes)
        git_diff_output_after = (
            container.exec_run(
                "git -c core.fileMode=false diff", workdir=CONTAINER_WORKDIR
            )
            .output.decode("utf-8")
            .strip()
        )

        # Check if git diff changed after running eval script
        logger.info(f"Git diff after:\n{git_diff_output_after}")
        if git_diff_output_after != git_diff_output_before:
            logger.info("Git diff changed after running eval script")

        # Get report from test output
        logger.info(f"Grading answer for {instance_id}...")
        report = get_eval_report(
            test_spec=test_spec,
            prediction=pred,
            test_log_path=test_output_path,
            include_tests_status=True,
        )
        logger.info(
            f"report: {report}\n"
            f"Result for {instance_id}: resolved: {report[instance_id]['resolved']}"
        )

        # Write report to report.json
        with open(report_path, "w") as f:
            f.write(json.dumps(report, indent=4))
        return instance_id, report
    except EvaluationError as e:
        error_msg = traceback.format_exc()
        logger.info(error_msg)
        print(e)
    except Exception as e:
        error_msg = (
            f"Error in evaluating model for {instance_id}: {e}\n"
            f"{traceback.format_exc()}\n"
            f"Check ({logger.log_file}) for more information."
        )
        logger.error(error_msg)
    finally:
        # Remove instance container + image, close logger
        cleanup_container(client, container, logger)
        close_logger(logger)
    return
run_instances 
run_instances(predictions: dict, instances: list, max_workers: int, run_id: str, timeout: int, rewrite_reports: bool = False, skip_patch: bool = False, task_repo: str | None = None)

Run all instances for the given predictions in parallel. Expects instances to have pre-built images.

Parameters:

Name Type Description Default
predictions dict

Predictions dict generated by the model

 required
instances list

List of instances with 'image' field

 required
max_workers int

Maximum number of workers

 required
run_id str

Run ID

 required
timeout int

Timeout for running tests

 required
rewrite_reports bool

True if eval run is just to reformat existing report

 False
Source code in swebench/harness/run_evaluation.py 
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
def run_instances(
    predictions: dict,
    instances: list,
    max_workers: int,
    run_id: str,
    timeout: int,
    rewrite_reports: bool = False,
    skip_patch: bool = False,
    task_repo: str | None = None,
):
    """
    Run all instances for the given predictions in parallel.
    Expects instances to have pre-built images.

    Args:
        predictions (dict): Predictions dict generated by the model
        instances (list): List of instances with 'image' field
        max_workers (int): Maximum number of workers
        run_id (str): Run ID
        timeout (int): Timeout for running tests
        rewrite_reports (bool): True if eval run is just to reformat existing report
    """
    client = _docker_client()
    test_specs = [make_test_spec(instance) for instance in instances]

    # run instances in parallel
    payloads = []
    for test_spec in test_specs:
        payloads.append(
            (
                test_spec,
                predictions[test_spec.instance_id],
                client,
                run_id,
                timeout,
                rewrite_reports,
                skip_patch,
                task_repo,
            )
        )

    # run instances in parallel
    print(f"Running {len(instances)} instances...")
    run_threadpool(run_instance, payloads, max_workers)
    print("All instances run.")
write_run_metadata 
write_run_metadata(run_id: str, dataset_name: str, split: str, task_repo: str | None) -> Path

Record what this run graded against.

Re-grading needs the expected tests and the log parser, which live in the dataset, not in the run's logs. Without this a later swebench report has to be told the dataset again, and gets it wrong silently if told the wrong one.

Source code in swebench/harness/run_evaluation.py 
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
def write_run_metadata(
    run_id: str, dataset_name: str, split: str, task_repo: str | None
) -> Path:
    """Record what this run graded against.

    Re-grading needs the expected tests and the log parser, which live in the
    dataset, not in the run's logs. Without this a later `swebench report` has to
    be told the dataset again, and gets it wrong silently if told the wrong one.
    """
    path = RUN_EVALUATION_LOG_DIR / run_id / LOG_RUN_METADATA
    path.parent.mkdir(parents=True, exist_ok=True)
    # a re-grade passes no task repo, and overwriting the recorded one with null
    # loses the only record of which tests the run was graded against
    if task_repo is None and path.is_file():
        task_repo = json.loads(path.read_text()).get("task_repo")
    path.write_text(
        json.dumps(
            {
                "dataset": dataset_name,
                "split": split,
                "task_repo": task_repo,
                "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
            },
            indent=2,
        )
        + "\n"
    )
    return path
read_run_metadata 
read_run_metadata(run_id: str) -> dict | None

What a previous run graded against, if it recorded it.

Source code in swebench/harness/run_evaluation.py 
509
510
511
512
def read_run_metadata(run_id: str) -> dict | None:
    """What a previous run graded against, if it recorded it."""
    path = RUN_EVALUATION_LOG_DIR / run_id / LOG_RUN_METADATA
    return json.loads(path.read_text()) if path.is_file() else None
load_instances 
load_instances(dataset_name: str, split: str, instance_ids: list | None, task_repo: str | None) -> list

Instances come from the task repo when one is given, else from the dataset.

A dataset is already one split; a task repo holds every split at once, so the split has to be applied here or a run picks up whatever else is in the tree -- the multimodal repo would evaluate its dev and deprecated tasks alongside test.

Named ids are honoured from any split, matching select_tasks: an instance being repaired can be run by name while it sits in an unpublished split.

Source code in swebench/harness/run_evaluation.py 
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
def load_instances(
    dataset_name: str, split: str, instance_ids: list | None, task_repo: str | None
) -> list:
    """Instances come from the task repo when one is given, else from the dataset.

    A dataset is already one split; a task repo holds every split at once, so the
    split has to be applied here or a run picks up whatever else is in the tree --
    the multimodal repo would evaluate its dev and deprecated tasks alongside test.

    Named ids are honoured from any split, matching `select_tasks`: an instance
    being repaired can be run by name while it sits in an unpublished split.
    """
    if not task_repo:
        return load_swebench_dataset(dataset_name, split, instance_ids)
    tasks = load_task_repo(task_repo, instance_ids)
    if instance_ids:
        return tasks
    wanted = [task for task in tasks if task.get("split") == split]
    if not wanted:
        available = sorted({task["split"] for task in tasks if task.get("split")})
        raise ValueError(
            f"{task_repo} has no tasks in split {split!r}. "
            f"It has: {' '.join(available) or 'none'}"
        )
    return wanted
get_dataset_from_preds 
get_dataset_from_preds(dataset_name: str, split: str, instance_ids: list, predictions: dict, run_id: str, rewrite_reports: bool, exclude_completed: bool = True, task_repo: str | None = None)

Return only instances that have predictions and are in the dataset. If instance_ids is provided, only return instances with those IDs. If exclude_completed is True, only return instances that have not been run yet.

Source code in swebench/harness/run_evaluation.py 
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
def get_dataset_from_preds(
    dataset_name: str,
    split: str,
    instance_ids: list,
    predictions: dict,
    run_id: str,
    rewrite_reports: bool,
    exclude_completed: bool = True,
    task_repo: str | None = None,
):
    """
    Return only instances that have predictions and are in the dataset.
    If instance_ids is provided, only return instances with those IDs.
    If exclude_completed is True, only return instances that have not been run yet.
    """
    # load dataset
    dataset = load_instances(dataset_name, split, None, task_repo)
    dataset_ids = {i["instance_id"] for i in dataset}

    if instance_ids:
        # check that all instance IDs have predictions
        missing_preds = set(instance_ids) - set(predictions.keys())
        if missing_preds:
            print(
                f"Warning: Missing predictions for {len(missing_preds)} instance IDs."
            )

    # check that all prediction IDs are in the dataset
    prediction_ids = set(predictions.keys())
    if prediction_ids - dataset_ids:
        raise ValueError(
            (
                "Some prediction IDs not found in dataset!"
                f"\nMissing IDs:\n{' '.join(prediction_ids - dataset_ids)}"
            )
        )
    if instance_ids:
        dataset = [i for i in dataset if i["instance_id"] in instance_ids]

    if rewrite_reports:
        # we only return instances that have existing test outputs
        test_output_ids = set()
        for instance in dataset:
            if instance["instance_id"] not in predictions:
                continue
            prediction = predictions[instance["instance_id"]]
            test_output_file = (
                RUN_EVALUATION_LOG_DIR
                / run_id
                / prediction["model_name_or_path"].replace("/", "__")
                / prediction["instance_id"]
                / "test_output.txt"
            )
            if test_output_file.exists():
                test_output_ids.add(instance["instance_id"])
        dataset = [
            i
            for i in dataset
            if i["instance_id"] in prediction_ids
            and i["instance_id"] in test_output_ids
        ]
        return dataset

    # check which instance IDs have already been run
    completed_ids = set()
    for instance in dataset:
        if instance["instance_id"] not in prediction_ids:
            # skip instances without predictions
            continue
        prediction = predictions[instance["instance_id"]]
        report_file = (
            RUN_EVALUATION_LOG_DIR
            / run_id
            / prediction["model_name_or_path"].replace("/", "__")
            / prediction["instance_id"]
            / LOG_REPORT
        )
        if report_file.exists():
            completed_ids.add(instance["instance_id"])

    if completed_ids and exclude_completed:
        # filter dataset to only instances that have not been run
        print(f"{len(completed_ids)} instances already run, skipping...")
        dataset = [i for i in dataset if i["instance_id"] not in completed_ids]

    empty_patch_ids = {
        k
        for k, v in predictions.items()
        if v["model_patch"] == "" or v["model_patch"] is None
    }

    # filter dataset to only instances with predictions
    dataset = [
        i
        for i in dataset
        if i["instance_id"] in prediction_ids
        and i["instance_id"] not in empty_patch_ids
    ]
    return dataset
main 
main(dataset_name: str, split: str, instance_ids: list, predictions_path: str, max_workers: int, open_file_limit: int, run_id: str, timeout: int, rewrite_reports: bool, modal: bool, task_repo: str | None = None)

Run evaluation harness for the given dataset and predictions.

Source code in swebench/harness/run_evaluation.py 
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
def main(
    dataset_name: str,
    split: str,
    instance_ids: list,
    predictions_path: str,
    max_workers: int,
    open_file_limit: int,
    run_id: str,
    timeout: int,
    rewrite_reports: bool,
    modal: bool,
    task_repo: str | None = None,
):
    """
    Run evaluation harness for the given dataset and predictions.
    """
    if dataset_name == "SWE-bench/SWE-bench_Multimodal" and split == "test":
        print(
            "ℹ️ Running local evaluation for the test split of SWE-bench Multimodal. "
            "You may also use sb-cli (https://github.com/swe-bench/sb-cli/) to submit predictions to the hosted evaluation."
        )

    # Modal builds its own images remotely, so a task repo's Dockerfiles would be
    # ignored while its tests were used -- the run would report on a tree it never
    # built. Refused here, before any work, rather than silently proving the wrong thing.
    if modal and task_repo:
        raise ValueError(
            "--modal cannot build from a task repo: it builds images remotely, so the "
            "repo's Dockerfiles would be ignored while its tests were used. Drop "
            "--task-repo to run on Modal, or drop --modal to build the repo."
        )

    # set open file limit
    assert len(run_id) > 0, "Run ID must be provided"
    # load predictions as map of instance_id to prediction
    predictions = get_predictions_from_file(
        predictions_path, dataset_name, split, task_repo, instance_ids
    )
    predictions = {pred["instance_id"]: pred for pred in predictions}
    write_run_metadata(run_id, dataset_name, split, task_repo)

    # get dataset from predictions
    dataset = get_dataset_from_preds(
        dataset_name,
        split,
        instance_ids,
        predictions,
        run_id,
        rewrite_reports,
        task_repo=task_repo,
    )
    full_dataset = load_instances(dataset_name, split, instance_ids, task_repo)

    if modal:
        # run instances on Modal
        if not dataset:
            print("No instances to run.")
        else:
            validate_modal_credentials()
            run_instances_modal(predictions, dataset, full_dataset, run_id, timeout)
        return

    # run instances locally
    if platform.system() == "Linux":
        resource.setrlimit(resource.RLIMIT_NOFILE, (open_file_limit, open_file_limit))
    client = _docker_client()

    if not dataset:
        print("No instances to run.")
        return make_run_report(predictions, full_dataset, run_id, client)
    else:
        # a re-grade reads existing logs and starts no container, so building the
        # images it names would cost hours and change nothing
        if task_repo and not rewrite_reports:
            _build_before_eval(
                dataset, dataset_name, split, task_repo, max_workers, client
            )
        # run instances (images assumed to be pre-built)
        run_instances(
            predictions,
            dataset,
            max_workers,
            run_id,
            timeout,
            rewrite_reports=rewrite_reports,
            task_repo=task_repo,
        )

    # make final report
    return make_run_report(predictions, full_dataset, run_id, client)

utils

EvaluationError 
EvaluationError(instance_id, message, logger)

Bases: Exception

Source code in swebench/harness/utils.py 
23
24
25
26
27
28
def __init__(self, instance_id, message, logger):
    super().__init__(message)
    self.super_str = super().__str__()
    self.instance_id = instance_id
    self.log_path = logger.log_file
    self.logger = logger
super_str instance-attribute 
super_str = super().__str__()
instance_id instance-attribute 
instance_id = instance_id
log_path instance-attribute 
log_path = logger.log_file
logger instance-attribute 
logger = logger
__str__ 
__str__()
Source code in swebench/harness/utils.py 
30
31
32
33
34
def __str__(self):
    return (
        f"Error in evaluation for {self.instance_id}: {self.super_str}\n"
        f"Check ({self.log_path}) for more information."
    )
get_predictions_from_file 
get_predictions_from_file(predictions_path: str, dataset_name: str, split: str, task_repo: str | None = None, instance_ids: list | None = None)
Source code in swebench/harness/utils.py 
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
def get_predictions_from_file(
    predictions_path: str,
    dataset_name: str,
    split: str,
    task_repo: str | None = None,
    instance_ids: list | None = None,
):
    if predictions_path == "gold":
        print("Using gold predictions - ignoring predictions_path")
        # the gold patch has to come from wherever the rest of the instance came from,
        # or a run against a task repo silently grades the dataset's patch instead and
        # never notices the two disagreeing
        if task_repo:
            from swebench.task.repo import load_task_repo

            dataset = load_task_repo(task_repo, instance_ids)
            if not instance_ids:
                dataset = [d for d in dataset if d.get("split") == split]
        else:
            dataset = load_swebench_dataset(dataset_name, split, instance_ids)
        return [
            {
                "instance_id": datum["instance_id"],
                "model_patch": datum["patch"],
                "model_name_or_path": "gold",
            }
            for datum in dataset
        ]
    if predictions_path.endswith(".json"):
        with open(predictions_path, "r") as f:
            predictions = json.load(f)
            if isinstance(predictions, dict):
                predictions = list(
                    predictions.values()
                )  # compatible with SWE-agent predictions
            if not isinstance(predictions, list):
                raise ValueError(
                    "Predictions must be a list[prediction] or a dictionary[instance_id: prediction]"
                )
    elif predictions_path.endswith(".jsonl"):
        with open(predictions_path, "r") as f:
            predictions = [json.loads(line) for line in f]
    else:
        raise ValueError("Predictions path must be .json or .jsonl")

    # Validate that each prediction has an instance_id
    for pred in predictions:
        if not isinstance(pred, dict):
            raise ValueError(f"Each prediction must be a dictionary, got {type(pred)}")
        if "instance_id" not in pred:
            raise ValueError(f"Each prediction must contain '{'instance_id'}'")

    return predictions
run_threadpool 
run_threadpool(func, payloads, max_workers)
Source code in swebench/harness/utils.py 
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
def run_threadpool(func, payloads, max_workers):
    if max_workers <= 0:
        return run_sequential(func, payloads)
    succeeded, failed = [], []
    with tqdm(total=len(payloads), smoothing=0) as pbar:
        with ThreadPoolExecutor(max_workers=max_workers) as executor:
            # Create a future for running each instance
            futures = {executor.submit(func, *payload): payload for payload in payloads}
            # Wait for each future to complete
            for future in as_completed(futures):
                try:
                    # Check if instance ran successfully
                    future.result()
                    succeeded.append(futures[future])
                except Exception as e:
                    print(f"{type(e)}: {e}")
                    traceback.print_exc()
                    failed.append(futures[future])
                # Update progress bar
                pbar.update(1)
                pbar.set_description(
                    f"{len(succeeded)} ran successfully, {len(failed)} failed"
                )
    return succeeded, failed
run_sequential 
run_sequential(func, args_list)

Run a function with a list of arguments sequentially

Source code in swebench/harness/utils.py 
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
def run_sequential(func, args_list):
    """
    Run a function with a list of arguments sequentially
    """
    succeeded, failed = [], []
    pbar = tqdm(total=len(args_list), smoothing=0)
    for args in args_list:
        try:
            func(*args)
            succeeded.append(args)
        except Exception:
            traceback.print_exc()
            failed.append(args)
        pbar.update(1)
        pbar.set_description(f"{len(succeeded)} ran successfully, {len(failed)} failed")
    pbar.close()
    return succeeded, failed
load_swebench_dataset 
load_swebench_dataset(name='SWE-bench/SWE-bench', split='test', instance_ids=None) -> list[SWEbenchInstance]

Load SWE-bench dataset from Hugging Face Datasets or local .json/.jsonl file

Source code in swebench/harness/utils.py 
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
def load_swebench_dataset(
    name="SWE-bench/SWE-bench", split="test", instance_ids=None
) -> list[SWEbenchInstance]:
    """
    Load SWE-bench dataset from Hugging Face Datasets or local .json/.jsonl file
    """
    # check that all instance IDs are in the dataset
    if instance_ids:
        instance_ids = set(instance_ids)
    # Load from local file
    if name.endswith(".json"):
        dataset = json.loads(Path(name).read_text())
    elif name.endswith(".jsonl"):
        dataset = [json.loads(line) for line in Path(name).read_text().splitlines()]
    elif name.endswith(".parquet"):
        dataset = cast(Dataset, load_dataset("parquet", data_files=name, split="train"))
    else:
        # Load from Hugging Face Datasets
        if name.lower() in {"swe-bench", "swebench", "swe_bench"}:
            name = "SWE-bench/SWE-bench"
        elif name.lower() in {
            "swe-bench-lite",
            "swebench-lite",
            "swe_bench_lite",
            "swe-bench_lite",
            "lite",
        }:
            name = "SWE-bench/SWE-bench_Lite"
        parquet_path = Path(name) / f"{split}.parquet"
        if parquet_path.exists():
            dataset = cast(
                Dataset,
                load_dataset("parquet", data_files=str(parquet_path), split="train"),
            )
        elif (Path(name) / split / "dataset_info.json").exists():
            dataset = cast(Dataset, load_from_disk(Path(name) / split))
        else:
            dataset = cast(Dataset, load_dataset(name, split=split))
    dataset_ids = {instance["instance_id"] for instance in dataset}
    if instance_ids:
        if instance_ids - dataset_ids:
            raise ValueError(
                (
                    "Some instance IDs not found in dataset!"
                    f"\nMissing IDs:\n{' '.join(instance_ids - dataset_ids)}"
                )
            )
        dataset = [
            instance for instance in dataset if instance["instance_id"] in instance_ids
        ]
    return [cast(SWEbenchInstance, instance) for instance in dataset]
str2bool 
str2bool(v)

Minor helper function to convert string to boolean

Source code in swebench/harness/utils.py 
190
191
192
193
194
195
196
197
198
199
200
201
def str2bool(v):
    """
    Minor helper function to convert string to boolean
    """
    if isinstance(v, bool):
        return v
    if v.lower() in ("yes", "true", "t", "y", "1"):
        return True
    elif v.lower() in ("no", "false", "f", "n", "0"):
        return False
    else:
        raise ArgumentTypeError("Boolean value expected.")
optional_str 
optional_str(value: str) -> str | None

Convert special string values to None, otherwise return the string as-is.

Source code in swebench/harness/utils.py 
204
205
206
207
208
209
210
def optional_str(value: str) -> str | None:
    """
    Convert special string values to None, otherwise return the string as-is.
    """
    if value.lower() in ("none", "null", ""):
        return None
    return value
parse_eval_script 
parse_eval_script(eval_script: str) -> list[str]

Parse an eval.sh script into a command list (strip shebang + set flags).

Source code in swebench/harness/utils.py 
213
214
215
216
217
218
219
def parse_eval_script(eval_script: str) -> list[str]:
    """Parse an eval.sh script into a command list (strip shebang + set flags)."""
    return [
        line
        for line in eval_script.strip().split("\n")
        if line not in ("#!/bin/bash", "set -uxo pipefail")
    ]
record_test_exit_code 
record_test_exit_code(eval_script_list: list[str]) -> list[str]

Make the eval script record the test command's own exit status.

Eval scripts end with a git checkout that resets the test files, and run under set -uxo pipefail without -e, so the script's exit status is the reset's, not the tests'. Capturing $? immediately after the test command and echoing it after the end marker keeps the value out of the parsed region.

Source code in swebench/harness/utils.py 
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
def record_test_exit_code(eval_script_list: list[str]) -> list[str]:
    """Make the eval script record the test command's own exit status.

    Eval scripts end with a `git checkout` that resets the test files, and run
    under `set -uxo pipefail` without `-e`, so the script's exit status is the
    reset's, not the tests'. Capturing `$?` immediately after the test command
    and echoing it after the end marker keeps the value out of the parsed region.
    """
    for i, line in enumerate(eval_script_list):
        if END_TEST_OUTPUT in line:
            return [
                *eval_script_list[:i],
                f"{TEST_EXIT_CODE_VAR}=$?",
                line,
                f'echo "{TEST_EXIT_CODE}: ${TEST_EXIT_CODE_VAR}"',
                *eval_script_list[i + 1 :],
            ]
    # No end marker (unrecognized script shape): leave it untouched, and grading
    # falls back to the pre-existing behavior of trusting the log alone.
    return eval_script_list
make_test_spec 
make_test_spec(instance: dict) -> TestSpec

Build a TestSpec from a dataset instance.

The instance dict must contain: instance_id, image, repo, version, FAIL_TO_PASS, PASS_TO_PASS, log_parser, eval_type, eval_script.

Source code in swebench/harness/utils.py 
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
def make_test_spec(instance: dict) -> TestSpec:
    """
    Build a TestSpec from a dataset instance.

    The instance dict must contain: instance_id, image, repo, version,
    FAIL_TO_PASS, PASS_TO_PASS, log_parser, eval_type, eval_script.
    """
    f2p = instance["FAIL_TO_PASS"]
    p2p = instance["PASS_TO_PASS"]
    return TestSpec(
        instance_id=instance["instance_id"],
        image=instance["image"],
        eval_script_list=record_test_exit_code(
            parse_eval_script(instance["eval_script"])
        ),
        repo=instance["repo"],
        version=instance["version"],
        FAIL_TO_PASS=json.loads(f2p) if isinstance(f2p, str) else f2p,
        PASS_TO_PASS=json.loads(p2p) if isinstance(p2p, str) else p2p,
        log_parser=instance["log_parser"],
        eval_type=instance["eval_type"],
        image_assets=_parse_image_assets(instance.get("image_assets")),
    )