{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "2b2de3562ed6fdf0fc11a74f530bb8c743fa49a036fe0800b4b5517cab37d851",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.1492175059975125
    },
    {
      "branch": "",
      "command": "find /workspace -type f -exec sha256sum {} + 2>/dev/null | grep -E \"^(49b9879a61cc55c6755476159101cb1bb49abb9a4e5cc9ba1988d7c64b610007)  \" | cut -c67- | xargs -I% rm -fv %",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.13621070299996063
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.1317714899778366
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.0877203090058174
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.1281954460137058
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.08422755298670381
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6454127049946692
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10360603299341165
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12046606600051746
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12599461400532164
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1268697440100368
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 20300,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 19.672819527011598
    },
    {
      "branch": "3001c1482b82",
      "command": "docker cp cb4732f0f41d8a0cea1f5a2e125a5d3088e05d238c46c6eb22ef25b53cbd8bee:/workspace/eval/results.xml /tmp/tmpf68p69tq.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 14891,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03633491101209074
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11544886100455187
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.10053539299406111
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.10436394298449159
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1068072929920163
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 12244,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 13.923132650990738
    },
    {
      "branch": "644934f68973",
      "command": "docker cp b46aab862fad73a2c4c934981b9559cf2f8bf0917fb35188d9b77cf8a553d5cc:/workspace/eval/results.xml /tmp/tmpydjcdos2.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 13898,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.037386167008662596
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10488237600657158
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11879591800970957
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13501002799603157
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.10110253602033481
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 53543,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 100.03071929499856
    },
    {
      "branch": "1fb751a1a8a1",
      "command": "docker cp e88c4594b7fcdb6d39af34959e4d5ea4a8fdc831aa77ea5b1b68177b4e53b6b2:/workspace/eval/results.xml /tmp/tmplhkxgz1r.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 39445,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03852704301243648
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.15072429602150805
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.08928726802696474
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12474461100646295
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12425289701786824
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 32055,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 8.672746345982887
    },
    {
      "branch": "c865682e93ee",
      "command": "docker cp ed19f10f22799f9cee112a89b87e7671467e01f7ffa9a89a393b9238f9adadcf:/workspace/eval/results.xml /tmp/tmphatnlp7d.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 21266,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.034434244997100905
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13937596199684776
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1278582910017576
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11555269901873544
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12171797998598777
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 13192,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 14.792984488012735
    },
    {
      "branch": "220f03dff485",
      "command": "docker cp d644867c960d7e18de192a7e9c028e632db56bba42a596700b7c9901ff0dcada:/workspace/eval/results.xml /tmp/tmpb6wk65sa.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 8828,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.023950471018906683
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10488775599515066
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11663027500617318
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13449104499886744
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12608401299803518
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 6631,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.4648254810017534
    },
    {
      "branch": "c77d1fe5cef8",
      "command": "docker cp df158361b60cd854bac59bde6e64c98bff7cf1009b94207db414ac96ffc16d9a:/workspace/eval/results.xml /tmp/tmpw7ac7x19.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3892,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03662450800766237
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11503118599648587
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.08783687700633891
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.09898122298181988
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12304357197717763
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1547,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.540180768002756
    },
    {
      "branch": "15241b94c9bf",
      "command": "docker cp 7ea1652c34a959b8d2ef8f943307d76f9b56463df397b1cbcc26e54a85ed024e:/workspace/eval/results.xml /tmp/tmp3bv32wev.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 485,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03331455998704769
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12323513100272976
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13391660200431943
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12241428002016619
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12660953699378297
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 16730,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 4.468009197007632
    },
    {
      "branch": "f2ad8ae32d40",
      "command": "docker cp 78d52be40fe5398df7cbaaddce56c138d8dfa5619a3e046f9c4345a55550bdbd:/workspace/eval/results.xml /tmp/tmp6f24unlm.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 11266,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.0283223649894353
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1273263869807124
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13275653700111434
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13254506900557317
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13135417600278743
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4680,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.593391681002686
    },
    {
      "branch": "0a5ff17822b7",
      "command": "docker cp ba3d488d3cbbcf4afa1e1bc28cb04ac2d974432fa6eca229445ff7f1f8f975cb:/workspace/eval/results.xml /tmp/tmp3ax2b8jp.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2571,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03761513400240801
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.09531311099999584
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.08380928900442086
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.09943223700975068
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.10235135001130402
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 14453,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 19.413545524992514
    },
    {
      "branch": "40a4c29f1824",
      "command": "docker cp bd419c454ee02bec090781a085c66bab65af007da60d9f31782803acede3a8f0:/workspace/eval/results.xml /tmp/tmphsgv_q8f.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 8787,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03712987099424936
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14050704697729088
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12938719999510795
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12256632899516262
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1285288320214022
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3332,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.7182320880237967
    },
    {
      "branch": "0df6b4d747de",
      "command": "docker cp 8544c28c58c850eee69336c926e097c98dec0167bf8d76171d33ad59de20c1e6:/workspace/eval/results.xml /tmp/tmpxvp8ya8c.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1610,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.030598174984334037
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12876102901645936
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1173517259885557
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11597497397451662
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13267263898160309
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 7615,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.3390990719781257
    },
    {
      "branch": "ec5f2f35f276",
      "command": "docker cp 27926e6e445dd44ee57afbb4506b132dbb162004f17e16b069badc14740fb29b:/workspace/eval/results.xml /tmp/tmpt26aovmi.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4419,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03262186801293865
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12788146000821143
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13179315999150276
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.10512980399653316
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11084063901216723
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 70,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 0.13725184899521992
    },
    {
      "branch": "a6516d59468d",
      "command": "docker cp b88f4377875499ff59e2511d0939ecc8bda1081db17c28c0da97898fec7af7bd:/workspace/eval/results.xml /tmp/tmpahn1ybqs.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 139,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.037466527021024376
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14307469499181025
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11527343199122697
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11984752301941626
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1236978640081361
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 80,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 0.11608244199305773
    },
    {
      "branch": "f83680a23703",
      "command": "docker cp cb99ea44beaae004b00def8425552ca88dbe676d4ef9dbae6be7df2da4555ddc:/workspace/eval/results.xml /tmp/tmp4ogenwg3.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 149,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.04168104301788844
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14824993797810748
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12449996298528276
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1294020509812981
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12958380801137537
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3369,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.721373396983836
    },
    {
      "branch": "63078964cd3f",
      "command": "docker cp d9d40a93121691a01aeea2c918a3c68dd87675c631ddc2fe4f9169ae2cdf75af:/workspace/eval/results.xml /tmp/tmpdkzs2jhh.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1657,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.0253389339777641
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1331563710118644
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.14074812500621192
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.10068399101146497
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13946652101003565
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 8147,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.845602086017607
    },
    {
      "branch": "045fa65d8a17",
      "command": "docker cp 1570a89d9902043b2a1dba6f97c3b1e4c27dacae43c84ea0f624fba5412d5602:/workspace/eval/results.xml /tmp/tmpm1gg5g96.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5190,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.022286513005383313
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 40,
    "passed": 751,
    "skipped": 1
  },
  "test_branch_errors": {},
  "test_branches": [
    "3001c1482b82",
    "644934f68973",
    "1fb751a1a8a1",
    "c865682e93ee",
    "220f03dff485",
    "c77d1fe5cef8",
    "15241b94c9bf",
    "f2ad8ae32d40",
    "0a5ff17822b7",
    "40a4c29f1824",
    "0df6b4d747de",
    "ec5f2f35f276",
    "a6516d59468d",
    "f83680a23703",
    "63078964cd3f",
    "045fa65d8a17"
  ],
  "test_records": 792,
  "test_results": [
    {
      "branch": "3001c1482b82",
      "extra": {
        "message_chars": 299,
        "message_redacted": true,
        "time": 0.193
      },
      "name": "eval.tests.test_error_handling.test_starting_password_with_dictionary_error",
      "status": "failure"
    },
    {
      "branch": "3001c1482b82",
      "extra": {
        "message_chars": 293,
        "message_redacted": true,
        "time": 0.125
      },
      "name": "eval.tests.test_error_handling.test_starting_password_too_long_error",
      "status": "failure"
    },
    {
      "branch": "3001c1482b82",
      "extra": {
        "message_chars": 194,
        "message_redacted": true,
        "time": 15.025
      },
      "name": "eval.tests.test_error_handling.test_starting_password_too_short_error",
      "status": "failure"
    },
    {
      "branch": "3001c1482b82",
      "extra": {
        "message_chars": 163,
        "message_redacted": true,
        "time": 15.025
      },
      "name": "eval.tests.test_error_handling.test_starting_password_out_of_charset_error",
      "status": "failure"
    },
    {
      "branch": "644934f68973",
      "extra": {
        "message_chars": 288,
        "message_redacted": true,
        "time": 0.121
      },
      "name": "eval.tests.test_starting_password.test_starting_password_outside_charset",
      "status": "failure"
    },
    {
      "branch": "644934f68973",
      "extra": {
        "message_chars": 266,
        "message_redacted": true,
        "time": 0.156
      },
      "name": "eval.tests.test_starting_password.test_starting_password_with_dictionary",
      "status": "failure"
    },
    {
      "branch": "644934f68973",
      "extra": {
        "message_chars": 292,
        "message_redacted": true,
        "time": 0.157
      },
      "name": "eval.tests.test_starting_password.test_starting_password_too_short",
      "status": "failure"
    },
    {
      "branch": "644934f68973",
      "extra": {
        "message_chars": 290,
        "message_redacted": true,
        "time": 0.137
      },
      "name": "eval.tests.test_starting_password.test_starting_password_too_long",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 336,
        "message_redacted": true,
        "time": 0.124
      },
      "name": "tests.test_bruteforce.test_starting_password_too_long",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 418,
        "message_redacted": true,
        "time": 0.145
      },
      "name": "tests.test_cli.test_inputfile_directory",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 305,
        "message_redacted": true,
        "time": 0.139
      },
      "name": "tests.test_cli.test_startingpassword_with_dictionary",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 380,
        "message_redacted": true,
        "time": 0.128
      },
      "name": "tests.test_cli.test_startingpassword_at_minlen_boundary",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 325,
        "message_redacted": true,
        "time": 0.162
      },
      "name": "tests.test_harvest.test_startingPassword_with_dictionary",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 321,
        "message_redacted": true,
        "time": 0.14
      },
      "name": "tests.test_harvest.test_startingPassword_length_exceeds_maxPasswordLen",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 308,
        "message_redacted": true,
        "time": 0.156
      },
      "name": "tests.test_performance.test_resume_from_invalid_starting_password_wrong_length",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 307,
        "message_redacted": true,
        "time": 0.126
      },
      "name": "tests.test_performance.test_resume_from_invalid_starting_password_wrong_charset",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 321,
        "message_redacted": true,
        "time": 0.162
      },
      "name": "tests.test_performance.test_starting_password_with_dictionary_error",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 386,
        "message_redacted": true,
        "time": 0.097
      },
      "name": "tests.test_performance.test_empty_charset_file_error",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 316,
        "message_redacted": true,
        "time": 0.145
      },
      "name": "tests.test_zip_files.test_starting_password_with_dictionary",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 340,
        "message_redacted": true,
        "time": 1.101
      },
      "name": "tests.test_bruteforce.test_starting_password_with_dictionary_error",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 380,
        "message_redacted": true,
        "time": 0.123
      },
      "name": "tests.test_cli.test_startingpassword_at_maxlen_boundary",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 317,
        "message_redacted": true,
        "time": 0.124
      },
      "name": "tests.test_harvest.test_startingPassword_out_of_charset",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 335,
        "message_redacted": true,
        "time": 0.144
      },
      "name": "tests.test_bruteforce.test_starting_password_out_of_charset",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 334,
        "message_redacted": true,
        "time": 0.112
      },
      "name": "tests.test_bruteforce.test_starting_password_too_short",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 157,
        "message_redacted": true,
        "time": 90.043
      },
      "name": "tests.test_cli.test_startingpassword_out_of_charset",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 221,
        "message_redacted": true,
        "time": 90.051
      },
      "name": "tests.test_harvest.test_startingPassword_length_below_minPasswordLen",
      "status": "failure"
    },
    {
      "branch": "1fb751a1a8a1",
      "extra": {
        "message_chars": 320,
        "message_redacted": true,
        "time": 3.329
      },
      "name": "tests.test_cli.test_startingpassword_exceeds_maxlen",
      "status": "failure"
    },
    {
      "branch": "c865682e93ee",
      "extra": {
        "message_chars": 559,
        "message_redacted": true,
        "time": 0.161
      },
      "name": "eval.tests.test_argument_parsing.TestPasswordDictionaryArgument.test_dictionary_is_directory",
      "status": "failure"
    },
    {
      "branch": "c865682e93ee",
      "extra": {
        "message_chars": 670,
        "message_redacted": true,
        "time": 0.138
      },
      "name": "eval.tests.test_argument_parsing.TestInputFileArgument.test_input_file_is_directory",
      "status": "failure"
    },
    {
      "branch": "c865682e93ee",
      "extra": {
        "message_chars": 568,
        "message_redacted": true,
        "time": 0.133
      },
      "name": "eval.tests.test_argument_parsing.TestCharsetFileArgument.test_charset_file_is_directory",
      "status": "failure"
    },
    {
      "branch": "c865682e93ee",
      "extra": {
        "message_chars": 13,
        "message_redacted": true,
        "time": 0.129
      },
      "name": "eval.tests.test_argument_parsing.TestStartingPasswordArgument.test_starting_password_with_dictionary",
      "status": "failure"
    },
    {
      "branch": "c865682e93ee",
      "extra": {
        "message_chars": 13,
        "message_redacted": true,
        "time": 3.012
      },
      "name": "eval.tests.test_argument_parsing.TestStartingPasswordArgument.test_starting_password_out_of_charset",
      "status": "failure"
    },
    {
      "branch": "c865682e93ee",
      "extra": {
        "message_chars": 13,
        "message_redacted": true,
        "time": 0.168
      },
      "name": "eval.tests.test_argument_parsing.TestStartingPasswordArgument.test_starting_password_too_long",
      "status": "failure"
    },
    {
      "branch": "c865682e93ee",
      "extra": {
        "message_chars": 13,
        "message_redacted": true,
        "time": 3.012
      },
      "name": "eval.tests.test_argument_parsing.TestStartingPasswordArgument.test_starting_password_too_short",
      "status": "failure"
    },
    {
      "branch": "220f03dff485",
      "extra": {
        "message_chars": 226,
        "message_redacted": true,
        "time": 0.167
      },
      "name": "eval.tests.test_argparse_validation.test_inputfile_accepts_space_and_equals_forms[args3]",
      "status": "failure"
    },
    {
      "branch": "220f03dff485",
      "extra": {
        "message_chars": 136,
        "message_redacted": true,
        "time": 0.174
      },
      "name": "eval.tests.test_argparse_validation.test_integer_option_rejects_overflow",
      "status": "failure"
    },
    {
      "branch": "220f03dff485",
      "extra": {
        "message_chars": 171,
        "message_redacted": true,
        "time": 9.019
      },
      "name": "eval.tests.test_argparse_validation.test_value_starting_with_dash_is_rejected_for_startingpassword",
      "status": "failure"
    },
    {
      "branch": "220f03dff485",
      "extra": {
        "message_chars": 179,
        "message_redacted": true,
        "time": 9.019
      },
      "name": "eval.tests.test_argparse_validation.test_duplicate_charset_flag_is_rejected_by_parser",
      "status": "failure"
    },
    {
      "branch": "f2ad8ae32d40",
      "extra": {
        "message_chars": 596,
        "message_redacted": true,
        "time": 0.151
      },
      "name": "tests.test_help_baseline.TestHelpBaseline.test_invalid_flag_format",
      "status": "failure"
    },
    {
      "branch": "40a4c29f1824",
      "extra": {
        "message_chars": 180,
        "message_redacted": true,
        "time": 14.998
      },
      "name": "eval.tests.test_zip_password_finder.TestPasswordLength.test_min_password_length_constraint",
      "status": "failure"
    },
    {
      "branch": "045fa65d8a17",
      "extra": {
        "message_chars": 56,
        "message_redacted": true,
        "time": 0.001
      },
      "name": "eval.tests.test_zip_password_finder.TestDictionaryAttack.test_dictionary_password_not_found",
      "status": "skipped"
    }
  ],
  "test_results_omitted": 0,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}