{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "ccdbbca3db42047df3d1e5760d6c962ce7c00855f840dd1c2fc9b1be8a05acc7",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.1515416319889482
    },
    {
      "branch": "",
      "command": "find /workspace -type f -exec sha256sum {} + 2>/dev/null | grep -E \"^(f08a1d9590ebc2710b86a22c3ea294272464acf53003d96911b6209f984d87aa)  \" | cut -c67- | xargs -I% rm -fv %",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.12166949298989493
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.15220650300034322
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.13648577399726491
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.11712242000794504
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.10741994199634064
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6627607779955724
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14534266899863724
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13470555400999729
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13046646901057102
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1337099969969131
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 12662,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.51891316100955
    },
    {
      "branch": "894a253da19f",
      "command": "docker cp 058be60589bf40bce63835739e4be2ad3ebea628d1ea0ddd9f95ba434ebc037a:/workspace/eval/results.xml /tmp/tmpkr7lqcad.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 8787,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03956747500342317
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10974872100632638
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1122110709984554
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11376520099292975
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.09833118699316401
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 19410,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 6.642701054995996
    },
    {
      "branch": "8210eaf4f82e",
      "command": "docker cp 5679df3642bfc1e82a1ee9b3a71705b98f5f0ed89ee2d8877486437ca04306a4:/workspace/eval/results.xml /tmp/tmpajjs_o6t.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 14793,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.04089499900874216
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11686691899376456
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.10015961399767548
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11044467199826613
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1276344829966547
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 100548,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 12.130438713007607
    },
    {
      "branch": "7877cb75132e",
      "command": "docker cp 2d6b0e183bee07a8305a51c3fc1e6eaa6838d4a5dd98009277aee94082923232:/workspace/eval/results.xml /tmp/tmpqdy9h7w3.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 91353,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.027643107998301275
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1396404780098237
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1274775839992799
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13623189298959915
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13396289099182468
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 13503,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.7666046249942156
    },
    {
      "branch": "a875f2e4ed22",
      "command": "docker cp fafb3faf12f3bc49e43d5fee9bb2b4eb787d5e941c3cfc2b85faf49e400e93d7:/workspace/eval/results.xml /tmp/tmp5gs0tiad.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 11232,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.038787615005276166
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1008212930028094
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12946053700579796
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1261836790072266
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12470311899960507
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4814,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.3655246129928855
    },
    {
      "branch": "7dad0a57ec5e",
      "command": "docker cp d8137184906231ccac2b34872229f086d575017a6efe43f72975d1c9a9497710:/workspace/eval/results.xml /tmp/tmp_8n6thrt.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2850,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.0403657650022069
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13826460498967208
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13475474900042173
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12886306499422062
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11280248999537434
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 8795,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.703406615008134
    },
    {
      "branch": "fb0689dd2ac7",
      "command": "docker cp 98db92dfbb9da5612208689433e7891489d5b080a94ec17271baccea03c52efb:/workspace/eval/results.xml /tmp/tmptu6nukc3.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 6829,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03648686400265433
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1275356799887959
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12495386399677955
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12521214000298642
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1249688740062993
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 14318,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.3446479470003396
    },
    {
      "branch": "6f9c47ca1eae",
      "command": "docker cp ac285b953c75efe8af2df714e964807e6aa4c7ee4edaf5f11672d71365d149bb:/workspace/eval/results.xml /tmp/tmpptt1zlju.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 8460,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.038414479000493884
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.09966151599655859
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.096138481007074
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11246223299531266
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12551316600001883
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5102,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.7400397440069355
    },
    {
      "branch": "85c953be94a7",
      "command": "docker cp efdf59b4177899a7550b37f4685fbc24c63e20a0ccaf901c356fa8018f49afa0:/workspace/eval/results.xml /tmp/tmpsc0kgrb7.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2600,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.026580890000332147
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12261185300303623
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12466765599674545
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1095226290053688
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11036523900111206
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 14416,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.6008275600033812
    },
    {
      "branch": "ec95564f8a2f",
      "command": "docker cp 5edda4368e9f8f06ef54102d060fb6e41bd708c6c299b2b234b4b395b01d4e5f:/workspace/eval/results.xml /tmp/tmp86eaqkwc.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 9602,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.025245507000363432
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14011733799998183
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13354798199725337
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12301638699136674
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13128818100085482
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4908,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.531622636000975
    },
    {
      "branch": "6df336510340",
      "command": "docker cp 66bf2107b7c37b00367b999ef01116f62fe993b3f052330e9ae8998cc7d489e7:/workspace/eval/results.xml /tmp/tmplephawbb.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2772,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03850112699728925
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14082884800154716
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.10753432499768678
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12136037800519262
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13059670900111087
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 12250,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.775962796004023
    },
    {
      "branch": "556b82945a18",
      "command": "docker cp 64641a9c1dc535718373e51f0cffb9dfde0be31e8bbf905d4836e030887f95b7:/workspace/eval/results.xml /tmp/tmp8mackokp.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 8174,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03891507200023625
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 70,
    "passed": 670
  },
  "test_branch_errors": {},
  "test_branches": [
    "894a253da19f",
    "8210eaf4f82e",
    "7877cb75132e",
    "a875f2e4ed22",
    "7dad0a57ec5e",
    "fb0689dd2ac7",
    "6f9c47ca1eae",
    "85c953be94a7",
    "ec95564f8a2f",
    "6df336510340",
    "556b82945a18"
  ],
  "test_records": 740,
  "test_results": [
    {
      "branch": "8210eaf4f82e",
      "extra": {
        "message_chars": 385,
        "message_redacted": true,
        "time": 0.199
      },
      "name": "tests.test_edge_cases.test_whitespace_in_replacement",
      "status": "failure"
    },
    {
      "branch": "8210eaf4f82e",
      "extra": {
        "message_chars": 381,
        "message_redacted": true,
        "time": 0.171
      },
      "name": "tests.test_edge_cases.test_replace_with_period",
      "status": "failure"
    },
    {
      "branch": "8210eaf4f82e",
      "extra": {
        "message_chars": 380,
        "message_redacted": true,
        "time": 0.174
      },
      "name": "tests.test_edge_cases.test_empty_replacement",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 250,
        "message_redacted": true,
        "time": 0.206
      },
      "name": "tests.test_errors.test_invalid_regex_unclosed_bracket",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 543,
        "message_redacted": true,
        "time": 0.172
      },
      "name": "tests.test_advanced_regex.test_backreference_in_pattern",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 389,
        "message_redacted": true,
        "time": 0.198
      },
      "name": "tests.test_advanced_regex.test_replace_limit_unlimited",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 202,
        "message_redacted": true,
        "time": 0.178
      },
      "name": "tests.test_advanced_regex.test_transform_ascii_slash_substitution",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 219,
        "message_redacted": true,
        "time": 0.142
      },
      "name": "tests.test_dumpfile_gaps.test_empty_source_path_handling",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 236,
        "message_redacted": true,
        "time": 0.149
      },
      "name": "tests.test_advanced_regex.test_zero_width_assertions_not_supported",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 389,
        "message_redacted": true,
        "time": 0.143
      },
      "name": "tests.test_advanced_regex.test_replace_limit_1",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 593,
        "message_redacted": true,
        "time": 0.132
      },
      "name": "tests.test_advanced_regex.test_invalid_regex_unclosed_bracket",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 1316,
        "message_redacted": true,
        "time": 0.17
      },
      "name": "tests.test_integration.test_color_output_modes",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 432,
        "message_redacted": true,
        "time": 0.176
      },
      "name": "tests.test_output_gaps.test_string_diff_middle_replacement",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 426,
        "message_redacted": true,
        "time": 0.13
      },
      "name": "tests.test_output_gaps.test_color_always_multiple_replacements",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 310,
        "message_redacted": true,
        "time": 0.125
      },
      "name": "tests.test_output_gaps.test_color_error_output",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 440,
        "message_redacted": true,
        "time": 0.117
      },
      "name": "tests.test_output_gaps.test_color_always_simple_diff",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 450,
        "message_redacted": true,
        "time": 0.155
      },
      "name": "tests.test_output_gaps.test_color_nested_deep_path",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 457,
        "message_redacted": true,
        "time": 0.143
      },
      "name": "tests.test_output_gaps.test_string_diff_deletion_shorter_replacement",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 218,
        "message_redacted": true,
        "time": 0.156
      },
      "name": "tests.test_dumpfile_gaps.test_empty_target_path_handling",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 263,
        "message_redacted": true,
        "time": 0.125
      },
      "name": "tests.test_errors.test_invalid_regex_unclosed_group",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 610,
        "message_redacted": true,
        "time": 0.126
      },
      "name": "tests.test_advanced_regex.test_invalid_regex_unclosed_group",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 389,
        "message_redacted": true,
        "time": 0.145
      },
      "name": "tests.test_advanced_regex.test_replace_limit_2",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 455,
        "message_redacted": true,
        "time": 0.128
      },
      "name": "tests.test_output_gaps.test_color_multiple_files_batch_operation",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 429,
        "message_redacted": true,
        "time": 0.141
      },
      "name": "tests.test_output_gaps.test_color_auto_defaults_no_color_without_tty",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 437,
        "message_redacted": true,
        "time": 0.125
      },
      "name": "tests.test_output_gaps.test_color_hyphen_in_filename",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 336,
        "message_redacted": true,
        "time": 0.19
      },
      "name": "tests.test_recursive.test_to_ascii_recursive",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 286,
        "message_redacted": true,
        "time": 0.153
      },
      "name": "tests.test_output_gaps.test_no_color_error_output",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 437,
        "message_redacted": true,
        "time": 0.144
      },
      "name": "tests.test_output_gaps.test_string_diff_prefix_replacement",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 439,
        "message_redacted": true,
        "time": 0.177
      },
      "name": "tests.test_output_gaps.test_color_always_with_parent_directory",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 479,
        "message_redacted": true,
        "time": 0.14
      },
      "name": "tests.test_output_gaps.test_string_diff_complete_replacement",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 448,
        "message_redacted": true,
        "time": 0.165
      },
      "name": "tests.test_output_gaps.test_string_diff_insertion_longer_replacement",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 463,
        "message_redacted": true,
        "time": 0.169
      },
      "name": "tests.test_output_gaps.test_color_backup_info_message",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 429,
        "message_redacted": true,
        "time": 0.132
      },
      "name": "tests.test_regex_basic.test_color_never_flag",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 287,
        "message_redacted": true,
        "time": 0.146
      },
      "name": "tests.test_regex_basic.test_invalid_regex_error",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 173,
        "message_redacted": true,
        "time": 0.132
      },
      "name": "tests.test_dumpfile_gaps.test_dump_file_creation_failure",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 426,
        "message_redacted": true,
        "time": 0.15
      },
      "name": "tests.test_output_gaps.test_string_diff_no_common_characters",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 266,
        "message_redacted": true,
        "time": 0.151
      },
      "name": "tests.test_errors.test_invalid_regex_empty_repetition",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 425,
        "message_redacted": true,
        "time": 0.148
      },
      "name": "tests.test_output_gaps.test_string_diff_repeated_characters",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 443,
        "message_redacted": true,
        "time": 0.118
      },
      "name": "tests.test_output_gaps.test_color_spaces_in_filename",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 432,
        "message_redacted": true,
        "time": 0.118
      },
      "name": "tests.test_output_gaps.test_color_extension_only_change",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 480,
        "message_redacted": true,
        "time": 0.112
      },
      "name": "tests.test_regex_basic.test_custom_dump_prefix",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 437,
        "message_redacted": true,
        "time": 0.155
      },
      "name": "tests.test_output_gaps.test_string_diff_suffix_replacement",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 435,
        "message_redacted": true,
        "time": 0.169
      },
      "name": "tests.test_output_gaps.test_color_never_with_parent_directory",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 249,
        "message_redacted": true,
        "time": 0.148
      },
      "name": "tests.test_regex_basic.test_dot_metacharacter_in_regex",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 393,
        "message_redacted": true,
        "time": 0.164
      },
      "name": "tests.test_regex_basic.test_replace_limit_two",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 301,
        "message_redacted": true,
        "time": 0.127
      },
      "name": "tests.test_solver_gaps.test_rename_chain_five_deep",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.153
      },
      "name": "tests.test_solver_gaps.test_rename_chain_three_deep",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 198,
        "message_redacted": true,
        "time": 0.181
      },
      "name": "tests.test_solver_gaps.test_rename_chain_execution_order",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 189,
        "message_redacted": true,
        "time": 0.121
      },
      "name": "tests.test_solver_gaps.test_backup_with_rename_chain",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 191,
        "message_redacted": true,
        "time": 0.126
      },
      "name": "tests.test_solver_gaps.test_mixed_chain_and_independent_renames",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 393,
        "message_redacted": true,
        "time": 0.133
      },
      "name": "tests.test_regex_basic.test_replace_limit_zero_replaces_all",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 422,
        "message_redacted": true,
        "time": 0.144
      },
      "name": "tests.test_output_gaps.test_string_diff_single_character_replacement",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 432,
        "message_redacted": true,
        "time": 0.128
      },
      "name": "tests.test_output_gaps.test_string_diff_case_change_only",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 2220,
        "message_redacted": true,
        "time": 0.12
      },
      "name": "tests.test_to_ascii.test_emoji_conversion",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 3410,
        "message_redacted": true,
        "time": 0.118
      },
      "name": "tests.test_to_ascii.test_multiple_languages",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 241,
        "message_redacted": true,
        "time": 0.161
      },
      "name": "tests.test_solver_gaps.test_multiple_files_same_target_error",
      "status": "failure"
    },
    {
      "branch": "7877cb75132e",
      "extra": {
        "message_chars": 229,
        "message_redacted": true,
        "time": 0.143
      },
      "name": "tests.test_solver_gaps.test_rename_chain_seven_deep_execution",
      "status": "failure"
    },
    {
      "branch": "a875f2e4ed22",
      "extra": {
        "message_chars": 922,
        "message_redacted": true,
        "time": 0.151
      },
      "name": "eval.tests.test_argparse_validation.test_enum_validation_color_rejects_invalid_values[argv1]",
      "status": "failure"
    },
    {
      "branch": "a875f2e4ed22",
      "extra": {
        "message_chars": 812,
        "message_redacted": true,
        "time": 0.198
      },
      "name": "eval.tests.test_argparse_validation.test_integer_validation_rejects_non_integer[argv2---replace-limit <LIMIT>]",
      "status": "failure"
    },
    {
      "branch": "a875f2e4ed22",
      "extra": {
        "message_chars": 358,
        "message_redacted": true,
        "time": 0.164
      },
      "name": "eval.tests.test_argparse_validation.test_duplicate_flag_rejected_color",
      "status": "failure"
    },
    {
      "branch": "a875f2e4ed22",
      "extra": {
        "message_chars": 305,
        "message_redacted": true,
        "time": 0.31
      },
      "name": "eval.tests.test_argparse_validation.test_flag_with_value_supports_space_and_equals_forms",
      "status": "failure"
    },
    {
      "branch": "fb0689dd2ac7",
      "extra": {
        "message_chars": 411,
        "message_redacted": true,
        "time": 0.196
      },
      "name": "eval.tests.test_regex_basic.test_regex_color_never_has_no_ansi_codes",
      "status": "failure"
    },
    {
      "branch": "fb0689dd2ac7",
      "extra": {
        "message_chars": 415,
        "message_redacted": true,
        "time": 0.209
      },
      "name": "eval.tests.test_regex_basic.test_regex_color_always_emits_ansi_codes",
      "status": "failure"
    },
    {
      "branch": "fb0689dd2ac7",
      "extra": {
        "message_chars": 396,
        "message_redacted": true,
        "time": 0.201
      },
      "name": "eval.tests.test_to_ascii.test_to_ascii_force_renames",
      "status": "failure"
    },
    {
      "branch": "fb0689dd2ac7",
      "extra": {
        "message_chars": 377,
        "message_redacted": true,
        "time": 0.192
      },
      "name": "eval.tests.test_to_ascii.test_to_ascii_dry_run_does_not_rename",
      "status": "failure"
    },
    {
      "branch": "85c953be94a7",
      "extra": {
        "message_chars": 146,
        "message_redacted": true,
        "time": 0.194
      },
      "name": "eval.tests.test_subcommand_dispatch.test_subcommand_specific_flag_rejected_globally_before_subcommand",
      "status": "failure"
    },
    {
      "branch": "556b82945a18",
      "extra": {
        "message_chars": 138,
        "message_redacted": true,
        "time": 0.168
      },
      "name": "eval.tests.test_externalized.test_ext_renamer_to_ascii_subcommand_renames_files",
      "status": "failure"
    },
    {
      "branch": "556b82945a18",
      "extra": {
        "message_chars": 35,
        "message_redacted": true,
        "time": 0.179
      },
      "name": "eval.tests.test_externalized.test_ext_fileutils_is_same_file_case_sensitive_default",
      "status": "failure"
    },
    {
      "branch": "556b82945a18",
      "extra": {
        "message_chars": 126,
        "message_redacted": true,
        "time": 0.175
      },
      "name": "eval.tests.test_externalized.test_ext_solver_existing_targets_with_symlinks",
      "status": "failure"
    },
    {
      "branch": "556b82945a18",
      "extra": {
        "message_chars": 126,
        "message_redacted": true,
        "time": 0.166
      },
      "name": "eval.tests.test_externalized.test_ext_solver_solve_rename_order_chain_uses_safe_order",
      "status": "failure"
    }
  ],
  "test_results_omitted": 0,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}