{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "2a5f351cf4a2938e3029c2119c6d150eed5fa20e09f9d02c2154e01e92d11451",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.1612152790039545
    },
    {
      "branch": "",
      "command": "find /workspace -type f -exec sha256sum {} + 2>/dev/null | grep -E \"^(1528514451c30447e93bc97bcda65118325cf80ceb92687716135f3ce12937aa)  \" | cut -c67- | xargs -I% rm -fv %",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.11994956100534182
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.1352618929959135
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.13634772499790415
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.12115340201125946
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.12993262999225408
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6131443010090152
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1466750760009745
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12718549399869516
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12732656400476117
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12380947900237516
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 7603,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 4.181995651000761
    },
    {
      "branch": "cd9161de8535",
      "command": "docker cp c9f22d9f710d651508f9b4b2effac069a685f025df2e38f080ea08ee6c7ee817:/workspace/eval/results.xml /tmp/tmpcsbmu3rp.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5016,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03719818800163921
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.15044539200607687
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1185304980026558
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12741924400324933
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13681793899741024
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 30556,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 8.814975839995896
    },
    {
      "branch": "f84528aebeb6",
      "command": "docker cp e728288885c65ba99869fc404cffbdacfa66cf76990a7c90ec3984480995ba72:/workspace/eval/results.xml /tmp/tmptpopvox_.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 35467,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03827334199741017
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11649286399187986
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12852479099819902
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1281711220071884
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13684392200957518
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 59448,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 12.904256252004416
    },
    {
      "branch": "b31afd05cac9",
      "command": "docker cp 8e038f7fc2c68cbd210cf6a86ca3d151c59dbd8f8025cb9bebbf7a6706e01ee5:/workspace/eval/results.xml /tmp/tmp9t2guece.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 54689,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03636729599384125
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13321747101144865
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11955234699416906
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.10614626000460703
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12659310799790546
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5306,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.630896016999031
    },
    {
      "branch": "9d5cf5af1740",
      "command": "docker cp c5a6dabb1b326f7aabf60f5c90df33845e8a3dcc4c22fae2c4de4180357e7cb6:/workspace/eval/results.xml /tmp/tmp1dnp45xf.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3001,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03713417200197
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12213534100737888
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.09781607899640221
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12578870699508116
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12444503400183748
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4669,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.610947876994032
    },
    {
      "branch": "4a4d83d3a514",
      "command": "docker cp 74a2ec41ac85fa5a5f159a5697800c9e6206a344c04b0888e8f82d05bffde39b:/workspace/eval/results.xml /tmp/tmpxqwb85fv.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3983,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03613117900385987
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13077739199798089
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11996085799182765
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12927282400778495
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12935043299512472
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 6676,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.631853963001049
    },
    {
      "branch": "e412e82747ad",
      "command": "docker cp 9949c66e49ab7987973557ee829a7b7f57a55c45a21f47a75df92f5ae5f2b94a:/workspace/eval/results.xml /tmp/tmpn9v0u1__.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4438,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.037332900988985784
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11090001300908625
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13277843200194184
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.09655590199690778
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.09261478899861686
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2267,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.4598146349890158
    },
    {
      "branch": "e3f0044ad001",
      "command": "docker cp c063c076aeff43705ede9fec83e161f20a5cd8791844f3b09879ca55f8ef2767:/workspace/eval/results.xml /tmp/tmpl1urbn9j.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 961,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.030963705998146906
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12359005599864759
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.08587299700593576
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.09394381100719329
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12044214698835276
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1338,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.43665305399918
    },
    {
      "branch": "ba079af3d7e0",
      "command": "docker cp 43bd8bb488b3bf202cd42bc68e6c9993083b46ab11fc4b70e2f39cf4b49fcb85:/workspace/eval/results.xml /tmp/tmpjptjesfg.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 352,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.026489437994314358
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.102773830993101
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12023536099877674
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13803087698761374
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.127685540006496
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 10573,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.620343781003612
    },
    {
      "branch": "a52aedcd2b5f",
      "command": "docker cp 84cbe697e70cf694264e35e39801c551b64ff09aac73b9e7f90678c2c3cdb856:/workspace/eval/results.xml /tmp/tmpiydz_inv.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 6818,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03660046799632255
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.09737173900066409
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12619337199430447
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13332446200365666
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12989292599377222
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2808,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.5691366129904054
    },
    {
      "branch": "d1569f338ac2",
      "command": "docker cp ada82d8996e3cdac56402c3a2a1703b7e48e7a6ba581a677b74d9a81a983f2f6:/workspace/eval/results.xml /tmp/tmpdn6_h__5.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1660,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03575157400337048
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13196164299733937
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1008496269932948
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12929127100505866
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13294284699077252
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4138,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 10.582887538999785
    },
    {
      "branch": "bf2b5020846e",
      "command": "docker cp 456936581b8d8c40dd7eb39dda435ba5c74ac40aa04f76cec190d6f67f12f995:/workspace/eval/results.xml /tmp/tmpk7h4kvm8.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3913,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03981785698852036
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14795470699027646
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13034640799742192
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1221093140047742
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11312832600378897
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5738,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.554985717011732
    },
    {
      "branch": "c671a2683d18",
      "command": "docker cp 81076d21d09bb1390276a74f300b114e592211aa1fd95ce946eacec7f18eeec2:/workspace/eval/results.xml /tmp/tmpc_y_6eel.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5000,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.034516653991886415
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 32,
    "passed": 365,
    "skipped": 3
  },
  "test_branch_errors": {},
  "test_branches": [
    "cd9161de8535",
    "f84528aebeb6",
    "b31afd05cac9",
    "9d5cf5af1740",
    "4a4d83d3a514",
    "e412e82747ad",
    "e3f0044ad001",
    "ba079af3d7e0",
    "a52aedcd2b5f",
    "d1569f338ac2",
    "bf2b5020846e",
    "c671a2683d18"
  ],
  "test_records": 400,
  "test_results": [
    {
      "branch": "f84528aebeb6",
      "extra": {
        "message_chars": 9700,
        "message_redacted": true,
        "time": 0.119
      },
      "name": "eval.tests.test_edge_cases.test_config_with_empty_repo_path",
      "status": "failure"
    },
    {
      "branch": "f84528aebeb6",
      "extra": {
        "message_chars": 464,
        "message_redacted": true,
        "time": 0.13
      },
      "name": "eval.tests.test_flag_combinations.test_unknown_flag_with_valid_flag",
      "status": "failure"
    },
    {
      "branch": "f84528aebeb6",
      "extra": {
        "message_chars": 172,
        "message_redacted": true,
        "time": 1.531
      },
      "name": "eval.tests.test_config_creation.test_first_time_setup_with_no_config",
      "status": "failure"
    },
    {
      "branch": "f84528aebeb6",
      "extra": {
        "message_chars": 159,
        "message_redacted": true,
        "time": 0.917
      },
      "name": "eval.tests.test_config_creation.test_config_dir_creation_pathway",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 169,
        "message_redacted": true,
        "time": 0.106
      },
      "name": "eval.tests.test_config_format.test_config_with_empty_string_path",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 479,
        "message_redacted": true,
        "time": 0.145
      },
      "name": "eval.tests.test_config_format.test_malformed_json_invalid_syntax",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 300,
        "message_redacted": true,
        "time": 0.141
      },
      "name": "eval.tests.test_config_format.test_empty_config_file",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 479,
        "message_redacted": true,
        "time": 0.165
      },
      "name": "eval.tests.test_config_format.test_config_with_json_comment_fails",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 205,
        "message_redacted": true,
        "time": 0.371
      },
      "name": "eval.tests.test_capture_git.test_editor_fallback_to_vi_when_unset",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 1310,
        "message_redacted": true,
        "time": 0.127
      },
      "name": "eval.tests.test_capture_git.test_git_init_invalid_repo",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 480,
        "message_redacted": true,
        "time": 0.121
      },
      "name": "eval.tests.test_config_format.test_malformed_json_missing_closing_brace",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 178,
        "message_redacted": true,
        "time": 0.128
      },
      "name": "eval.tests.test_edge_cases.test_view_with_nonexistent_repo_path",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 85,
        "message_redacted": true,
        "time": 0.254
      },
      "name": "eval.tests.test_integration.test_git_repository_with_uncommitted_changes",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 267,
        "message_redacted": true,
        "time": 0.096
      },
      "name": "eval.tests.test_edge_cases.test_empty_config_file_parsed_as_default_config",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 372,
        "message_redacted": true,
        "time": 0.09
      },
      "name": "eval.tests.test_io_errors.test_config_read_with_malformed_json",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 1299,
        "message_redacted": true,
        "time": 0.228
      },
      "name": "eval.tests.test_io_errors.test_git_init_error_with_invalid_repo_path",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 465,
        "message_redacted": true,
        "time": 0.126
      },
      "name": "eval.tests.test_messages.test_fts_empty_input_re_prompts",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 487,
        "message_redacted": true,
        "time": 0.117
      },
      "name": "eval.tests.test_setup_config.test_first_time_setup_retries_on_empty_input",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 65,
        "message_redacted": true,
        "time": 0.093
      },
      "name": "eval.tests.test_setup_config.test_multiple_path_validations_in_sequence",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 174,
        "message_redacted": true,
        "time": 0.188
      },
      "name": "eval.tests.test_config_format.test_config_with_whitespace_only_path",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 378,
        "message_redacted": true,
        "time": 0.12
      },
      "name": "eval.tests.test_config_format.test_config_missing_repo_key",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 430,
        "message_redacted": true,
        "time": 0.162
      },
      "name": "eval.tests.test_config_format.test_config_file_only_whitespace",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 1223,
        "message_redacted": true,
        "time": 0.26
      },
      "name": "eval.tests.test_edge_cases.test_corrupted_git_repository_displays_error",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 364,
        "message_redacted": true,
        "time": 0.429
      },
      "name": "eval.tests.test_capture_git.test_git_add_only_readme_not_other_files",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 908,
        "message_redacted": true,
        "time": 0.221
      },
      "name": "eval.tests.test_integration.test_repository_with_no_commits_yet",
      "status": "failure"
    },
    {
      "branch": "b31afd05cac9",
      "extra": {
        "message_chars": 4574,
        "message_redacted": true,
        "time": 0.18
      },
      "name": "eval.tests.test_edge_cases.test_invalid_git_repository_path_not_a_git_repo",
      "status": "failure"
    },
    {
      "branch": "9d5cf5af1740",
      "extra": {
        "message_chars": 235,
        "message_redacted": true,
        "time": 0.182
      },
      "name": "eval.tests.test_args_parsing.test_combined_short_flags_vh_triggers_help_instead_of_parse_error",
      "status": "failure"
    },
    {
      "branch": "4a4d83d3a514",
      "extra": {
        "message_chars": 1125,
        "message_redacted": true,
        "time": 0.157
      },
      "name": "eval.tests.test_config_env.test_empty_config_file_results_in_os_error_logged",
      "status": "failure"
    },
    {
      "branch": "e412e82747ad",
      "extra": {
        "message_chars": 464,
        "message_redacted": true,
        "time": 0.185
      },
      "name": "eval.tests.test_help_usage.test_double_dash_treats_help_as_positional_and_errors",
      "status": "failure"
    },
    {
      "branch": "e412e82747ad",
      "extra": {
        "message_chars": 469,
        "message_redacted": true,
        "time": 0.173
      },
      "name": "eval.tests.test_help_usage.test_invalid_flag_with_help_still_errors_but_shows_usage_hint",
      "status": "failure"
    },
    {
      "branch": "d1569f338ac2",
      "extra": {
        "message_chars": 109,
        "message_redacted": true,
        "time": 0.001
      },
      "name": "eval.tests.test_config_and_clear.test_clear_config_removes_file",
      "status": "skipped"
    },
    {
      "branch": "bf2b5020846e",
      "extra": {
        "message_chars": 724,
        "message_redacted": true,
        "time": 6.159
      },
      "name": "eval.tests.test_tui_interaction.test_tui_starts_and_prompts_for_idea_summary",
      "status": "failure"
    },
    {
      "branch": "bf2b5020846e",
      "extra": {
        "message_chars": 60,
        "message_redacted": true,
        "time": 0.001
      },
      "name": "eval.tests.test_tui_interaction.test_empty_enter_reprompts_for_summary",
      "status": "skipped"
    },
    {
      "branch": "bf2b5020846e",
      "extra": {
        "message_chars": 69,
        "message_redacted": true,
        "time": 0.001
      },
      "name": "eval.tests.test_tui_interaction.test_nonempty_summary_accepts_and_exits_quickly",
      "status": "skipped"
    },
    {
      "branch": "c671a2683d18",
      "extra": {
        "message_chars": 1294,
        "message_redacted": true,
        "time": 0.183
      },
      "name": "eval.tests.test_externalized.test_ext_test_setup_repo_path_asks_until_user_provides_value",
      "status": "failure"
    }
  ],
  "test_results_omitted": 0,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}