{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "a8c15eb9239137dc62eaf9bfa00a11ba9b2e11b4b98c914b7d771d35d81f4162",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.15011421300005168
    },
    {
      "branch": "",
      "command": "find /workspace -type f -exec sha256sum {} + 2>/dev/null | grep -E \"^(776f1c415ab5cb4e73320eb016f8627ae1fc2364c308dace7794989e9fb4a210)  \" | cut -c67- | xargs -I% rm -fv %",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.12991949799470603
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.08896912401542068
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.2675534570007585
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.08742444001836702
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.09947776299668476
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.7465729460236616
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12531922495691106
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12149786099325866
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12428954901406541
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=signal/--timeout-method=thread/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1276943470002152
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 24608,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 7.4062139359884895
    },
    {
      "branch": "79b69dd3fd98",
      "command": "docker cp eaec10d050fbc3330bb459471fc0f96d1c5e5a789d431535e1792d309df63643:/workspace/eval/results.xml /tmp/tmp4etpghw2.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 16612,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03591315500671044
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13772278901888058
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11694219801574945
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13579603598918766
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=signal/--timeout-method=thread/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1379270179895684
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 14929,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 4.136206547962502
    },
    {
      "branch": "1b991a57d4e9",
      "command": "docker cp 9d045bcc3aac61277cc238cb7461a5d638935ccdfc6964847c1d2effa72d1cb0:/workspace/eval/results.xml /tmp/tmpxemoeuzc.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 9463,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.037460769002791494
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13728465099120513
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.08777752803871408
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13030714698834345
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=signal/--timeout-method=thread/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11144963797414675
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 21617,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.320518799999263
    },
    {
      "branch": "bcaceaff5bda",
      "command": "docker cp 410e814747f68e83b2eb19b42c28294bc9a398e51923cb009b22d48aaf3cabea:/workspace/eval/results.xml /tmp/tmpcidjbg3c.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 13680,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.022492895019240677
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1379457479924895
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12747260800097138
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.10089754802174866
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=signal/--timeout-method=thread/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11521226400509477
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 15318,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.323913042957429
    },
    {
      "branch": "f53d318371c8",
      "command": "docker cp 92fd5d98c07865afb4ba684c5661e022c036648514553f856fdacc40033e9022:/workspace/eval/results.xml /tmp/tmpf8pygxa6.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 7357,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.021107189997565
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13860004500020295
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12271611602045596
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.125422554963734
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=signal/--timeout-method=thread/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13042491604574025
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 11800,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.1296258920338005
    },
    {
      "branch": "1d30a9bd67f8",
      "command": "docker cp c542df7a575663bd6bb592e150f0b4ffa441b15eb50c5dcfdf1dd63e2476e84e:/workspace/eval/results.xml /tmp/tmpxhfryr5z.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 7177,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03444833995308727
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12400330102536827
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12252372602233663
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11224486498394981
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=signal/--timeout-method=thread/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.10928092699032277
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 12305,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.046287856006529
    },
    {
      "branch": "2a51b65a0988",
      "command": "docker cp a6df6b9711016cb4ced41281dfbb28303d174329f9548f2a827d5e19e101b0f3:/workspace/eval/results.xml /tmp/tmpdejyy9dm.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 7661,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.021618917991872877
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13186359097016975
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12285033497028053
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12767047702800483
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=signal/--timeout-method=thread/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12614961300278082
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 10770,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.322637406003196
    },
    {
      "branch": "c9a563c41e59",
      "command": "docker cp a88179f8a800b1821c96ed79cf7cd68bffad349925659d1fa6d444aa515f4826:/workspace/eval/results.xml /tmp/tmp0bdw9nj_.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 6576,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03527015697909519
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12765855703037232
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1234959929715842
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12183113902574405
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=signal/--timeout-method=thread/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12003173597622663
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 7307,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 4.088305860001128
    },
    {
      "branch": "47e777b60c5e",
      "command": "docker cp e483ca570a6ce31320b823757d817551fffd760a6acfd8ff8fe7c5f1c7909306:/workspace/eval/results.xml /tmp/tmp33wmavlb.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4572,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03728705900721252
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10497657197993249
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.09172729303827509
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.09944894397631288
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=signal/--timeout-method=thread/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11321940104244277
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2141,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.2603054910432547
    },
    {
      "branch": "f40262460d4c",
      "command": "docker cp 592231749b0dd2932c79d692ec3ed2dbdbfb2b386c06d1c20159150c193f81b3:/workspace/eval/results.xml /tmp/tmpmg6j6dzh.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 869,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03371591295581311
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.08561145601561293
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.10335426800884306
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11928176897345111
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=signal/--timeout-method=thread/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12955955002689734
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 6168,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.0452481799875386
    },
    {
      "branch": "4325cb75f082",
      "command": "docker cp d1a31b8ed2fcc91613ef42468245391d71f669cf77ec9f37490500c6d82e7e00:/workspace/eval/results.xml /tmp/tmpum7s5voz.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3589,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03574066498549655
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1324103589868173
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12054925301345065
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11928485898533836
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=signal/--timeout-method=thread/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.10337579902261496
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 20144,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 15.774537744990084
    },
    {
      "branch": "60fce8bbb0c8",
      "command": "docker cp 7a9d085b6da35ff63a683bbf8a31341ebe6201db0df251f787725e9f5f6c39a4:/workspace/eval/results.xml /tmp/tmpv11_62df.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 12910,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03767440700903535
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13451008102856576
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13202840997837484
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12946849002037197
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=signal/--timeout-method=thread/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13537396799074486
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2159,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.4076397469616495
    },
    {
      "branch": "28d2b640a014",
      "command": "docker cp 11198c53baa120f9be2ff3decf5d6a0e86016ec191d0b6de62b42b44c626d953:/workspace/eval/results.xml /tmp/tmp1uctg12o.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1537,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.029284938995260745
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1265154710272327
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13773256802232936
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11773362499661744
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=signal/--timeout-method=thread/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1136712389998138
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4312,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 34.94955704401946
    },
    {
      "branch": "b75df89939ed",
      "command": "docker cp 514db04e78e14a06aea7ea68f942d33663449e3205ea95eff381173404a1e949:/workspace/eval/results.xml /tmp/tmpmoglaii6.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4566,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03680381097365171
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14119953499175608
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13061370502691716
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12243122700601816
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=signal/--timeout-method=thread/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1347842599498108
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1562,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 6.9366337390383705
    },
    {
      "branch": "aa26a2ba3c36",
      "command": "docker cp bef228dbf8a7a191448df0f6f522a175f44d6c4e99d3220c4896d06b5dc73940:/workspace/eval/results.xml /tmp/tmp4tmxmtp4.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 781,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03640503197675571
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 15,
    "passed": 754
  },
  "test_branch_errors": {},
  "test_branches": [
    "79b69dd3fd98",
    "1b991a57d4e9",
    "bcaceaff5bda",
    "f53d318371c8",
    "1d30a9bd67f8",
    "2a51b65a0988",
    "c9a563c41e59",
    "47e777b60c5e",
    "f40262460d4c",
    "4325cb75f082",
    "60fce8bbb0c8",
    "28d2b640a014",
    "b75df89939ed",
    "aa26a2ba3c36"
  ],
  "test_records": 769,
  "test_results": [
    {
      "branch": "79b69dd3fd98",
      "extra": {
        "message_chars": 14,
        "message_redacted": true,
        "time": 0.008
      },
      "name": "tests.test_error_handling.test_invalid_color",
      "status": "failure"
    },
    {
      "branch": "79b69dd3fd98",
      "extra": {
        "message_chars": 14,
        "message_redacted": true,
        "time": 0.009
      },
      "name": "tests.test_error_handling.test_invalid_color_numeric",
      "status": "failure"
    },
    {
      "branch": "79b69dd3fd98",
      "extra": {
        "message_chars": 14,
        "message_redacted": true,
        "time": 0.009
      },
      "name": "tests.test_error_handling.test_invalid_color_typo",
      "status": "failure"
    },
    {
      "branch": "1b991a57d4e9",
      "extra": {
        "message_chars": 14,
        "message_redacted": true,
        "time": 0.008
      },
      "name": "eval.tests.test_cmatrix.TestColorOptions.test_color_invalid_shows_error_message",
      "status": "failure"
    },
    {
      "branch": "f53d318371c8",
      "extra": {
        "message_chars": 57,
        "message_redacted": true,
        "time": 0.01
      },
      "name": "eval.tests.test_args_parsing.test_u_valid_values_start_matrix_render_loop[args3]",
      "status": "failure"
    },
    {
      "branch": "f53d318371c8",
      "extra": {
        "message_chars": 57,
        "message_redacted": true,
        "time": 0.009
      },
      "name": "eval.tests.test_args_parsing.test_u_valid_values_start_matrix_render_loop[args1]",
      "status": "failure"
    },
    {
      "branch": "f53d318371c8",
      "extra": {
        "message_chars": 57,
        "message_redacted": true,
        "time": 0.009
      },
      "name": "eval.tests.test_args_parsing.test_u_valid_values_start_matrix_render_loop[args2]",
      "status": "failure"
    },
    {
      "branch": "f53d318371c8",
      "extra": {
        "message_chars": 57,
        "message_redacted": true,
        "time": 0.01
      },
      "name": "eval.tests.test_args_parsing.test_u_out_of_range_or_overflow_values_still_enter_render_loop[args0]",
      "status": "failure"
    },
    {
      "branch": "f53d318371c8",
      "extra": {
        "message_chars": 57,
        "message_redacted": true,
        "time": 0.01
      },
      "name": "eval.tests.test_args_parsing.test_u_valid_values_start_matrix_render_loop[args0]",
      "status": "failure"
    },
    {
      "branch": "f53d318371c8",
      "extra": {
        "message_chars": 57,
        "message_redacted": true,
        "time": 0.008
      },
      "name": "eval.tests.test_args_parsing.test_t_with_value_enters_render_loop",
      "status": "failure"
    },
    {
      "branch": "f53d318371c8",
      "extra": {
        "message_chars": 57,
        "message_redacted": true,
        "time": 0.009
      },
      "name": "eval.tests.test_args_parsing.test_dash_dash_is_treated_as_valid_invocation_and_enters_render_loop",
      "status": "failure"
    },
    {
      "branch": "f53d318371c8",
      "extra": {
        "message_chars": 57,
        "message_redacted": true,
        "time": 0.01
      },
      "name": "eval.tests.test_args_parsing.test_u_out_of_range_or_overflow_values_still_enter_render_loop[args1]",
      "status": "failure"
    },
    {
      "branch": "f53d318371c8",
      "extra": {
        "message_chars": 57,
        "message_redacted": true,
        "time": 0.008
      },
      "name": "eval.tests.test_args_parsing.test_positional_argument_is_accepted_and_enters_render_loop",
      "status": "failure"
    },
    {
      "branch": "f53d318371c8",
      "extra": {
        "message_chars": 57,
        "message_redacted": true,
        "time": 0.01
      },
      "name": "eval.tests.test_args_parsing.test_u_out_of_range_or_overflow_values_still_enter_render_loop[args2]",
      "status": "failure"
    },
    {
      "branch": "60fce8bbb0c8",
      "extra": {
        "message_chars": 14,
        "message_redacted": true,
        "time": 0.01
      },
      "name": "eval.tests.test_help_and_version.test_version_flag",
      "status": "failure"
    }
  ],
  "test_results_omitted": 0,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}