{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "9769a9648602e50e5b937b2ec3ee182f5863ba0f4d9aa98a1b88cae978a3ca65",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.13684379500000432
    },
    {
      "branch": "",
      "command": "find /workspace -type f -exec sha256sum {} + 2>/dev/null | grep -E \"^(b7985a789a31f8939072bd488f2e461e9e35724b2fbee2cf386ef474a41efd4c)  \" | cut -c67- | xargs -I% rm -fv %",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.1357077269999536
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.1342264200000045
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.10836970799994106
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.11961891300006755
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.09406539400004021
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6255841500000088
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12217419399996743
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11384928899997249
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11822048099998028
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11963875399999324
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 290205,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 13.76226670199992
    },
    {
      "branch": "61b7ba1e3b42",
      "command": "docker cp 7073b54d91031cd4804ef45dbfecdd47c97ee16b51414d76ce05183076a6ef48:/workspace/eval/results.xml /tmp/tmp1ro9mw9a.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 202477,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.032023163999951976
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12109489099998427
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12108867300003112
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12994603099991764
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11098849799998334
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 34225,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 6.432121493999944
    },
    {
      "branch": "b919df31faac",
      "command": "docker cp a42bf341809eb03e1eebbfe1e9fee96264d55c6ade2f7409d2ac659faac1f36b:/workspace/eval/results.xml /tmp/tmpcusrtxhg.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 28350,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.022026063999987855
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12231920499993976
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.0937181970000438
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.10140230999991218
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11931228900004953
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 78562,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 9.325019573000077
    },
    {
      "branch": "b9ea5e667678",
      "command": "docker cp 6b5907dc26384fa4ff3c0da3e20205c5d83a04f382926df6dfcc8c45384a016d:/workspace/eval/results.xml /tmp/tmp1k3m6iq9.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 63200,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.033562185000050704
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13183422000008704
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11507498899993607
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11578866299998936
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12792064599989317
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 6124,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.3405055919999995
    },
    {
      "branch": "21d5564e91ec",
      "command": "docker cp b2ac5e8c0281823c367d97e54cdf942819c1c324758f4c08e1a4621aa83a2ad8:/workspace/eval/results.xml /tmp/tmpw_dagay3.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3467,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.02982870399989679
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11596795999992082
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11132360900001004
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12148987800003397
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11530970599994816
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1410,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.480100470000025
    },
    {
      "branch": "2b274c60673c",
      "command": "docker cp 923d17880aa2e5fdef8c23cc99d8b0b4c33be1504deb9cc0ad93417966872291:/workspace/eval/results.xml /tmp/tmpjqts_41c.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 388,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03524142200001279
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13294942699997137
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11167036200004077
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.0965025020000212
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11126326999999492
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 6518,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.4162027670000725
    },
    {
      "branch": "622f5c31ea25",
      "command": "docker cp cc9e63979721e56807283787894d93dc7caafe30f891b022b9d8e67d40393bf0:/workspace/eval/results.xml /tmp/tmpp_i80b0t.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3864,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.017660374000001866
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13042479199998525
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11936533100003999
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12066845000003923
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12231711399999767
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4551,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 5.5667423390000295
    },
    {
      "branch": "26402d0b3a4b",
      "command": "docker cp 4a1de159f6c07ec26d1f3776919d89f95b27743c2895e163e4ca3663973e679c:/workspace/eval/results.xml /tmp/tmpo4937kfb.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2890,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.025493264999909115
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12536656499992205
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12863806299992575
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11596979699993426
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13365030099998876
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4693,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.116564892999918
    },
    {
      "branch": "cfab88d04dba",
      "command": "docker cp 485a5b572fbd8b9bc9f73065a95e5f21c4bf5a85289c5e5e9b9ba880eebb8e83:/workspace/eval/results.xml /tmp/tmplf7zp7ec.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2786,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.035205244000053426
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1406022490000396
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11703451500000028
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.0989735829999745
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.09688549400004831
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4857,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.3307547699999986
    },
    {
      "branch": "ac8b93176216",
      "command": "docker cp 4b42ebf108e41e88e0d4be922466f108eb1d667b6648f7ba7d239ed17d987c77:/workspace/eval/results.xml /tmp/tmpbd0hpsjj.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3112,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.034078532000080486
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13340990599999714
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12567200199998751
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11901181700000052
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11739396300004046
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1607,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.4215462570000454
    },
    {
      "branch": "1e0724bc6817",
      "command": "docker cp d7bf134a57017069ef87fff82144077f10e9c238e6cff17223b7ed6c30e5c479:/workspace/eval/results.xml /tmp/tmp63xjnpra.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 544,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.029512282000041523
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 72,
    "passed": 1984,
    "skipped": 2
  },
  "test_branch_errors": {},
  "test_branches": [
    "61b7ba1e3b42",
    "b919df31faac",
    "b9ea5e667678",
    "21d5564e91ec",
    "2b274c60673c",
    "622f5c31ea25",
    "26402d0b3a4b",
    "cfab88d04dba",
    "ac8b93176216",
    "1e0724bc6817"
  ],
  "test_records": 2058,
  "test_results": [
    {
      "branch": "61b7ba1e3b42",
      "extra": {
        "message_chars": 254,
        "message_redacted": true,
        "time": 0.104
      },
      "name": "eval.tests.test_09_urls_and_links.test_link_with_base",
      "status": "failure"
    },
    {
      "branch": "61b7ba1e3b42",
      "extra": {
        "message_chars": 254,
        "message_redacted": true,
        "time": 0.104
      },
      "name": "eval.tests.test_07_massive_parametrized.test_base_url_rewriting[https://example.com-/path-https://example.com/path]",
      "status": "failure"
    },
    {
      "branch": "61b7ba1e3b42",
      "extra": {
        "message_chars": 274,
        "message_redacted": true,
        "time": 0.091
      },
      "name": "eval.tests.test_04_flags.test_base_flag",
      "status": "failure"
    },
    {
      "branch": "61b7ba1e3b42",
      "extra": {
        "message_chars": 228,
        "message_redacted": true,
        "time": 0.085
      },
      "name": "eval.tests.test_09_urls_and_links.test_base_tag_detection",
      "status": "failure"
    },
    {
      "branch": "61b7ba1e3b42",
      "extra": {
        "message_chars": 18,
        "message_redacted": true,
        "time": 0.001
      },
      "name": "eval.tests.test_15_final_massive_push.test_flag_combinations_matrix[True-True]",
      "status": "skipped"
    },
    {
      "branch": "61b7ba1e3b42",
      "extra": {
        "message_chars": 255,
        "message_redacted": true,
        "time": 0.093
      },
      "name": "eval.tests.test_07_massive_parametrized.test_base_url_rewriting[https://example.com/-/path-https://example.com/path]",
      "status": "failure"
    },
    {
      "branch": "61b7ba1e3b42",
      "extra": {
        "message_chars": 248,
        "message_redacted": true,
        "time": 0.09
      },
      "name": "eval.tests.test_04_flags.test_base_flag_short",
      "status": "failure"
    },
    {
      "branch": "61b7ba1e3b42",
      "extra": {
        "message_chars": 345,
        "message_redacted": true,
        "time": 0.142
      },
      "name": "eval.tests.test_09_urls_and_links.test_multiple_links",
      "status": "failure"
    },
    {
      "branch": "61b7ba1e3b42",
      "extra": {
        "message_chars": 278,
        "message_redacted": true,
        "time": 0.147
      },
      "name": "eval.tests.test_07_massive_parametrized.test_base_url_rewriting[https://example.com/base/-relative-https://example.com/base/relative]",
      "status": "failure"
    },
    {
      "branch": "61b7ba1e3b42",
      "extra": {
        "message_chars": 239,
        "message_redacted": true,
        "time": 0.123
      },
      "name": "eval.tests.test_04_flags.test_detect_base_flag",
      "status": "failure"
    },
    {
      "branch": "61b7ba1e3b42",
      "extra": {
        "message_chars": 286,
        "message_redacted": true,
        "time": 0.155
      },
      "name": "eval.tests.test_09_urls_and_links.test_link_with_fragment",
      "status": "failure"
    },
    {
      "branch": "61b7ba1e3b42",
      "extra": {
        "message_chars": 266,
        "message_redacted": true,
        "time": 0.17
      },
      "name": "eval.tests.test_07_massive_parametrized.test_base_url_rewriting[http://test.org-/page.html-http://test.org/page.html]",
      "status": "failure"
    },
    {
      "branch": "61b7ba1e3b42",
      "extra": {
        "message_chars": 225,
        "message_redacted": true,
        "time": 0.167
      },
      "name": "eval.tests.test_04_flags.test_detect_base_short",
      "status": "failure"
    },
    {
      "branch": "61b7ba1e3b42",
      "extra": {
        "message_chars": 280,
        "message_redacted": true,
        "time": 0.136
      },
      "name": "eval.tests.test_09_urls_and_links.test_relative_path",
      "status": "failure"
    },
    {
      "branch": "b919df31faac",
      "extra": {
        "message_chars": 277,
        "message_redacted": true,
        "time": 0.126
      },
      "name": "eval.tests.test_base_url.test_detect_base_overrides_base_flag",
      "status": "failure"
    },
    {
      "branch": "b919df31faac",
      "extra": {
        "message_chars": 274,
        "message_redacted": true,
        "time": 0.172
      },
      "name": "eval.tests.test_base_url.test_base_url_flag",
      "status": "failure"
    },
    {
      "branch": "b919df31faac",
      "extra": {
        "message_chars": 331,
        "message_redacted": true,
        "time": 0.15
      },
      "name": "eval.tests.test_base_url.test_base_url_multiple_links",
      "status": "failure"
    },
    {
      "branch": "b919df31faac",
      "extra": {
        "message_chars": 317,
        "message_redacted": true,
        "time": 0.169
      },
      "name": "eval.tests.test_base_url.test_base_url_rewrite_link_tag",
      "status": "failure"
    },
    {
      "branch": "b919df31faac",
      "extra": {
        "message_chars": 232,
        "message_redacted": true,
        "time": 0.151
      },
      "name": "eval.tests.test_base_url.test_detect_base_short_flag",
      "status": "failure"
    },
    {
      "branch": "b919df31faac",
      "extra": {
        "message_chars": 243,
        "message_redacted": true,
        "time": 0.137
      },
      "name": "eval.tests.test_base_url.test_detect_base_flag",
      "status": "failure"
    },
    {
      "branch": "b919df31faac",
      "extra": {
        "message_chars": 243,
        "message_redacted": true,
        "time": 0.127
      },
      "name": "eval.tests.test_combined_flags.test_detect_base_with_pretty",
      "status": "failure"
    },
    {
      "branch": "b919df31faac",
      "extra": {
        "message_chars": 331,
        "message_redacted": true,
        "time": 0.087
      },
      "name": "eval.tests.test_combined_flags.test_base_with_multiple_links",
      "status": "failure"
    },
    {
      "branch": "b919df31faac",
      "extra": {
        "message_chars": 96,
        "message_redacted": true,
        "time": 0.102
      },
      "name": "eval.tests.test_combined_flags.test_all_flags_together",
      "status": "failure"
    },
    {
      "branch": "b919df31faac",
      "extra": {
        "message_chars": 109,
        "message_redacted": true,
        "time": 0.141
      },
      "name": "eval.tests.test_remove_nodes.test_remove_nodes_all_filtered_out",
      "status": "failure"
    },
    {
      "branch": "b919df31faac",
      "extra": {
        "message_chars": 284,
        "message_redacted": true,
        "time": 0.107
      },
      "name": "eval.tests.test_base_url.test_base_url_with_path",
      "status": "failure"
    },
    {
      "branch": "b919df31faac",
      "extra": {
        "message_chars": 281,
        "message_redacted": true,
        "time": 0.116
      },
      "name": "eval.tests.test_base_url.test_base_url_rewrite_area_tag",
      "status": "failure"
    },
    {
      "branch": "b919df31faac",
      "extra": {
        "message_chars": 277,
        "message_redacted": true,
        "time": 0.144
      },
      "name": "eval.tests.test_base_url.test_detect_base_with_fallback",
      "status": "failure"
    },
    {
      "branch": "b919df31faac",
      "extra": {
        "message_chars": 254,
        "message_redacted": true,
        "time": 0.141
      },
      "name": "eval.tests.test_base_url.test_base_url_short_flag",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 138,
        "message_redacted": true,
        "time": 0.162
      },
      "name": "tests.test_gaps.test_empty_href_attribute",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 360,
        "message_redacted": true,
        "time": 0.168
      },
      "name": "tests.test_gaps.test_link_rewriting_link_tag",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 150,
        "message_redacted": true,
        "time": 0.183
      },
      "name": "tests.test_gaps.test_detect_base_with_fallback",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 197,
        "message_redacted": true,
        "time": 0.182
      },
      "name": "tests.test_cli_edge.test_multiple_attribute_flags_error",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 247,
        "message_redacted": true,
        "time": 0.147
      },
      "name": "tests.test_gaps.test_pretty_print_with_comments",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 202,
        "message_redacted": true,
        "time": 0.127
      },
      "name": "tests.test_gaps.test_url_with_fragment_and_query",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 139,
        "message_redacted": true,
        "time": 0.116
      },
      "name": "tests.test_cli_edge.test_four_slash_url_special_handling",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 202,
        "message_redacted": true,
        "time": 0.09
      },
      "name": "tests.test_gaps.test_href_without_scheme",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 346,
        "message_redacted": true,
        "time": 0.092
      },
      "name": "tests.test_integration.test_navigation_links_with_base_detection",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 235,
        "message_redacted": true,
        "time": 0.114
      },
      "name": "tests.test_cli_edge.test_binary_data_in_stdin",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 224,
        "message_redacted": true,
        "time": 0.095
      },
      "name": "tests.test_integration.test_pretty_print_with_comments",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 254,
        "message_redacted": true,
        "time": 0.113
      },
      "name": "tests.test_integration.test_selector_pretty_print",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 196,
        "message_redacted": true,
        "time": 0.098
      },
      "name": "tests.test_integration.test_malformed_html_parsing",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 149,
        "message_redacted": true,
        "time": 0.091
      },
      "name": "tests.test_io.test_base_url_rewriting",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 249,
        "message_redacted": true,
        "time": 0.124
      },
      "name": "tests.test_output.test_pretty_print_indentation",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 244,
        "message_redacted": true,
        "time": 0.137
      },
      "name": "tests.test_output.test_pretty_print_with_complex_nested_structure",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 200,
        "message_redacted": true,
        "time": 0.12
      },
      "name": "tests.test_gaps.test_protocol_relative_url",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 176,
        "message_redacted": true,
        "time": 0.098
      },
      "name": "tests.test_gaps.test_quadruple_slash_href",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 240,
        "message_redacted": true,
        "time": 0.117
      },
      "name": "tests.test_gaps.test_link_rewriting_with_comment_nodes",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 218,
        "message_redacted": true,
        "time": 0.109
      },
      "name": "tests.test_integration.test_quad_slash_url_handling",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 476,
        "message_redacted": true,
        "time": 0.092
      },
      "name": "tests.test_gaps.test_link_rewriting_area_tag",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 117,
        "message_redacted": true,
        "time": 0.083
      },
      "name": "tests.test_io.test_file_input_dash_means_stdin",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 244,
        "message_redacted": true,
        "time": 0.099
      },
      "name": "tests.test_integration.test_anchor_without_href_with_base",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 131,
        "message_redacted": true,
        "time": 0.093
      },
      "name": "tests.test_integration.test_whitespace_only_input",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 232,
        "message_redacted": true,
        "time": 0.084
      },
      "name": "tests.test_output.test_list_with_block_children_pretty",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 268,
        "message_redacted": true,
        "time": 0.106
      },
      "name": "tests.test_output.test_script_style_elements_in_pretty_print",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 227,
        "message_redacted": true,
        "time": 0.11
      },
      "name": "tests.test_output.test_head_elements_pretty_print",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 149,
        "message_redacted": true,
        "time": 0.112
      },
      "name": "tests.test_io.test_detect_base_from_document",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 164,
        "message_redacted": true,
        "time": 0.102
      },
      "name": "tests.test_removal.test_removal_with_class_selector",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 240,
        "message_redacted": true,
        "time": 0.172
      },
      "name": "tests.test_output.test_table_pretty_print",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 260,
        "message_redacted": true,
        "time": 0.091
      },
      "name": "tests.test_selectors.test_universal_selector",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 153,
        "message_redacted": true,
        "time": 0.142
      },
      "name": "tests.test_gaps.test_triple_slash_href",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 245,
        "message_redacted": true,
        "time": 0.114
      },
      "name": "tests.test_gaps.test_non_link_element_with_href",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 149,
        "message_redacted": true,
        "time": 0.125
      },
      "name": "tests.test_gaps.test_detect_base_with_base_tag",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 98,
        "message_redacted": true,
        "time": 0.086
      },
      "name": "tests.test_io.test_output_dash_means_stdout",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 295,
        "message_redacted": true,
        "time": 0.113
      },
      "name": "tests.test_output.test_void_elements_in_pretty_print",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 380,
        "message_redacted": true,
        "time": 0.144
      },
      "name": "tests.test_urls.test_basic_url_resolution_with_base",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 189,
        "message_redacted": true,
        "time": 0.136
      },
      "name": "tests.test_urls.test_empty_href_attribute_resolution",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 279,
        "message_redacted": true,
        "time": 0.103
      },
      "name": "tests.test_urls.test_four_slash_href_special_handling",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 273,
        "message_redacted": true,
        "time": 0.119
      },
      "name": "tests.test_urls.test_selector_scope_with_html_base_rewriting",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 148,
        "message_redacted": true,
        "time": 0.125
      },
      "name": "tests.test_urls.test_html_output_with_base_rewrites_hrefs",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 156,
        "message_redacted": true,
        "time": 0.134
      },
      "name": "tests.test_urls.test_non_link_elements_not_rewritten",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 281,
        "message_redacted": true,
        "time": 0.118
      },
      "name": "tests.test_urls.test_non_link_elements_html_output_selective_rewriting",
      "status": "failure"
    },
    {
      "branch": "b9ea5e667678",
      "extra": {
        "message_chars": 281,
        "message_redacted": true,
        "time": 0.107
      },
      "name": "tests.test_removal.test_deeply_nested_removal",
      "status": "failure"
    },
    {
      "branch": "26402d0b3a4b",
      "extra": {
        "message_chars": 328,
        "message_redacted": true,
        "time": 0.157
      },
      "name": "eval.tests.test_htmlq_io.test_default_selector_html_echoes_document_from_stdin",
      "status": "failure"
    },
    {
      "branch": "ac8b93176216",
      "extra": {
        "message_chars": 60,
        "message_redacted": true,
        "time": 0.0
      },
      "name": "eval.tests.test_htmlq_cli.test_help_long_flag_matches_short",
      "status": "skipped"
    }
  ],
  "test_results_omitted": 0,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}