{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "9f2a26ad2fd4c2a9833126087aa51922801b189ff4dc93a0136cb1d6130d6c6d",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.17046138700970914
    },
    {
      "branch": "",
      "command": "find /workspace -type f -exec sha256sum {} + 2>/dev/null | grep -E \"^(a99e4951d8e1c8583cb76c6c104e014e0b17dabafb7db27ca9acea9461a08ce8)  \" | cut -c67- | xargs -I% rm -fv %",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.13778421899769455
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.15130831899296027
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.09677820499928202
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.12474552399362437
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.11812547699082643
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6634212669887347
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14177115699567366
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12419124400184955
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1206695220025722
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1352205200091703
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 10674,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 4.57507133600302
    },
    {
      "branch": "8384425dbf58",
      "command": "docker cp d8468dc05ddc50826a09c4c7ccd1abd1b121139b498c4281ac567d1b8040ca8b:/workspace/eval/results.xml /tmp/tmpdauk8dtk.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 7499,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.02376330400875304
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12531852698884904
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.128487520996714
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1009711810038425
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13105670300137717
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 18347,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 6.844051035994198
    },
    {
      "branch": "79af22e6a745",
      "command": "docker cp 8cde8b67e4ce0f7b2bf4efdc28b58d23367cc6a9714214057c59069506520dda:/workspace/eval/results.xml /tmp/tmp3kxc7xyb.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 16446,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.0368918659951305
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13099308899836615
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.10571751301176846
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1295307749969652
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1356565479945857
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 103875,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 37.05495025700657
    },
    {
      "branch": "36519d111ac2",
      "command": "docker cp c3587b7699875fc0d7ec0eb3034527000c5084210f896f57f9cf5ad096935cb1:/workspace/eval/results.xml /tmp/tmpouzhky4w.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 110885,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.02319155698933173
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12012437400699127
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1404163040133426
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1436525850003818
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1314379760005977
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 8862,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.845827723009279
    },
    {
      "branch": "174ba0ea8bbc",
      "command": "docker cp 9c1707822814c320b7b7ec47d41570a5eeafa495a7cf5b48ccce2185d0b1e9b1:/workspace/eval/results.xml /tmp/tmpjwmi7hm4.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5272,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.04158295701199677
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14420906698796898
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13124632499238942
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13367143599316478
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1320213050057646
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 10377,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.891142314998433
    },
    {
      "branch": "dab235980e78",
      "command": "docker cp 15a68190a557a3db450adf5eba2d8a2ab1f2a7e7ebd279c0eca53df7f5d05dcb:/workspace/eval/results.xml /tmp/tmph_p7fkq9.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 7857,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03936956099641975
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13892726300400682
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12654839499737136
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12314729299396276
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1309028749965364
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4152,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.747122897999361
    },
    {
      "branch": "744a20b591d4",
      "command": "docker cp 8c276523295edf58ae7577849f1ed3b5d6d03d342845ecd139b5a1cad2583f17:/workspace/eval/results.xml /tmp/tmpvxw8zwlx.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2043,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03899527400790248
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12413661999744363
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11182062000443693
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12692482600687072
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13266242199460976
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4425,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.221989619996748
    },
    {
      "branch": "fafedd2177bc",
      "command": "docker cp c2a219f5dce6b749089c03fdbeeac5a1c49cd386f61ee85a1394cf40df99740d:/workspace/eval/results.xml /tmp/tmppg2um5cs.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2433,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.019853952995617874
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14868188599939458
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11404328299977351
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13822579600673635
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12419289600802585
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 12733,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 4.575612147993525
    },
    {
      "branch": "5972c17f0779",
      "command": "docker cp f275303c80f98f800bbc46a7a2cb574a543f4e118a81e3127e1c8a2644aa8279:/workspace/eval/results.xml /tmp/tmpe9yfs8fb.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 11756,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03787957500026096
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13482870199368335
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13680133898742497
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1271206999954302
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13067097000021022
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 10621,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.8296394429926295
    },
    {
      "branch": "9ce5aad8d0a3",
      "command": "docker cp f0a047d35a1c60768d8fa5b419e0a12e41a49649b90882e2a6e101185db373b3:/workspace/eval/results.xml /tmp/tmp_1ox7ey6.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5908,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03004719299497083
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13833535800222307
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12091538899403531
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.10262488899752498
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12779091999982484
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 8154,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 7.5353751509974245
    },
    {
      "branch": "966baa6441ca",
      "command": "docker cp 57a6cfd8617972b36efc114b77c1a273a013dff622fefb07dc435a1e95400fd4:/workspace/eval/results.xml /tmp/tmpjda95l_v.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4690,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.040464322999469005
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 83,
    "passed": 336
  },
  "test_branch_errors": {},
  "test_branches": [
    "8384425dbf58",
    "79af22e6a745",
    "36519d111ac2",
    "174ba0ea8bbc",
    "dab235980e78",
    "744a20b591d4",
    "fafedd2177bc",
    "5972c17f0779",
    "9ce5aad8d0a3",
    "966baa6441ca"
  ],
  "test_records": 419,
  "test_results": [
    {
      "branch": "8384425dbf58",
      "extra": {
        "message_chars": 450,
        "message_redacted": true,
        "time": 0.262
      },
      "name": "eval.tests.test_basic.TestInputHandling.test_empty_file_path",
      "status": "failure"
    },
    {
      "branch": "8384425dbf58",
      "extra": {
        "message_chars": 273,
        "message_redacted": true,
        "time": 0.29
      },
      "name": "eval.tests.test_basic.TestOutputGeneration.test_output_same_as_input_error",
      "status": "failure"
    },
    {
      "branch": "79af22e6a745",
      "extra": {
        "message_chars": 1083,
        "message_redacted": true,
        "time": 0.275
      },
      "name": "eval.tests.test_error_handling.test_output_file_same_as_input",
      "status": "failure"
    },
    {
      "branch": "79af22e6a745",
      "extra": {
        "message_chars": 1465,
        "message_redacted": true,
        "time": 0.301
      },
      "name": "eval.tests.test_error_handling.test_empty_wsdl_file",
      "status": "failure"
    },
    {
      "branch": "79af22e6a745",
      "extra": {
        "message_chars": 602,
        "message_redacted": true,
        "time": 0.212
      },
      "name": "eval.tests.test_error_handling.test_treat_last_arg_as_wsdl",
      "status": "failure"
    },
    {
      "branch": "79af22e6a745",
      "extra": {
        "message_chars": 408,
        "message_redacted": true,
        "time": 0.238
      },
      "name": "eval.tests.test_error_handling.test_invalid_xml_but_looks_like_wsdl",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 1118,
        "message_redacted": true,
        "time": 0.265
      },
      "name": "tests.test_cli.test_invalid_xml_eof_error",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 118,
        "message_redacted": true,
        "time": 0.245
      },
      "name": "tests.test_cli.test_directory_input_error",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 2451,
        "message_redacted": true,
        "time": 0.217
      },
      "name": "tests.test_cli.test_make_public_flag_false",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 462,
        "message_redacted": true,
        "time": 0.246
      },
      "name": "tests.test_errors.test_invalid_output_directory",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 490,
        "message_redacted": true,
        "time": 0.241
      },
      "name": "tests.test_errors.test_output_file_same_as_input",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 175,
        "message_redacted": true,
        "time": 0.227
      },
      "name": "tests.test_errors.test_empty_file",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 528,
        "message_redacted": true,
        "time": 0.221
      },
      "name": "tests.test_errors.test_empty_output_filename_creates_directory_error",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 398,
        "message_redacted": true,
        "time": 0.253
      },
      "name": "tests.test_errors.test_package_name_starts_with_digit",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 370,
        "message_redacted": true,
        "time": 0.233
      },
      "name": "tests.test_gap_wsdl_xsd.test_xsd_complextype_invalid_boolean_mixed",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 403,
        "message_redacted": true,
        "time": 0.24
      },
      "name": "tests.test_gap_wsdl_xsd.test_xsd_element_invalid_boolean_nillable",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 1359,
        "message_redacted": true,
        "time": 0.186
      },
      "name": "tests.test_errors.test_no_arguments",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 575,
        "message_redacted": true,
        "time": 0.406
      },
      "name": "tests.test_harvest.test_unmarshal_vim_wsdl",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 576,
        "message_redacted": true,
        "time": 1.771
      },
      "name": "tests.test_errors.test_http_500_error",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 543,
        "message_redacted": true,
        "time": 0.251
      },
      "name": "tests.test_location.test_relative_import_resolution",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 643,
        "message_redacted": true,
        "time": 3.149
      },
      "name": "tests.test_errors.test_https_with_insecure_flag_bypasses_cert_check",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 317,
        "message_redacted": true,
        "time": 0.26
      },
      "name": "tests.test_cli.test_output_file_equals_input_error",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 586,
        "message_redacted": true,
        "time": 0.276
      },
      "name": "tests.test_cli.test_malformed_xml_syntax_error",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 364,
        "message_redacted": true,
        "time": 0.281
      },
      "name": "tests.test_gap_wsdl_xsd.test_xsd_element_multiple_invalid_booleans",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 819,
        "message_redacted": true,
        "time": 0.194
      },
      "name": "tests.test_errors.test_unsupported_url_scheme",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 171,
        "message_redacted": true,
        "time": 0.235
      },
      "name": "tests.test_errors.test_invalid_xml_content",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 370,
        "message_redacted": true,
        "time": 0.281
      },
      "name": "tests.test_gap_wsdl_xsd.test_wsdl_types_nested_schema_decode_error",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 2547,
        "message_redacted": true,
        "time": 0.26
      },
      "name": "tests.test_errors.test_file_path_exceeds_system_limit",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 374,
        "message_redacted": true,
        "time": 0.268
      },
      "name": "tests.test_gap_wsdl_xsd.test_xsd_complextype_invalid_boolean_abstract",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 826,
        "message_redacted": true,
        "time": 0.308
      },
      "name": "tests.test_errors.test_invalid_url_dns_failure",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 70,
        "message_redacted": true,
        "time": 2.337
      },
      "name": "tests.test_harvest.test_enumerations_generated_correctly",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 430,
        "message_redacted": true,
        "time": 0.304
      },
      "name": "tests.test_location.test_http_url_404_error",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 592,
        "message_redacted": true,
        "time": 0.248
      },
      "name": "tests.test_harvest.test_output_file_same_as_input",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 744,
        "message_redacted": true,
        "time": 0.271
      },
      "name": "tests.test_location.test_nested_parent_directory_reference",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 918,
        "message_redacted": true,
        "time": 1.191
      },
      "name": "tests.test_errors.test_https_certificate_verification_failure",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 569,
        "message_redacted": true,
        "time": 0.236
      },
      "name": "tests.test_location.test_path_with_spaces",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 540,
        "message_redacted": true,
        "time": 0.266
      },
      "name": "tests.test_location.test_symlink_resolution",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 366,
        "message_redacted": true,
        "time": 0.255
      },
      "name": "tests.test_gap_wsdl_xsd.test_xsd_complextype_both_abstract_and_mixed_invalid",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 402,
        "message_redacted": true,
        "time": 0.263
      },
      "name": "tests.test_errors.test_invalid_package_name_with_spaces",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 476,
        "message_redacted": true,
        "time": 0.273
      },
      "name": "tests.test_errors.test_malformed_xml",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 573,
        "message_redacted": true,
        "time": 0.288
      },
      "name": "tests.test_location.test_epcis_multi_file_import_chain",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 731,
        "message_redacted": true,
        "time": 0.21
      },
      "name": "tests.test_location.test_current_directory_dot_slash_references",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 447,
        "message_redacted": true,
        "time": 0.296
      },
      "name": "tests.test_location.test_url_encoded_characters_in_http_path",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 409,
        "message_redacted": true,
        "time": 0.29
      },
      "name": "tests.test_location.test_file_wsdl_with_http_import",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 552,
        "message_redacted": true,
        "time": 1.91
      },
      "name": "tests.test_errors.test_http_404_error",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 592,
        "message_redacted": true,
        "time": 0.246
      },
      "name": "tests.test_location.test_special_characters_in_path",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 746,
        "message_redacted": true,
        "time": 0.237
      },
      "name": "tests.test_errors.test_http_connection_refused",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 409,
        "message_redacted": true,
        "time": 0.427
      },
      "name": "tests.test_location.test_https_import_in_wsdl",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 555,
        "message_redacted": true,
        "time": 0.273
      },
      "name": "tests.test_location.test_absolute_path_import",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 319,
        "message_redacted": true,
        "time": 0.319
      },
      "name": "tests.test_location.test_http_url_as_main_input",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 691,
        "message_redacted": true,
        "time": 0.276
      },
      "name": "tests.test_location.test_deeply_nested_parent_directory_traversal",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 780,
        "message_redacted": true,
        "time": 0.278
      },
      "name": "tests.test_location.test_malformed_url_treated_as_file",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 705,
        "message_redacted": true,
        "time": 0.232
      },
      "name": "tests.test_location.test_multiple_imports_same_wsdl",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 321,
        "message_redacted": true,
        "time": 0.4
      },
      "name": "tests.test_location.test_https_url_as_main_input",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 58,
        "message_redacted": true,
        "time": 0.299
      },
      "name": "tests.test_operations.test_operation_interface_with_multiple_methods",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 606,
        "message_redacted": true,
        "time": 0.227
      },
      "name": "tests.test_location.test_deeply_nested_relative_path",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 246,
        "message_redacted": true,
        "time": 0.278
      },
      "name": "tests.test_operations.test_operations_with_faults_generate_error_documentation",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 2089,
        "message_redacted": true,
        "time": 0.298
      },
      "name": "tests.test_operations.test_operation_documentation_preserved",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 552,
        "message_redacted": true,
        "time": 0.293
      },
      "name": "tests.test_operations.test_empty_string_type_for_missing_io",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 307,
        "message_redacted": true,
        "time": 0.295
      },
      "name": "tests.test_type_generation.test_basic_xsd_types_generation",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 247,
        "message_redacted": true,
        "time": 0.232
      },
      "name": "tests.test_operations.test_operation_with_no_input",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 306,
        "message_redacted": true,
        "time": 0.263
      },
      "name": "tests.test_type_generation.test_complex_content_extension",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 306,
        "message_redacted": true,
        "time": 0.255
      },
      "name": "tests.test_type_generation.test_reserved_go_keywords_escaping",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 2013,
        "message_redacted": true,
        "time": 0.205
      },
      "name": "tests.test_type_generation.test_unicode_names_in_types",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 416,
        "message_redacted": true,
        "time": 0.274
      },
      "name": "tests.test_operations.test_operation_with_no_output",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 234,
        "message_redacted": true,
        "time": 0.24
      },
      "name": "tests.test_type_generation.test_invalid_wsdl_file_error_handling",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 306,
        "message_redacted": true,
        "time": 0.273
      },
      "name": "tests.test_type_generation.test_enumeration_restriction_generation",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 666,
        "message_redacted": true,
        "time": 0.239
      },
      "name": "tests.test_type_generation.test_list_type_generation",
      "status": "failure"
    },
    {
      "branch": "36519d111ac2",
      "extra": {
        "message_chars": 127,
        "message_redacted": true,
        "time": 30.046
      },
      "name": "tests.test_errors.test_invalid_port_number_in_url",
      "status": "failure"
    },
    {
      "branch": "174ba0ea8bbc",
      "extra": {
        "message_chars": 199,
        "message_redacted": true,
        "time": 0.239
      },
      "name": "eval.tests.test_argparse_validation.test_bool_flag_invalid_value_is_error",
      "status": "failure"
    },
    {
      "branch": "174ba0ea8bbc",
      "extra": {
        "message_chars": 529,
        "message_redacted": true,
        "time": 0.266
      },
      "name": "eval.tests.test_argparse_validation.test_double_dash_forces_following_token_to_be_positional_not_flag",
      "status": "failure"
    },
    {
      "branch": "174ba0ea8bbc",
      "extra": {
        "message_chars": 95,
        "message_redacted": true,
        "time": 0.248
      },
      "name": "eval.tests.test_argparse_validation.test_positional_first_then_flag_treated_as_filename_and_causes_open_error",
      "status": "failure"
    },
    {
      "branch": "dab235980e78",
      "extra": {
        "message_chars": 671,
        "message_redacted": true,
        "time": 0.093
      },
      "name": "eval.tests.test_help_output.test_help_has_usage_line",
      "status": "failure"
    },
    {
      "branch": "dab235980e78",
      "extra": {
        "message_chars": 492,
        "message_redacted": true,
        "time": 0.091
      },
      "name": "eval.tests.test_help_output.test_help_indentation_uses_tabbed_description_lines",
      "status": "failure"
    },
    {
      "branch": "dab235980e78",
      "extra": {
        "message_chars": 382,
        "message_redacted": true,
        "time": 0.247
      },
      "name": "eval.tests.test_help_output.test_help_output_matches_baseline_exactly",
      "status": "failure"
    },
    {
      "branch": "744a20b591d4",
      "extra": {
        "message_chars": 122,
        "message_redacted": true,
        "time": 0.267
      },
      "name": "eval.tests.test_error_cases.test_invalid_wsdl_content_exits_nonzero_and_prints_EOF",
      "status": "failure"
    },
    {
      "branch": "5972c17f0779",
      "extra": {
        "message_chars": 286,
        "message_redacted": true,
        "time": 0.294
      },
      "name": "eval.tests.test_gowsdl.TestErrorHandling.test_nonexistent_output_directory",
      "status": "failure"
    },
    {
      "branch": "5972c17f0779",
      "extra": {
        "message_chars": 2451,
        "message_redacted": true,
        "time": 0.258
      },
      "name": "eval.tests.test_gowsdl.TestCommandLineFlags.test_make_public_false",
      "status": "failure"
    },
    {
      "branch": "9ce5aad8d0a3",
      "extra": {
        "message_chars": 406,
        "message_redacted": true,
        "time": 0.282
      },
      "name": "eval.tests.test_gowsdl_cli.test_help_long_exact_except_usage_path",
      "status": "failure"
    },
    {
      "branch": "9ce5aad8d0a3",
      "extra": {
        "message_chars": 250,
        "message_redacted": true,
        "time": 0.253
      },
      "name": "eval.tests.test_gowsdl_cli.test_make_public_flag_changes_generated_code",
      "status": "failure"
    },
    {
      "branch": "9ce5aad8d0a3",
      "extra": {
        "message_chars": 13,
        "message_redacted": true,
        "time": 0.25
      },
      "name": "eval.tests.test_gowsdl_cli.test_d_flag_requires_directory_to_exist",
      "status": "failure"
    },
    {
      "branch": "9ce5aad8d0a3",
      "extra": {
        "message_chars": 406,
        "message_redacted": true,
        "time": 0.259
      },
      "name": "eval.tests.test_gowsdl_cli.test_help_h_exact_except_usage_path",
      "status": "failure"
    },
    {
      "branch": "966baa6441ca",
      "extra": {
        "message_chars": 69,
        "message_redacted": true,
        "time": 2.03
      },
      "name": "eval.tests.test_externalized_gowsdl.test_ext_enumerations_generated_correctly[vboxweb.wsdl-SettingsVersionV1_14-SettingsVersion-v1_14]",
      "status": "failure"
    }
  ],
  "test_results_omitted": 0,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}