{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "d27ae75404d218eb1b8f977b55afe0f65d01e533c72846e21e769dc700c50023",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.15058872500958387
    },
    {
      "branch": "",
      "command": "find /workspace -type f -exec sha256sum {} + 2>/dev/null | grep -E \"^(e7464f4078a4be427e752bfdd7aae891f74287fe0afe4c347a31d87b204467ff)  \" | cut -c67- | xargs -I% rm -fv %",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.13100430701160803
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.16255715998704545
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.12079081899719313
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.137374017998809
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.1260869649995584
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6775703619932756
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12591981700097676
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1391361540008802
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12608154500776436
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11870053700113203
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 10751,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 54.80558359400311
    },
    {
      "branch": "d95dad695c95",
      "command": "docker cp 83b6ef5a75d2ad67a4a6249570ad17471c30dd8bb7cb9fab241d208cd375817d:/workspace/eval/results.xml /tmp/tmpn5j_akvu.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 12594,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03912033000960946
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10424268701171968
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.08283275400754064
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12388817600731272
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11368525600119028
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 103779,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 509.43507073998626
    },
    {
      "branch": "8f4b78a7e9eb",
      "command": "docker cp cab14ab03bd2d7a778e6469b36323b87bbe4b47e77b4ee09ed14a3c50b2e1e11:/workspace/eval/results.xml /tmp/tmpwqy7ldhe.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 82166,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.04042940100771375
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12405088200466707
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11901849300193135
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11857768800109625
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12070492000202648
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 13767,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 55.74773182599165
    },
    {
      "branch": "e454759f15a4",
      "command": "docker cp 25cb4aa7dfc121d8ad33b4b73b8f2f5955650839aab0289f7122698331e7e074:/workspace/eval/results.xml /tmp/tmp1pys9vbn.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 10855,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.04076129599707201
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14603699900908396
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13532903700252064
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11321107299590949
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11707053999998607
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4036,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.602137642999878
    },
    {
      "branch": "f1e76ae50499",
      "command": "docker cp a6fd553ebda332ed5ea153e9f8976fb42992cf4deaf6ee11e6b4bf0bf0939e46:/workspace/eval/results.xml /tmp/tmp1jt0bi9b.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2133,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.030886181004461832
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13261635499657132
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.10976165000465699
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1337369690008927
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12899186499998905
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4183,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.941319584992016
    },
    {
      "branch": "c977eb4a4e02",
      "command": "docker cp b5fcbeeedc754a3d11b329add689896d4333632ddc614d4d637ad59a47bce2b5:/workspace/eval/results.xml /tmp/tmpjtg_72ob.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2084,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.0339510199992219
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11829644298995845
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11529874400002882
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.09683409999706782
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1403690769948298
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3514,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.6429867679980816
    },
    {
      "branch": "4acd63872783",
      "command": "docker cp c2186a8f9d160c7a8dfd6f4b60994ffe6b65cf4d7bc21ee9acc910c8b6018c3d:/workspace/eval/results.xml /tmp/tmpmqx68r71.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1758,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.042120707003050484
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13297068000247236
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1312294329982251
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.10160541399091017
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.128081476999796
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2532,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 4.598823807013105
    },
    {
      "branch": "6ce775dbab8a",
      "command": "docker cp 6309694ba8a3404a9124a93405229e587f37948eb7c593045780f521eab184bc:/workspace/eval/results.xml /tmp/tmp27zmtp2i.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 500,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03669577201071661
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1454602060111938
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13712054200004786
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12215155000740197
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12450970699137542
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1902,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.436329158008448
    },
    {
      "branch": "ac290e5f97d2",
      "command": "docker cp 2a0027b184e4a7ef2b059264ed5bbc150b63f442e9d5c542e863475d733993c3:/workspace/eval/results.xml /tmp/tmpctbhr47u.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1147,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03733280400047079
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1406067339994479
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13155963900499046
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12043777399230748
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13418240200553555
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 12390,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 54.34015903700492
    },
    {
      "branch": "183567ae769e",
      "command": "docker cp 117fc4a02c8c1f9a276971677a377f95b87bc0660974e129465b6f59b5862d10:/workspace/eval/results.xml /tmp/tmpn6_ssv2q.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 7712,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03816316199663561
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 60,
    "passed": 390
  },
  "test_branch_errors": {},
  "test_branches": [
    "d95dad695c95",
    "8f4b78a7e9eb",
    "e454759f15a4",
    "f1e76ae50499",
    "c977eb4a4e02",
    "4acd63872783",
    "6ce775dbab8a",
    "ac290e5f97d2",
    "183567ae769e"
  ],
  "test_records": 450,
  "test_results": [
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 336,
        "message_redacted": true,
        "time": 0.268
      },
      "name": "tests.test_cli_basics.test_env_bore_server_used_when_no_to_flag",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 207,
        "message_redacted": true,
        "time": 0.162
      },
      "name": "tests.test_cli_basics.test_server_min_port_explicit_1024",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 212,
        "message_redacted": true,
        "time": 1.111
      },
      "name": "tests.test_cli_basics.test_local_long_host_name",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 478,
        "message_redacted": true,
        "time": 1.083
      },
      "name": "tests.test_client.test_port_boundary_minimum",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 409,
        "message_redacted": true,
        "time": 0.169
      },
      "name": "tests.test_client.test_env_server_invalid",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 484,
        "message_redacted": true,
        "time": 1.069
      },
      "name": "tests.test_client.test_port_boundary_maximum",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 502,
        "message_redacted": true,
        "time": 1.072
      },
      "name": "tests.test_client.test_remote_port_boundary_maximum",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 335,
        "message_redacted": true,
        "time": 0.237
      },
      "name": "tests.test_client.test_dns_lookup_failure",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 264,
        "message_redacted": true,
        "time": 0.2
      },
      "name": "tests.test_client.test_remote_port_boundary_minimum",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 483,
        "message_redacted": true,
        "time": 0.976
      },
      "name": "tests.test_client.test_localhost_hostname",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 147,
        "message_redacted": true,
        "time": 1.888
      },
      "name": "tests.test_client.test_unreachable_ip_address",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 426,
        "message_redacted": true,
        "time": 5.945
      },
      "name": "tests.test_auth.test_auth_wrong_secret_error@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 739,
        "message_redacted": true,
        "time": 0.145
      },
      "name": "tests.test_client.test_very_long_hostname",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 409,
        "message_redacted": true,
        "time": 0.119
      },
      "name": "tests.test_client.test_invalid_domain_name",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 303,
        "message_redacted": true,
        "time": 0.182
      },
      "name": "tests.test_client.test_empty_server_address",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 304,
        "message_redacted": true,
        "time": 0.155
      },
      "name": "tests.test_client.test_whitespace_server_address",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 439,
        "message_redacted": true,
        "time": 5.918
      },
      "name": "tests.test_auth.test_auth_empty_secret_mismatch@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 824,
        "message_redacted": true,
        "time": 10.439
      },
      "name": "tests.test_auth.test_auth_long_secret@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 346,
        "message_redacted": true,
        "time": 5.949
      },
      "name": "tests.test_auth.test_auth_case_sensitive_secret@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 423,
        "message_redacted": true,
        "time": 10.467
      },
      "name": "tests.test_auth.test_auth_repeated_char_secret@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 330,
        "message_redacted": true,
        "time": 5.952
      },
      "name": "tests.test_auth.test_auth_multiple_clients_different_secrets@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 330,
        "message_redacted": true,
        "time": 5.932
      },
      "name": "tests.test_auth.test_auth_prefix_suffix_mismatch@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 366,
        "message_redacted": true,
        "time": 6.62
      },
      "name": "tests.test_auth.test_auth_rapid_authentication_attempts@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 201,
        "message_redacted": true,
        "time": 1.068
      },
      "name": "tests.test_client_gaps.test_client_handles_wrong_authentication_secret@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 1781,
        "message_redacted": true,
        "time": 3.489
      },
      "name": "tests.test_client_gaps.test_client_handles_local_connection_failure@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 567,
        "message_redacted": true,
        "time": 0.167
      },
      "name": "tests.test_edge_cases.test_local_empty_hostname@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 643,
        "message_redacted": true,
        "time": 0.188
      },
      "name": "tests.test_edge_cases.test_local_nonexistent_hostname@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 3796,
        "message_redacted": true,
        "time": 0.194
      },
      "name": "tests.test_edge_cases.test_local_255_char_hostname@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 615,
        "message_redacted": true,
        "time": 0.209
      },
      "name": "tests.test_edge_cases.test_local_unicode_hostname@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 672,
        "message_redacted": true,
        "time": 0.198
      },
      "name": "tests.test_edge_cases.test_local_to_arg_with_spaces@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 668,
        "message_redacted": true,
        "time": 0.199
      },
      "name": "tests.test_edge_cases.test_local_to_arg_with_tab@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 85,
        "message_redacted": true,
        "time": 11.886
      },
      "name": "tests.test_harvest.test_mismatched_secret[None-my secret]@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 119,
        "message_redacted": true,
        "time": 15.028
      },
      "name": "tests.test_harvest.test_invalid_address@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 120,
        "message_redacted": true,
        "time": 1.38
      },
      "name": "tests.test_harvest.test_very_long_frame@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 459,
        "message_redacted": true,
        "time": 2.832
      },
      "name": "tests.test_integration.test_client_wrong_secret_to_protected_server@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 459,
        "message_redacted": true,
        "time": 2.852
      },
      "name": "tests.test_integration.test_authentication_mismatched_secrets@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 199,
        "message_redacted": true,
        "time": 1.534
      },
      "name": "tests.test_protocol.test_oversized_frame_rejected@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 85,
        "message_redacted": true,
        "time": 1.514
      },
      "name": "tests.test_protocol.test_wrong_authentication_rejected@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 23,
        "message_redacted": true,
        "time": 19.535
      },
      "name": "tests.test_protocol.test_timeout_waiting_for_authentication@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 23,
        "message_redacted": true,
        "time": 19.531
      },
      "name": "tests.test_protocol.test_timeout_waiting_for_hello@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 23,
        "message_redacted": true,
        "time": 7.543
      },
      "name": "tests.test_protocol.test_heartbeat_messages_sent@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 178,
        "message_redacted": true,
        "time": 1.537
      },
      "name": "tests.test_protocol.test_message_exactly_at_max_length@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 262,
        "message_redacted": true,
        "time": 1.517
      },
      "name": "tests.test_protocol.test_negative_port_number@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 190,
        "message_redacted": true,
        "time": 1.518
      },
      "name": "tests.test_protocol.test_connection_with_unknown_message_type@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 261,
        "message_redacted": true,
        "time": 1.516
      },
      "name": "tests.test_protocol.test_very_large_port_number@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 286,
        "message_redacted": true,
        "time": 0.174
      },
      "name": "tests.test_server.test_server_negative_port_error@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 23,
        "message_redacted": true,
        "time": 9.031
      },
      "name": "tests.test_server_gaps.test_server_sends_heartbeat_after_hello@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 126,
        "message_redacted": true,
        "time": 3.022
      },
      "name": "tests.test_server_gaps.test_server_handles_unexpected_authenticate_message@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 23,
        "message_redacted": true,
        "time": 9.033
      },
      "name": "tests.test_server_gaps.test_server_sends_connection_message_on_incoming_tunnel@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 23,
        "message_redacted": true,
        "time": 9.031
      },
      "name": "tests.test_server_gaps.test_server_accept_message_initiates_forwarding@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 23,
        "message_redacted": true,
        "time": 9.031
      },
      "name": "tests.test_server_gaps.test_server_multiple_incoming_connections_get_unique_uuids@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 23,
        "message_redacted": true,
        "time": 9.032
      },
      "name": "tests.test_server_gaps.test_server_stale_connection_cleanup@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 82,
        "message_redacted": true,
        "time": 4.541
      },
      "name": "tests.test_server_gaps.test_server_handles_random_port_allocation_exhaustion@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 23,
        "message_redacted": true,
        "time": 9.033
      },
      "name": "tests.test_server_gaps.test_server_connection_message_timing@server_tests",
      "status": "failure"
    },
    {
      "branch": "8f4b78a7e9eb",
      "extra": {
        "message_chars": 23,
        "message_redacted": true,
        "time": 9.035
      },
      "name": "tests.test_server_gaps.test_server_heartbeat_continues_after_tunnel_disconnect@server_tests",
      "status": "failure"
    },
    {
      "branch": "e454759f15a4",
      "extra": {
        "message_chars": 469,
        "message_redacted": true,
        "time": 0.161
      },
      "name": "eval.tests.test_args_parsing.test_unexpected_args[args0-needles0]",
      "status": "failure"
    },
    {
      "branch": "e454759f15a4",
      "extra": {
        "message_chars": 539,
        "message_redacted": true,
        "time": 0.154
      },
      "name": "eval.tests.test_args_parsing.test_local_missing_option_values[args3-needles3]",
      "status": "failure"
    },
    {
      "branch": "c977eb4a4e02",
      "extra": {
        "message_chars": 188,
        "message_redacted": true,
        "time": 0.33
      },
      "name": "eval.tests.test_cli_io.test_local_invalid_host_name_exit_1_and_shows_name_or_service_not_known",
      "status": "failure"
    },
    {
      "branch": "183567ae769e",
      "extra": {
        "message_chars": 127,
        "message_redacted": true,
        "time": 15.023
      },
      "name": "eval.tests.test_bore.TestEdgeCases.test_local_short_form_options",
      "status": "failure"
    },
    {
      "branch": "183567ae769e",
      "extra": {
        "message_chars": 130,
        "message_redacted": true,
        "time": 15.026
      },
      "name": "eval.tests.test_bore.TestEdgeCases.test_local_very_high_port",
      "status": "failure"
    }
  ],
  "test_results_omitted": 0,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}