{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "e5eb568d2968aaa8c990c135ac3aa119a6761abc0c5a69f3b638cb000ab24f28",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.16106578899780288
    },
    {
      "branch": "",
      "command": "find /workspace -type f -exec sha256sum {} + 2>/dev/null | grep -E \"^(45d4c1e593b02ebef000be4d4918c32dc5f20b6b2b69f5c7f13ba7d222d1a24a)  \" | cut -c67- | xargs -I% rm -fv %",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.12448085899814032
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.15226076800900046
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.14178007800364867
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.11979998300375883
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.1327893679990666
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6668971530016279
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10069372800353449
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12412482500076294
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.09631918699597009
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.09126504500454757
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 91559,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 6.609135894992505
    },
    {
      "branch": "3487890d9158",
      "command": "docker cp 70e98b23ac19186221444d2acd58ea38a8bb21fb4cf7379ac15310685f1b9748:/workspace/eval/results.xml /tmp/tmpmmhnp2q9.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 129116,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03964458900736645
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12839747799444012
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.08260588100529276
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1406831430067541
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11483979799959343
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 29573,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 4.571964367001783
    },
    {
      "branch": "f2d92adaccd5",
      "command": "docker cp e14d19c709e0b1dcc90c8674e87e102b755c2cb32156196540f046f54d901bcd:/workspace/eval/results.xml /tmp/tmpe01690lq.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 33907,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.04006969199690502
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13275125800282694
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12863749499956612
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.08712075099174399
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13327768199087586
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 107931,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 13.296143336003297
    },
    {
      "branch": "b112ab0c5f2b",
      "command": "docker cp 7ed6326af948358bd6d2d147e40d155fdd4a7cd47d5c10b11a70cff9c91d6da1:/workspace/eval/results.xml /tmp/tmpho21x3hh.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 114199,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.040509407001081854
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1335251279961085
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12566737399902195
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13157263401080854
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1374358159955591
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 6662,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.6738729429926025
    },
    {
      "branch": "3a4310d7442e",
      "command": "docker cp af88cd1abad3f977dc92ed88435f300997dd73c6d25c1f8d211c05bda5983f12:/workspace/eval/results.xml /tmp/tmpmno973e8.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3660,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03882488000090234
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12942476299940608
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12497534300200641
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12706994499603752
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1323669039993547
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1187,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.499389192991657
    },
    {
      "branch": "ada7b98489d3",
      "command": "docker cp 0df119a9c86fea249c3d14ff777f101b30592ceecf2799f54e7ccea48a876002:/workspace/eval/results.xml /tmp/tmp4m5gf2_e.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 599,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.037134461003006436
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14082708899513818
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.10437046999868471
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13773489199229516
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1029090389929479
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 8202,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.661571539007127
    },
    {
      "branch": "a1bf006807df",
      "command": "docker cp 3cceea55328bfa663504a665e56f4097a259f3723f413cf4f6b660861824cc33:/workspace/eval/results.xml /tmp/tmpfqm55pjc.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4777,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.038752950000343844
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10746083900448866
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12521777900110465
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12265428400132805
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13952620700001717
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2889,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.4947287249960937
    },
    {
      "branch": "44654f978932",
      "command": "docker cp 939ed06c811d9bd6acdbe583d302d6bc8083ea0ed4df3f5dfc02d4f1ec18bc07:/workspace/eval/results.xml /tmp/tmp_qoklfen.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1359,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03743892700003926
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10329820399056189
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11265414800436702
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1204281239915872
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11748064300627448
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 21326,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 4.521900190011365
    },
    {
      "branch": "deed8607e050",
      "command": "docker cp d451ff1f6cbf8ad6ed7653addac26d37099effe1b755a16ea3ba4d1fda332b01:/workspace/eval/results.xml /tmp/tmp3b9f01c5.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 15785,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03930041300191078
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12798244200530462
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11577085699536838
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11862402799306437
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12926124499063008
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 10031,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.64368048698816
    },
    {
      "branch": "cddd1439473b",
      "command": "docker cp 9373a12caafcf92b8d1720efa7dd7a79aa96c775c3c6ad985b6f3a2af88d4e21:/workspace/eval/results.xml /tmp/tmp71b77mil.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 8624,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.028382877993863076
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12679238899727352
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13608282299537677
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.09104449799633585
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13352594900061376
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4988,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.695823753994773
    },
    {
      "branch": "3d5b05faf9e6",
      "command": "docker cp ce43e046ab4fe6c9fdccc184519c73d74834ce809199a8a04174b3fd5c9267c9:/workspace/eval/results.xml /tmp/tmpbj1av5d2.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2896,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.0388920379919
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 95,
    "passed": 461
  },
  "test_branch_errors": {},
  "test_branches": [
    "3487890d9158",
    "f2d92adaccd5",
    "b112ab0c5f2b",
    "3a4310d7442e",
    "ada7b98489d3",
    "a1bf006807df",
    "44654f978932",
    "deed8607e050",
    "cddd1439473b",
    "3d5b05faf9e6"
  ],
  "test_records": 556,
  "test_results": [
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 117,
        "message_redacted": true,
        "time": 0.202
      },
      "name": "tests.test_basic_invocation.test_version_flag_equivalence",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.109
      },
      "name": "tests.test_additional_coverage.test_package_with_build_error",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.142
      },
      "name": "tests.test_display_options.test_failed_tests_always_shown",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 1770,
        "message_redacted": true,
        "time": 0.138
      },
      "name": "tests.test_basic_invocation.test_version_flag_version",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.083
      },
      "name": "tests.test_edge_cases.test_multiple_packages_mixed_results",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.105
      },
      "name": "tests.test_failure_details.test_failure_with_stack_trace",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 7254,
        "message_redacted": true,
        "time": 0.133
      },
      "name": "tests.test_follow_mode.test_follow_flag_prints_raw_output",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.133
      },
      "name": "tests.test_failure_details.test_failure_details_shown",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 34,
        "message_redacted": true,
        "time": 0.115
      },
      "name": "tests.test_follow_mode.test_follow_output_writes_to_file",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.14
      },
      "name": "tests.test_failure_details.test_subtest_failure_shown",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.112
      },
      "name": "tests.test_package_summary.test_package_summary_failed_package",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.108
      },
      "name": "tests.test_panic_handling.test_multiple_panics",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.127
      },
      "name": "tests.test_panic_handling.test_panic_detection",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.108
      },
      "name": "tests.test_progress_mode.test_progress_with_failed_package",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.079
      },
      "name": "tests.test_panic_handling.test_panic_with_stack_trace",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.085
      },
      "name": "tests.test_real_testdata.test_failed_test_01",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.132
      },
      "name": "tests.test_real_testdata.test_race_testdata",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.121
      },
      "name": "tests.test_failure_details.test_failure_output_formatting",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.131
      },
      "name": "tests.test_failure_details.test_multiple_failures_shown",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.123
      },
      "name": "tests.test_real_testdata.test_panic_testdata",
      "status": "failure"
    },
    {
      "branch": "3487890d9158",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.18
      },
      "name": "tests.test_real_testdata.test_cover_testdata",
      "status": "failure"
    },
    {
      "branch": "f2d92adaccd5",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.206
      },
      "name": "eval.tests.test_edge_cases.test_skip_with_coverage_data",
      "status": "failure"
    },
    {
      "branch": "f2d92adaccd5",
      "extra": {
        "message_chars": 1770,
        "message_redacted": true,
        "time": 0.113
      },
      "name": "eval.tests.test_basic_invocation.test_version_flag_long",
      "status": "failure"
    },
    {
      "branch": "f2d92adaccd5",
      "extra": {
        "message_chars": 1855,
        "message_redacted": true,
        "time": 0.125
      },
      "name": "eval.tests.test_follow_mode.test_follow_verbose_flag",
      "status": "failure"
    },
    {
      "branch": "f2d92adaccd5",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.174
      },
      "name": "eval.tests.test_parsing.test_parse_coverage_data",
      "status": "failure"
    },
    {
      "branch": "f2d92adaccd5",
      "extra": {
        "message_chars": 2455,
        "message_redacted": true,
        "time": 0.135
      },
      "name": "eval.tests.test_summary_table.test_summary_coverage_column",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 590,
        "message_redacted": true,
        "time": 0.111
      },
      "name": "tests.test_follow.test_follow_with_timestamp",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 386,
        "message_redacted": true,
        "time": 0.142
      },
      "name": "tests.test_follow.test_follow_verbose_without_noisy_output_filtered",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 511,
        "message_redacted": true,
        "time": 0.151
      },
      "name": "tests.test_follow.test_progress_mode_basic",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 157,
        "message_redacted": true,
        "time": 0.137
      },
      "name": "tests.test_follow.test_follow_mode_without_file_flag_requires_pipe",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 84,
        "message_redacted": true,
        "time": 0.105
      },
      "name": "tests.test_follow.test_follow_output_file_error_handling",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 1750,
        "message_redacted": true,
        "time": 0.097
      },
      "name": "tests.test_follow.test_follow_verbose_with_parallel_subtests",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 452,
        "message_redacted": true,
        "time": 0.107
      },
      "name": "tests.test_follow.test_follow_with_timestamp_and_file_output",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 149,
        "message_redacted": true,
        "time": 0.103
      },
      "name": "tests.test_follow.test_follow_timestamp_format_is_rfc3339",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 451,
        "message_redacted": true,
        "time": 0.093
      },
      "name": "tests.test_follow.test_follow_mode_basic",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 293,
        "message_redacted": true,
        "time": 0.075
      },
      "name": "tests.test_display.test_notests_only_package_displayed",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 383,
        "message_redacted": true,
        "time": 0.078
      },
      "name": "tests.test_format.test_format_basic_nocolor_exact_match_against_golden",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 622,
        "message_redacted": true,
        "time": 0.094
      },
      "name": "tests.test_format.test_format_plain_exact_match_against_golden",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 84,
        "message_redacted": true,
        "time": 0.072
      },
      "name": "tests.test_format.test_legacy_noborders_flag_produces_plain_format",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 617,
        "message_redacted": true,
        "time": 0.109
      },
      "name": "tests.test_format.test_plain_format_with_all_flag_shows_test_table",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 638,
        "message_redacted": true,
        "time": 0.108
      },
      "name": "tests.test_format.test_failed_test_basic_format_multiline_output",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 557,
        "message_redacted": true,
        "time": 0.17
      },
      "name": "tests.test_format.test_format_basic_with_no_color_env_and_all_flag",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 131,
        "message_redacted": true,
        "time": 0.135
      },
      "name": "tests.test_follow.test_follow_verbose_mode",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 450,
        "message_redacted": true,
        "time": 0.132
      },
      "name": "tests.test_follow.test_follow_output_to_file",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 1750,
        "message_redacted": true,
        "time": 0.102
      },
      "name": "tests.test_follow.test_follow_verbose_preserves_test_hierarchy",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 622,
        "message_redacted": true,
        "time": 0.138
      },
      "name": "tests.test_format.test_format_basic_exact_match_against_golden",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 383,
        "message_redacted": true,
        "time": 0.088
      },
      "name": "tests.test_format.test_no_color_env_exact_match_against_golden",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 84,
        "message_redacted": true,
        "time": 0.116
      },
      "name": "tests.test_format.test_noborders_with_format_flag_format_takes_precedence",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 323,
        "message_redacted": true,
        "time": 0.131
      },
      "name": "tests.test_format.test_format_markdown_exact_match_against_golden",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 103,
        "message_redacted": true,
        "time": 0.129
      },
      "name": "tests.test_follow.test_follow_mode_with_failed_build",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 336,
        "message_redacted": true,
        "time": 0.135
      },
      "name": "tests.test_follow.test_progress_mode_multiple_packages",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 129,
        "message_redacted": true,
        "time": 0.089
      },
      "name": "tests.test_follow.test_follow_verbose_shows_panic_output",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 439,
        "message_redacted": true,
        "time": 0.129
      },
      "name": "tests.test_format.test_mixed_results_markdown_format",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 602,
        "message_redacted": true,
        "time": 0.114
      },
      "name": "tests.test_format.test_basic_format_with_all_flag_shows_test_table",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 480,
        "message_redacted": true,
        "time": 0.135
      },
      "name": "tests.test_harvest.test_failed_fixtures[test_02-1]",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 333,
        "message_redacted": true,
        "time": 0.127
      },
      "name": "tests.test_format.test_failed_test_markdown_format_code_block",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 557,
        "message_redacted": true,
        "time": 0.124
      },
      "name": "tests.test_format.test_basic_format_nocolor_with_all_flag",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 281,
        "message_redacted": true,
        "time": 0.531
      },
      "name": "tests.test_format.test_markdown_format_with_all_flag_shows_collapsible_details",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 1750,
        "message_redacted": true,
        "time": 0.151
      },
      "name": "tests.test_follow.test_follow_verbose_with_notests",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 451,
        "message_redacted": true,
        "time": 0.124
      },
      "name": "tests.test_follow.test_follow_and_progress_combined",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 397,
        "message_redacted": true,
        "time": 0.137
      },
      "name": "tests.test_harvest.test_follow_verbose_fixtures[test_06-2]",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 1889,
        "message_redacted": true,
        "time": 0.131
      },
      "name": "tests.test_harvest.test_follow_verbose_fixtures[test_05-1]",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 476,
        "message_redacted": true,
        "time": 0.233
      },
      "name": "tests.test_harvest.test_failed_fixtures[test_04-1]",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 260,
        "message_redacted": true,
        "time": 0.08
      },
      "name": "tests.test_harvest.test_follow_fixtures[test_01-0]",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 2496,
        "message_redacted": true,
        "time": 0.098
      },
      "name": "tests.test_harvest.test_race_fixtures[test_03-1]",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 477,
        "message_redacted": true,
        "time": 0.392
      },
      "name": "tests.test_harvest.test_failed_fixtures[test_03-1]",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 1889,
        "message_redacted": true,
        "time": 0.102
      },
      "name": "tests.test_harvest.test_follow_verbose_fixtures[test_02-0]",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 257,
        "message_redacted": true,
        "time": 0.1
      },
      "name": "tests.test_harvest.test_prescan_fixtures[test_01.txt-True]",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 1889,
        "message_redacted": true,
        "time": 0.101
      },
      "name": "tests.test_harvest.test_follow_verbose_fixtures[test_03-0]",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 624,
        "message_redacted": true,
        "time": 0.155
      },
      "name": "tests.test_path.test_trimpath_auto_detects_common_prefix",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 574,
        "message_redacted": true,
        "time": 0.08
      },
      "name": "tests.test_path.test_trimpath_in_pass_test_table",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 477,
        "message_redacted": true,
        "time": 0.995
      },
      "name": "tests.test_harvest.test_failed_fixtures[test_01-1]",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 507,
        "message_redacted": true,
        "time": 0.121
      },
      "name": "tests.test_path.test_trimpath_with_markdown_format",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 1889,
        "message_redacted": true,
        "time": 0.12
      },
      "name": "tests.test_harvest.test_follow_verbose_fixtures[test_01-1]",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 1889,
        "message_redacted": true,
        "time": 0.168
      },
      "name": "tests.test_harvest.test_follow_verbose_fixtures[test_04-0]",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 466,
        "message_redacted": true,
        "time": 0.119
      },
      "name": "tests.test_path.test_trimpath_with_all_flag_mixed_statuses",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 632,
        "message_redacted": true,
        "time": 0.128
      },
      "name": "tests.test_path.test_trimpath_with_plain_format",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 714,
        "message_redacted": true,
        "time": 0.146
      },
      "name": "tests.test_path.test_trimpath_failed_section_shows_full_path",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 483,
        "message_redacted": true,
        "time": 0.099
      },
      "name": "tests.test_path.test_versioned_packages_smallscreen",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 690,
        "message_redacted": true,
        "time": 0.101
      },
      "name": "tests.test_path.test_trimpath_basic_removes_specified_prefix",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 466,
        "message_redacted": true,
        "time": 0.105
      },
      "name": "tests.test_sort.test_sort_by_coverage_descending_packages",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 454,
        "message_redacted": true,
        "time": 0.118
      },
      "name": "tests.test_sort.test_sort_by_elapsed_descending_packages",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 585,
        "message_redacted": true,
        "time": 0.122
      },
      "name": "tests.test_path.test_trimpath_auto_with_nested_hierarchy",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 614,
        "message_redacted": true,
        "time": 0.242
      },
      "name": "tests.test_sort.test_sort_cover_with_real_data",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 708,
        "message_redacted": true,
        "time": 0.119
      },
      "name": "tests.test_sort.test_smallscreen_with_extremely_long_subtest_chain",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 517,
        "message_redacted": true,
        "time": 0.173
      },
      "name": "tests.test_sort.test_sort_elapsed_with_cached_packages",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 708,
        "message_redacted": true,
        "time": 0.158
      },
      "name": "tests.test_sort.test_smallscreen_splits_subtest_names_vertically",
      "status": "failure"
    },
    {
      "branch": "b112ab0c5f2b",
      "extra": {
        "message_chars": 499,
        "message_redacted": true,
        "time": 0.165
      },
      "name": "tests.test_path.test_trimpath_with_smallscreen_flag",
      "status": "failure"
    },
    {
      "branch": "a1bf006807df",
      "extra": {
        "message_chars": 111,
        "message_redacted": true,
        "time": 0.131
      },
      "name": "eval.tests.test_help_usage.test_no_args_requires_pipe_error_message",
      "status": "failure"
    },
    {
      "branch": "a1bf006807df",
      "extra": {
        "message_chars": 111,
        "message_redacted": true,
        "time": 0.189
      },
      "name": "eval.tests.test_help_usage.test_help_precedence_even_with_invalid_flag",
      "status": "failure"
    },
    {
      "branch": "deed8607e050",
      "extra": {
        "message_chars": 1767,
        "message_redacted": true,
        "time": 0.117
      },
      "name": "tests.test_basic_invocation.TestHelpAndVersion.test_version_flag_long",
      "status": "failure"
    },
    {
      "branch": "cddd1439473b",
      "extra": {
        "message_chars": 451,
        "message_redacted": true,
        "time": 0.13
      },
      "name": "eval.tests.test_tparse_golden_outputs.test_golden_follow_basic",
      "status": "failure"
    },
    {
      "branch": "cddd1439473b",
      "extra": {
        "message_chars": 461,
        "message_redacted": true,
        "time": 0.176
      },
      "name": "eval.tests.test_tparse_golden_outputs.test_golden_failed_basic",
      "status": "failure"
    },
    {
      "branch": "cddd1439473b",
      "extra": {
        "message_chars": 754,
        "message_redacted": true,
        "time": 0.169
      },
      "name": "eval.tests.test_tparse_golden_outputs.test_golden_panic_basic",
      "status": "failure"
    },
    {
      "branch": "3d5b05faf9e6",
      "extra": {
        "message_chars": 326,
        "message_redacted": true,
        "time": 0.133
      },
      "name": "eval.tests.test_tparse_externalized.test_ext_prescan_test_01_outputs_anything",
      "status": "failure"
    }
  ],
  "test_results_omitted": 0,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}