{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "e0792b4c44e22245510c2dec0acd457b369025c66cbe62c1cc86fa84272a823a",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.15375427799881436
    },
    {
      "branch": "",
      "command": "find /workspace -type f -exec sha256sum {} + 2>/dev/null | grep -E \"^(30fe43c188f080026577d2fa935ebf33f0b902077bb2138724f839680eb45944)  \" | cut -c67- | xargs -I% rm -fv %",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.132699698995566
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.15363886900013313
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.12839953300135676
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.12395554999238811
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.14152712900249753
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6658387990028132
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.09478627600765321
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.10857485799351707
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11479960198630579
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.110061568004312
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 40283,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 4.9211459109938005
    },
    {
      "branch": "547860b2846e",
      "command": "docker cp 0c877f8fdf88b60b48ddb64c3592939af9e27caf49f8527aa41e0d12d05990be:/workspace/eval/results.xml /tmp/tmpnbvcakyh.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 38237,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03877060700324364
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.07958077400689945
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12671726400731131
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11616732500260696
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1307392210001126
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 411822,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 48.53705018000619
    },
    {
      "branch": "9cd5a99e237f",
      "command": "docker cp d064ba401cd54839e382de8f7b248494a91ac288da610936e64a6a807ade2cf0:/workspace/eval/results.xml /tmp/tmp7sog1207.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 429789,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.0447298640065128
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.15354933899652679
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11517452000407502
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12609403699752875
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1320296630001394
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 26029,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 4.600884105006116
    },
    {
      "branch": "6538f3c3ff0d",
      "command": "docker cp 284ac62170c26b69562be7f5b756ebad430e2b5c46be744c6563207dacf05ec7:/workspace/eval/results.xml /tmp/tmp0xtjk4rm.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 18707,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.024594788992544636
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11125883299973793
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12977399599913042
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13653453699953388
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13737606300855987
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 16379,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 7.9003648049983894
    },
    {
      "branch": "52fd780a87d4",
      "command": "docker cp b2ec162d83220597254947d27a7ee468303dac231a2eb8fbde42211ab125b341:/workspace/eval/results.xml /tmp/tmppqlilnf_.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 10608,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.0361816320073558
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1297059869975783
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1179999120067805
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.08850801301014144
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.09890390301006846
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2609,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.5482299129944295
    },
    {
      "branch": "6e1109189421",
      "command": "docker cp 3e41a1c5ba9965d84a81c66c2893a31b379190d98e92a211f9bfdc4a5f65b333:/workspace/eval/results.xml /tmp/tmpn1tvp3gl.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1190,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.041558641009032726
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14690153700939845
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13099957999656908
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12104657499003224
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12704377199406736
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 8426,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 4.5834147529967595
    },
    {
      "branch": "f583a5a1b4d7",
      "command": "docker cp c7d54de4c6d06f5167fb7d6ae75ce5881627ff3b60c4c880d8951911ef2d5064:/workspace/eval/results.xml /tmp/tmpxwn401pl.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5085,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.04002996000053827
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10779899200133514
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11945773499610368
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13869752499158494
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1308549200039124
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 28570,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 5.605282287011505
    },
    {
      "branch": "3cd87e647073",
      "command": "docker cp 942bd1f804c21d057e38a8ee269a49bc8885a8b443d7c6d993d55bf46fd5eb14:/workspace/eval/results.xml /tmp/tmpzhi7hi3s.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 24377,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03673727999557741
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11285067300195806
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12259963800897822
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11653108299651649
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12663339400023688
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 6773,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 6.034189860991319
    },
    {
      "branch": "bb21c29e365f",
      "command": "docker cp b358fa9d1ac1e0c41b6c6e66aedcdf450353ddd9b26019586be5d865d1cf25e4:/workspace/eval/results.xml /tmp/tmp9zsiodq6.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4509,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.036789240009966306
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1257885789964348
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12472561500908341
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1287145729875192
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12957230699248612
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 9239,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.670830836999812
    },
    {
      "branch": "32f6d9a86e53",
      "command": "docker cp a881d092518345bcf195aea4bf4b13ce742952c0dec851ff5828dfcc1f8b290e:/workspace/eval/results.xml /tmp/tmp9kf0jins.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5954,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.035366657000849955
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13860591500997543
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13184084500244353
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11926800700894091
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12989901599939913
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3240,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.665660985992872
    },
    {
      "branch": "549ea83da9dc",
      "command": "docker cp ed317a340a85d97a2c1f51ec510caced19b2bfe1418d3d831b176de0aff7dfc3:/workspace/eval/results.xml /tmp/tmp7lb1gv33.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1564,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.036909079004544765
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 334,
    "passed": 553
  },
  "test_branch_errors": {},
  "test_branches": [
    "547860b2846e",
    "9cd5a99e237f",
    "6538f3c3ff0d",
    "52fd780a87d4",
    "6e1109189421",
    "f583a5a1b4d7",
    "3cd87e647073",
    "bb21c29e365f",
    "32f6d9a86e53",
    "549ea83da9dc"
  ],
  "test_records": 887,
  "test_results": [
    {
      "branch": "547860b2846e",
      "extra": {
        "message_chars": 34,
        "message_redacted": true,
        "time": 0.243
      },
      "name": "eval.tests.test_actual_rendering.test_render_with_cover_image",
      "status": "failure"
    },
    {
      "branch": "547860b2846e",
      "extra": {
        "message_chars": 297,
        "message_redacted": true,
        "time": 0.234
      },
      "name": "eval.tests.test_actual_rendering.test_render_to_stdout",
      "status": "failure"
    },
    {
      "branch": "547860b2846e",
      "extra": {
        "message_chars": 125,
        "message_redacted": true,
        "time": 0.179
      },
      "name": "eval.tests.test_create_book.test_create_book_with_filename",
      "status": "failure"
    },
    {
      "branch": "547860b2846e",
      "extra": {
        "message_chars": 2193,
        "message_redacted": true,
        "time": 0.181
      },
      "name": "eval.tests.test_create_book.test_create_book_file_already_exists",
      "status": "failure"
    },
    {
      "branch": "547860b2846e",
      "extra": {
        "message_chars": 539,
        "message_redacted": true,
        "time": 0.19
      },
      "name": "eval.tests.test_output_formats.test_html_dir_format",
      "status": "failure"
    },
    {
      "branch": "547860b2846e",
      "extra": {
        "message_chars": 3890,
        "message_redacted": true,
        "time": 0.398
      },
      "name": "eval.tests.test_print_template.test_print_template_multiple_types",
      "status": "failure"
    },
    {
      "branch": "547860b2846e",
      "extra": {
        "message_chars": 301,
        "message_redacted": true,
        "time": 0.179
      },
      "name": "eval.tests.test_single_file_rendering.test_single_to_stdout",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 556,
        "message_redacted": true,
        "time": 0.187
      },
      "name": "tests.test_cli.test_stdin_single_mode",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 90,
        "message_redacted": true,
        "time": 0.18
      },
      "name": "tests.test_cli.test_verbose_flag_shows_debug_output",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 207,
        "message_redacted": true,
        "time": 0.203
      },
      "name": "tests.test_book.test_option_type_error_int_as_string",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 1291,
        "message_redacted": true,
        "time": 0.161
      },
      "name": "tests.test_book.test_option_float_invalid_string",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 209,
        "message_redacted": true,
        "time": 0.194
      },
      "name": "tests.test_cli.test_output_formats_html_dir",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 713,
        "message_redacted": true,
        "time": 0.203
      },
      "name": "tests.test_book.test_option_strvec_invalid_element_type",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 2416,
        "message_redacted": true,
        "time": 0.179
      },
      "name": "tests.test_book.test_rendering_chapter_template_option",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 140,
        "message_redacted": true,
        "time": 0.182
      },
      "name": "tests.test_book.test_book_file_with_yaml_style_multiline_string",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 4846,
        "message_redacted": true,
        "time": 0.181
      },
      "name": "tests.test_book.test_option_resources_base_path_templates",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 73,
        "message_redacted": true,
        "time": 0.197
      },
      "name": "tests.test_epub.test_cover_image_handling",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 384,
        "message_redacted": true,
        "time": 0.146
      },
      "name": "tests.test_cli.test_set_invalid_option_error",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 91,
        "message_redacted": true,
        "time": 0.158
      },
      "name": "tests.test_epub.test_epub_version_3_explicit",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 64,
        "message_redacted": true,
        "time": 0.169
      },
      "name": "tests.test_book.test_additional_metadata_fields",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 134,
        "message_redacted": true,
        "time": 0.146
      },
      "name": "tests.test_book.test_part_syntax_at_sign",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 140,
        "message_redacted": true,
        "time": 0.147
      },
      "name": "tests.test_book.test_specific_chapter_number_syntax",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 799,
        "message_redacted": true,
        "time": 0.15
      },
      "name": "tests.test_epub.test_toc_extras_option",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 544,
        "message_redacted": true,
        "time": 0.18
      },
      "name": "tests.test_cli.test_stdin_book_mode",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 1291,
        "message_redacted": true,
        "time": 0.178
      },
      "name": "tests.test_book.test_option_float_wrong_yaml_type",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 168,
        "message_redacted": true,
        "time": 0.185
      },
      "name": "tests.test_cli.test_no_fancy_flag_disables_ui",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 207,
        "message_redacted": true,
        "time": 0.174
      },
      "name": "tests.test_book.test_option_strvec_not_array",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 207,
        "message_redacted": true,
        "time": 0.2
      },
      "name": "tests.test_book.test_option_type_error_array_as_string",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 235,
        "message_redacted": true,
        "time": 0.199
      },
      "name": "tests.test_cli.test_stats_flag_shows_word_count",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 2490,
        "message_redacted": true,
        "time": 0.179
      },
      "name": "tests.test_cli.test_autograph_mode_requires_stdin",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 70,
        "message_redacted": true,
        "time": 0.189
      },
      "name": "tests.test_epub.test_images_in_content",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 65,
        "message_redacted": true,
        "time": 0.169
      },
      "name": "tests.test_book.test_option_multiple_imports_chain",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 58,
        "message_redacted": true,
        "time": 0.158
      },
      "name": "tests.test_epub.test_metadata_in_opf",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 140,
        "message_redacted": true,
        "time": 0.148
      },
      "name": "tests.test_book.test_subchapter_double_dash_syntax",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 81,
        "message_redacted": true,
        "time": 0.166
      },
      "name": "tests.test_book.test_rendering_inline_toc_option",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 722,
        "message_redacted": true,
        "time": 0.145
      },
      "name": "tests.test_book.test_tex_options_paper_size",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 137,
        "message_redacted": true,
        "time": 0.165
      },
      "name": "tests.test_book.test_unnumbered_part_syntax",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 716,
        "message_redacted": true,
        "time": 0.18
      },
      "name": "tests.test_errors.test_multiple_error_types_in_same_book",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 235,
        "message_redacted": true,
        "time": 0.187
      },
      "name": "tests.test_errors.test_template_not_found_error_format",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 237,
        "message_redacted": true,
        "time": 0.194
      },
      "name": "tests.test_cli.test_verbose_quiet_conflict",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 713,
        "message_redacted": true,
        "time": 0.225
      },
      "name": "tests.test_book.test_option_strvec_invalid_output_format",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 207,
        "message_redacted": true,
        "time": 0.197
      },
      "name": "tests.test_book.test_option_type_error_string_as_int",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 303,
        "message_redacted": true,
        "time": 0.202
      },
      "name": "tests.test_cli.test_multiple_set_flags_error",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 577,
        "message_redacted": true,
        "time": 0.183
      },
      "name": "tests.test_cli.test_create_book_with_set_options",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 921,
        "message_redacted": true,
        "time": 0.203
      },
      "name": "tests.test_epub.test_footnote_handling_epub2",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 203,
        "message_redacted": true,
        "time": 0.171
      },
      "name": "tests.test_errors.test_output_directory_not_found",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 2421,
        "message_redacted": true,
        "time": 0.17
      },
      "name": "tests.test_book.test_inline_toc_name_option",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 132,
        "message_redacted": true,
        "time": 0.145
      },
      "name": "tests.test_epub.test_custom_css_addition",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 451,
        "message_redacted": true,
        "time": 0.17
      },
      "name": "tests.test_errors.test_malformed_yaml_configuration",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 2113,
        "message_redacted": true,
        "time": 0.171
      },
      "name": "tests.test_html_dir.test_html_dir_error_when_output_is_file_not_directory",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 68,
        "message_redacted": true,
        "time": 0.204
      },
      "name": "tests.test_html_dir.test_html_dir_special_characters_in_titles",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 72,
        "message_redacted": true,
        "time": 0.171
      },
      "name": "tests.test_book.test_rendering_roman_numerals_option",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 731,
        "message_redacted": true,
        "time": 0.15
      },
      "name": "tests.test_errors.test_invalid_book_option_error_formatting",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 484,
        "message_redacted": true,
        "time": 0.17
      },
      "name": "tests.test_errors.test_invalid_yaml_structure_error",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 768,
        "message_redacted": true,
        "time": 0.214
      },
      "name": "tests.test_errors.test_epub_zipper_error_format",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 207,
        "message_redacted": true,
        "time": 0.209
      },
      "name": "tests.test_book.test_option_type_error_bool_as_int",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 207,
        "message_redacted": true,
        "time": 0.216
      },
      "name": "tests.test_book.test_option_import_nonexistent_file",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 454,
        "message_redacted": true,
        "time": 0.167
      },
      "name": "tests.test_cli.test_output_to_stdout",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 159,
        "message_redacted": true,
        "time": 0.225
      },
      "name": "tests.test_errors.test_missing_image_in_markdown",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 147,
        "message_redacted": true,
        "time": 0.216
      },
      "name": "tests.test_html_dir.test_html_dir_metadata_in_index_page",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 857,
        "message_redacted": true,
        "time": 0.193
      },
      "name": "tests.test_epub.test_footnote_handling_epub3",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 23,
        "message_redacted": true,
        "time": 0.242
      },
      "name": "tests.test_html_dir.test_html_dir_prev_next_chapter_links",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 380,
        "message_redacted": true,
        "time": 0.175
      },
      "name": "tests.test_errors.test_empty_book_file",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 245,
        "message_redacted": true,
        "time": 0.196
      },
      "name": "tests.test_errors.test_invalid_boolean_option_value",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 228,
        "message_redacted": true,
        "time": 0.18
      },
      "name": "tests.test_epub.test_missing_cover_error",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 236,
        "message_redacted": true,
        "time": 0.174
      },
      "name": "tests.test_errors.test_book_option_type_error_formatting",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 144,
        "message_redacted": true,
        "time": 0.169
      },
      "name": "tests.test_html_dir.test_html_dir_menu_svg_is_valid",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 200,
        "message_redacted": true,
        "time": 0.343
      },
      "name": "tests.test_html_dir.test_html_dir_cleans_previous_output",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 146,
        "message_redacted": true,
        "time": 0.176
      },
      "name": "tests.test_html_dir.test_html_dir_favicon_on_index_page",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 646,
        "message_redacted": true,
        "time": 0.189
      },
      "name": "tests.test_errors.test_render_error_epub_missing_cover",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 146,
        "message_redacted": true,
        "time": 0.202
      },
      "name": "tests.test_html_dir.test_html_dir_creates_required_files",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 207,
        "message_redacted": true,
        "time": 0.225
      },
      "name": "tests.test_book.test_option_type_error_string_as_bool",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 67,
        "message_redacted": true,
        "time": 0.212
      },
      "name": "tests.test_book.test_option_import_merges_settings",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 147,
        "message_redacted": true,
        "time": 0.205
      },
      "name": "tests.test_html_dir.test_html_dir_inline_toc_on_index_when_enabled",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 692,
        "message_redacted": true,
        "time": 0.227
      },
      "name": "tests.test_errors.test_missing_cover_image",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 147,
        "message_redacted": true,
        "time": 0.178
      },
      "name": "tests.test_html_dir.test_html_dir_viewport_meta_for_mobile",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 232,
        "message_redacted": true,
        "time": 0.16
      },
      "name": "tests.test_errors.test_template_error_formatting",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 303,
        "message_redacted": true,
        "time": 0.199
      },
      "name": "tests.test_errors.test_invalid_integer_option_value",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 182,
        "message_redacted": true,
        "time": 0.175
      },
      "name": "tests.test_html_dir.test_html_dir_images_directory_created",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 285,
        "message_redacted": true,
        "time": 0.182
      },
      "name": "tests.test_errors.test_verbose_mode_shows_debug_info",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 23,
        "message_redacted": true,
        "time": 0.151
      },
      "name": "tests.test_html_dir.test_html_dir_print_css_separate_file",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 147,
        "message_redacted": true,
        "time": 0.151
      },
      "name": "tests.test_html_dir.test_html_dir_valid_html5_structure",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 68,
        "message_redacted": true,
        "time": 0.158
      },
      "name": "tests.test_epub.test_guide_section_in_opf",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 146,
        "message_redacted": true,
        "time": 0.191
      },
      "name": "tests.test_html_dir.test_html_dir_author_title_subtitle_on_index",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 236,
        "message_redacted": true,
        "time": 0.196
      },
      "name": "tests.test_html_single.test_markdown_headings_h1_to_h3",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 75,
        "message_redacted": true,
        "time": 0.158
      },
      "name": "tests.test_html_dir.test_html_dir_favicon_link_in_chapter_pages",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 146,
        "message_redacted": true,
        "time": 0.178
      },
      "name": "tests.test_html_dir.test_html_dir_index_contains_toc_with_all_chapters",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 993,
        "message_redacted": true,
        "time": 0.208
      },
      "name": "tests.test_book.test_option_custom_metadata_fields",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 673,
        "message_redacted": true,
        "time": 0.197
      },
      "name": "tests.test_book.test_deprecated_option_with_alias",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 644,
        "message_redacted": true,
        "time": 0.159
      },
      "name": "tests.test_errors.test_zipper_error_epub_creation",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 147,
        "message_redacted": true,
        "time": 0.168
      },
      "name": "tests.test_html_dir.test_html_dir_no_inline_toc_when_disabled",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 176,
        "message_redacted": true,
        "time": 0.164
      },
      "name": "tests.test_errors.test_conflicting_quiet_verbose_flags",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 283,
        "message_redacted": true,
        "time": 0.191
      },
      "name": "tests.test_errors.test_output_to_stdout_with_dash",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 700,
        "message_redacted": true,
        "time": 0.173
      },
      "name": "tests.test_errors.test_invalid_yaml_in_book_config",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 46,
        "message_redacted": true,
        "time": 0.178
      },
      "name": "tests.test_html_dir.test_html_dir_chapter_content_matches_markdown",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 196,
        "message_redacted": true,
        "time": 0.299
      },
      "name": "tests.test_html_dir.test_html_dir_deletes_existing_directory",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 146,
        "message_redacted": true,
        "time": 0.199
      },
      "name": "tests.test_html_dir.test_html_dir_index_next_link_to_first_chapter",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 133,
        "message_redacted": true,
        "time": 0.178
      },
      "name": "tests.test_html_dir.test_html_dir_multiple_chapters_with_sections",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 146,
        "message_redacted": true,
        "time": 0.175
      },
      "name": "tests.test_html_dir.test_html_dir_is_chapter_flag_in_template_data",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 202,
        "message_redacted": true,
        "time": 0.207
      },
      "name": "tests.test_html_dir.test_html_dir_nested_resource_files_preserve_structure",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 23,
        "message_redacted": true,
        "time": 0.212
      },
      "name": "tests.test_html_dir.test_html_dir_first_chapter_has_only_next_link",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 23,
        "message_redacted": true,
        "time": 0.168
      },
      "name": "tests.test_html_dir.test_html_dir_chapter_files_have_navigation_menu",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 217,
        "message_redacted": true,
        "time": 0.335
      },
      "name": "tests.test_html_single.test_long_document_performance",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 669,
        "message_redacted": true,
        "time": 0.218
      },
      "name": "tests.test_book.test_deprecated_option_removed_no_replacement",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 734,
        "message_redacted": true,
        "time": 0.18
      },
      "name": "tests.test_book.test_option_custom_metadata_invalid_type",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 230,
        "message_redacted": true,
        "time": 0.198
      },
      "name": "tests.test_errors.test_render_error_invalid_html_template_var",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 458,
        "message_redacted": true,
        "time": 0.203
      },
      "name": "tests.test_errors.test_set_invalid_option_key",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 925,
        "message_redacted": true,
        "time": 0.177
      },
      "name": "tests.test_errors.test_create_mode_with_multiple_chapters",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 256,
        "message_redacted": true,
        "time": 0.198
      },
      "name": "tests.test_errors.test_missing_css_file_in_config",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 2060,
        "message_redacted": true,
        "time": 0.197
      },
      "name": "tests.test_html_dir.test_html_dir_error_when_output_path_is_file",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 133,
        "message_redacted": true,
        "time": 0.189
      },
      "name": "tests.test_html_dir.test_html_dir_anchor_ids_on_headings",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 35,
        "message_redacted": true,
        "time": 0.203
      },
      "name": "tests.test_html_dir.test_html_dir_image_references_copied_and_linked",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 2443,
        "message_redacted": true,
        "time": 0.169
      },
      "name": "tests.test_html_dir.test_html_dir_javascript_toggle_function_present",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 188,
        "message_redacted": true,
        "time": 0.145
      },
      "name": "tests.test_html_dir.test_html_dir_highlight_js_files_included",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 705,
        "message_redacted": true,
        "time": 0.177
      },
      "name": "tests.test_latex.test_tex_font_size_setting",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 183,
        "message_redacted": true,
        "time": 0.221
      },
      "name": "tests.test_html_dir.test_html_dir_menu_svg_always_included",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 35,
        "message_redacted": true,
        "time": 0.193
      },
      "name": "tests.test_html_single.test_markdown_table_rendering",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 23,
        "message_redacted": true,
        "time": 0.207
      },
      "name": "tests.test_html_dir.test_html_dir_last_chapter_has_only_prev_link",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 800,
        "message_redacted": true,
        "time": 0.155
      },
      "name": "tests.test_latex.test_article_class_uses_section_not_chapter",
      "status": "failure"
    },
    {
      "branch": "9cd5a99e237f",
      "extra": {
        "message_chars": 233,
        "message_redacted": true,
        "time": 0.154
      },
      "name": "tests.test_errors.test_file_not_found_with_description",
      "status": "failure"
    }
  ],
  "test_results_omitted": 214,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}