{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "4ff66a65d3b3930bcf47b8353bba25330127794e673b00f6dd691c53d26142d1",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.1686466140090488
    },
    {
      "branch": "",
      "command": "find /workspace -type f -exec sha256sum {} + 2>/dev/null | grep -E \"^(8992cdf99f9dc6e1922ae72979042c7dda5b76f37e8c099301c8f8a6f76452e3)  \" | cut -c67- | xargs -I% rm -fv %",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.12088724601198919
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.15258624700072687
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.1445285590016283
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.13283691999095026
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.1299087410006905
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6661141150107142
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1378788110014284
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.131633957993472
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11556619999464601
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13038848599535413
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 39719,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 6.940697019992513
    },
    {
      "branch": "c7d152868a35",
      "command": "docker cp 4f1e03354251a98cbdf2d614950f4aba062f191d74a8e6123ab015fcea8b96c9:/workspace/eval/results.xml /tmp/tmp4ypth309.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 29358,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03849042800720781
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13472510600695387
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11888908498804085
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12287900999945123
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1259715569904074
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 131601,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 19.10066572499636
    },
    {
      "branch": "93267c5d44be",
      "command": "docker cp a3371bd9a72460284feb0dec5a970781585b42faca270d9a5c411aef1eccb0fa:/workspace/eval/results.xml /tmp/tmpt0sn0_26.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 111549,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.040687857996090315
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14185978900059126
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12115699501009658
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12985089600260835
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12646554299863055
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 43392,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 4.886368553998182
    },
    {
      "branch": "ffd7bbf54795",
      "command": "docker cp 8103a1d1fb47f5b229158cfaafc1fc889cf9ea0efcbbce69111c8d73b9bbc583:/workspace/eval/results.xml /tmp/tmp1cmo8l8t.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 25893,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03905071999179199
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12415298499399796
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1293804330052808
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1243010320031317
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12256808701204136
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 10834,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 5.601225520003936
    },
    {
      "branch": "12b2a3e58e6b",
      "command": "docker cp 47f79854ec2935533b084694ccbbac612b5e3b2d4f8ef59bab89cd7f4e030310:/workspace/eval/results.xml /tmp/tmp36cc0poy.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 6427,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.040163956000469625
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11099862698756624
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11565629200777039
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11227425999823026
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13200932698964607
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1965,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.3689655479975045
    },
    {
      "branch": "f88dbdf5549e",
      "command": "docker cp d9f9d9eb3577434e785e93b798667a32c2e11c10d31dbcfbacf67ce77acf7282:/workspace/eval/results.xml /tmp/tmpcqkecpgg.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 752,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03780902799917385
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1353872909967322
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12423325401323382
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1255294479924487
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12087322000297718
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 8809,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.226234093992389
    },
    {
      "branch": "1226dc95b5e2",
      "command": "docker cp f5fff59f5613b1ae258b137254bd27ea436251316d6ed1f57de082052a48a88e:/workspace/eval/results.xml /tmp/tmp81ynko9g.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5165,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.026624311998602934
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1400259580113925
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13143080500594806
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12899424000352155
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12973025899555068
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2095,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.4501212849863805
    },
    {
      "branch": "f84478ba4e51",
      "command": "docker cp ba2414bddeff1ea7400ffe041c2c106c97cc007c0335bca9a81a3afa02b5fbd6:/workspace/eval/results.xml /tmp/tmp9sa5jkz_.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 846,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.038491718005388975
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14843408200249542
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12161430099513382
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1281936410086928
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1325706799980253
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 9408,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.605322735005757
    },
    {
      "branch": "d955d8053fa3",
      "command": "docker cp f51fc4b7c9360ce7e68aba28216769ec9b7408b61e5c1abeeb951dec14173b9b:/workspace/eval/results.xml /tmp/tmpj3mc71z0.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5817,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.038543778005987406
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14372045701020397
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1254675079981098
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13497330799873453
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1288628709880868
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2301,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.5165239319903776
    },
    {
      "branch": "d82ca9192dc0",
      "command": "docker cp ce24e9f47c221765dd5d5a93dd2c6ed489b8cebac5e66cc0b406ea77be9b8cf4:/workspace/eval/results.xml /tmp/tmpqpw94p3j.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 949,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.02485460699244868
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13811358499515336
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.129821479000384
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1361586109997006
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1290266209980473
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4130,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.723152997001307
    },
    {
      "branch": "1bd74ab559f5",
      "command": "docker cp 89251466bd281df10d08b046961d7fcb6837b6c5dfe006f39f6ab66bcb9be9c7:/workspace/eval/results.xml /tmp/tmp74e06y_f.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2527,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03639196901349351
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 110,
    "passed": 668
  },
  "test_branch_errors": {},
  "test_branches": [
    "c7d152868a35",
    "93267c5d44be",
    "ffd7bbf54795",
    "12b2a3e58e6b",
    "f88dbdf5549e",
    "1226dc95b5e2",
    "f84478ba4e51",
    "d955d8053fa3",
    "d82ca9192dc0",
    "1bd74ab559f5"
  ],
  "test_records": 778,
  "test_results": [
    {
      "branch": "c7d152868a35",
      "extra": {
        "message_chars": 236,
        "message_redacted": true,
        "time": 0.278
      },
      "name": "eval.tests.test_clog.test_from_conflicts_with_from_latest_tag",
      "status": "failure"
    },
    {
      "branch": "c7d152868a35",
      "extra": {
        "message_chars": 389,
        "message_redacted": true,
        "time": 0.274
      },
      "name": "eval.tests.test_clog.test_changelog_conflicts_with_outfile",
      "status": "failure"
    },
    {
      "branch": "c7d152868a35",
      "extra": {
        "message_chars": 227,
        "message_redacted": true,
        "time": 0.214
      },
      "name": "eval.tests.test_clog_errors.test_conflict_flags_error",
      "status": "failure"
    },
    {
      "branch": "c7d152868a35",
      "extra": {
        "message_chars": 236,
        "message_redacted": true,
        "time": 0.216
      },
      "name": "eval.tests.test_clog_errors.test_from_and_from_latest_tag_conflict",
      "status": "failure"
    },
    {
      "branch": "c7d152868a35",
      "extra": {
        "message_chars": 206,
        "message_redacted": true,
        "time": 0.182
      },
      "name": "eval.tests.test_clog_toml_heavy.test_toml_with_multiline_strings",
      "status": "failure"
    },
    {
      "branch": "c7d152868a35",
      "extra": {
        "message_chars": 270,
        "message_redacted": true,
        "time": 0.229
      },
      "name": "eval.tests.test_simple_coverage_boost.test_conflicting_flags_error",
      "status": "failure"
    },
    {
      "branch": "c7d152868a35",
      "extra": {
        "message_chars": 387,
        "message_redacted": true,
        "time": 0.339
      },
      "name": "eval.tests.test_clog.test_changelog_conflicts_with_infile",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 277,
        "message_redacted": true,
        "time": 0.138
      },
      "name": "tests.test_config_errors.test_toml_with_invalid_field_type",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 305,
        "message_redacted": true,
        "time": 0.154
      },
      "name": "tests.test_config_errors.test_conflicting_from_and_from_latest_tag_flags",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 338,
        "message_redacted": true,
        "time": 0.133
      },
      "name": "tests.test_config_errors.test_malformed_toml_unclosed_string",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 869,
        "message_redacted": true,
        "time": 0.29
      },
      "name": "tests.test_cli_gaps.test_subtitle_flag_with_various_formats",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 144,
        "message_redacted": true,
        "time": 0.181
      },
      "name": "tests.test_coverage_gaps.test_json_format_with_both_flags",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 234,
        "message_redacted": true,
        "time": 0.211
      },
      "name": "tests.test_edge_cases.test_conflicting_from_flags_error",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 725,
        "message_redacted": true,
        "time": 0.196
      },
      "name": "tests.test_edge_cases.test_very_long_subtitle_1000_chars",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.199
      },
      "name": "tests.test_edge_cases.test_whitespace_only_commit_message",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.158
      },
      "name": "tests.test_edge_cases.test_empty_subtitle_flag",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 205,
        "message_redacted": true,
        "time": 0.193
      },
      "name": "tests.test_coverage_gaps.test_both_work_tree_and_git_dir_flags",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 280,
        "message_redacted": true,
        "time": 0.293
      },
      "name": "tests.test_file_io.test_changelog_flag_combines_infile_and_outfile",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 256,
        "message_redacted": true,
        "time": 0.317
      },
      "name": "tests.test_file_io.test_changelog_flag_conflicts_with_infile",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 126,
        "message_redacted": true,
        "time": 0.313
      },
      "name": "tests.test_file_io.test_infile_with_crlf_line_endings",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 74,
        "message_redacted": true,
        "time": 0.533
      },
      "name": "tests.test_file_io.test_multiple_changelog_append_operations",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 139,
        "message_redacted": true,
        "time": 0.238
      },
      "name": "tests.test_final_gaps.test_format_json_explicit",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 306,
        "message_redacted": true,
        "time": 0.15
      },
      "name": "tests.test_config_errors.test_conflicting_infile_and_changelog_flags",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.31
      },
      "name": "tests.test_final_gaps.test_link_style_cgit_conversion",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 205,
        "message_redacted": true,
        "time": 0.231
      },
      "name": "tests.test_coverage_gaps.test_stash_link_style",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.218
      },
      "name": "tests.test_final_gaps.test_clog_init_git_dir_only_with_config",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.228
      },
      "name": "tests.test_edge_cases.test_repo_with_symlinks",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 370,
        "message_redacted": true,
        "time": 0.195
      },
      "name": "tests.test_edge_cases.test_conflicting_changelog_infile_error",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.209
      },
      "name": "tests.test_edge_cases.test_from_and_to_same_commit",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.241
      },
      "name": "tests.test_edge_cases.test_detached_head_state",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 205,
        "message_redacted": true,
        "time": 0.176
      },
      "name": "tests.test_coverage_gaps.test_work_tree_only_with_config",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 245,
        "message_redacted": true,
        "time": 0.365
      },
      "name": "tests.test_file_io.test_stdout_output_default_behavior",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 105,
        "message_redacted": true,
        "time": 0.341
      },
      "name": "tests.test_file_io.test_outfile_prepends_to_existing_file",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 257,
        "message_redacted": true,
        "time": 0.378
      },
      "name": "tests.test_file_io.test_changelog_flag_conflicts_with_outfile",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 376,
        "message_redacted": true,
        "time": 0.246
      },
      "name": "tests.test_final_gaps.test_outfile_flag_sets_clog_outfile_field",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.201
      },
      "name": "tests.test_final_gaps.test_format_markdown_explicit",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 10.609
      },
      "name": "tests.test_edge_cases.test_repo_with_200_commits",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 139,
        "message_redacted": true,
        "time": 0.218
      },
      "name": "tests.test_final_gaps.test_format_json_with_repo_flag",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 308,
        "message_redacted": true,
        "time": 0.135
      },
      "name": "tests.test_config_errors.test_conflicting_outfile_and_changelog_flags",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 205,
        "message_redacted": true,
        "time": 0.189
      },
      "name": "tests.test_coverage_gaps.test_cgit_link_style",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 205,
        "message_redacted": true,
        "time": 0.179
      },
      "name": "tests.test_coverage_gaps.test_git_dir_only_with_config",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.206
      },
      "name": "tests.test_final_gaps.test_clog_init_work_tree_and_git_dir_both",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 371,
        "message_redacted": true,
        "time": 0.178
      },
      "name": "tests.test_edge_cases.test_conflicting_changelog_outfile_error",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.178
      },
      "name": "tests.test_edge_cases.test_invalid_from_commit_hash",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.3
      },
      "name": "tests.test_edge_cases.test_repo_with_1mb_binary_file",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 855,
        "message_redacted": true,
        "time": 0.146
      },
      "name": "tests.test_final_gaps.test_work_tree_without_git_dir_uses_config",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 1.558
      },
      "name": "tests.test_edge_cases.test_many_tags_120",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 279,
        "message_redacted": true,
        "time": 0.333
      },
      "name": "tests.test_file_io.test_outfile_writes_to_file_not_stdout",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 254,
        "message_redacted": true,
        "time": 0.251
      },
      "name": "tests.test_final_gaps.test_infile_and_outfile_flags_append_behavior",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 73,
        "message_redacted": true,
        "time": 0.337
      },
      "name": "tests.test_git_ops.test_from_latest_tag_json_format",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.331
      },
      "name": "tests.test_final_gaps.test_from_flag_sets_clog_from_field",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.209
      },
      "name": "tests.test_final_gaps.test_link_style_gitlab_conversion",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 656,
        "message_redacted": true,
        "time": 0.222
      },
      "name": "tests.test_final_gaps.test_link_style_github_default",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 285,
        "message_redacted": true,
        "time": 0.186
      },
      "name": "tests.test_edge_cases.test_setversion_with_invalid_semver_format",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.193
      },
      "name": "tests.test_final_gaps.test_clog_init_work_tree_only_with_config",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 858,
        "message_redacted": true,
        "time": 0.159
      },
      "name": "tests.test_final_gaps.test_git_dir_without_work_tree_uses_config",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.213
      },
      "name": "tests.test_edge_cases.test_very_long_commit_message_20kb",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 73,
        "message_redacted": true,
        "time": 0.225
      },
      "name": "tests.test_git_ops.test_json_format_structure",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.282
      },
      "name": "tests.test_edge_cases.test_mixed_conventional_and_non_conventional_commits",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 235,
        "message_redacted": true,
        "time": 0.289
      },
      "name": "tests.test_git_ops.test_conflicting_flags_from_and_from_latest_tag",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 280,
        "message_redacted": true,
        "time": 0.352
      },
      "name": "tests.test_file_io.test_infile_appends_to_existing_changelog",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 254,
        "message_redacted": true,
        "time": 0.226
      },
      "name": "tests.test_final_gaps.test_changelog_flag_sets_both_infile_and_outfile",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.241
      },
      "name": "tests.test_final_gaps.test_from_latest_tag_flag_sets_clog_from_field",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.207
      },
      "name": "tests.test_final_gaps.test_link_style_stash_conversion",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 687,
        "message_redacted": true,
        "time": 0.154
      },
      "name": "tests.test_final_gaps.test_multiple_flags_combination",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.271
      },
      "name": "tests.test_edge_cases.test_very_long_file_path_108_chars",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 762,
        "message_redacted": true,
        "time": 0.291
      },
      "name": "tests.test_final_gaps.test_from_and_to_flags_combined",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.242
      },
      "name": "tests.test_edge_cases.test_empty_commit_message",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 347,
        "message_redacted": true,
        "time": 0.316
      },
      "name": "tests.test_integration.test_broken_config_file_fails_gracefully",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 313,
        "message_redacted": true,
        "time": 0.324
      },
      "name": "tests.test_integration.test_conflicting_flags_from_and_from_latest_tag",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 489,
        "message_redacted": true,
        "time": 0.266
      },
      "name": "tests.test_output.test_json_with_subtitle",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 404,
        "message_redacted": true,
        "time": 0.264
      },
      "name": "tests.test_output.test_json_format_output_structure",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 430,
        "message_redacted": true,
        "time": 0.321
      },
      "name": "tests.test_output.test_infile_and_changelog_mutually_exclusive",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 433,
        "message_redacted": true,
        "time": 0.302
      },
      "name": "tests.test_output.test_outfile_and_changelog_mutually_exclusive",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 298,
        "message_redacted": true,
        "time": 0.31
      },
      "name": "tests.test_output.test_from_and_from_latest_tag_mutually_exclusive",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 205,
        "message_redacted": true,
        "time": 0.193
      },
      "name": "tests.test_version.test_setversion_simple_format",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 205,
        "message_redacted": true,
        "time": 0.184
      },
      "name": "tests.test_version.test_setversion_zero_version",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 210,
        "message_redacted": true,
        "time": 0.181
      },
      "name": "tests.test_version.test_setversion_with_v_prefix",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 595,
        "message_redacted": true,
        "time": 0.364
      },
      "name": "tests.test_integration.test_work_tree_and_git_dir_together_fails_from_different_cwd",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 205,
        "message_redacted": true,
        "time": 0.242
      },
      "name": "tests.test_version.test_setversion_overrides_increment_flags",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 195,
        "message_redacted": true,
        "time": 0.23
      },
      "name": "tests.test_version.test_setversion_accepts_partial_version",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 369,
        "message_redacted": true,
        "time": 0.201
      },
      "name": "tests.test_version.test_setversion_accepts_empty_string",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 453,
        "message_redacted": true,
        "time": 0.46
      },
      "name": "tests.test_integration.test_conflicting_flags_changelog_with_infile_outfile",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 370,
        "message_redacted": true,
        "time": 0.235
      },
      "name": "tests.test_version.test_from_latest_tag_without_tags_produces_empty_version",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 233,
        "message_redacted": true,
        "time": 0.211
      },
      "name": "tests.test_version.test_setversion_with_unicode",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 220,
        "message_redacted": true,
        "time": 0.229
      },
      "name": "tests.test_version.test_setversion_with_leading_zeros",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 215,
        "message_redacted": true,
        "time": 0.212
      },
      "name": "tests.test_version.test_setversion_with_whitespace",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 240,
        "message_redacted": true,
        "time": 0.249
      },
      "name": "tests.test_version.test_setversion_with_prerelease",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 255,
        "message_redacted": true,
        "time": 0.229
      },
      "name": "tests.test_version.test_setversion_with_build_metadata",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 270,
        "message_redacted": true,
        "time": 0.233
      },
      "name": "tests.test_version.test_setversion_complex_prerelease",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 234,
        "message_redacted": true,
        "time": 0.213
      },
      "name": "tests.test_version.test_conflicting_from_flags",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 205,
        "message_redacted": true,
        "time": 0.304
      },
      "name": "tests.test_version.test_setversion_with_from_latest_tag",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 235,
        "message_redacted": true,
        "time": 0.23
      },
      "name": "tests.test_version.test_setversion_large_version_numbers",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 210,
        "message_redacted": true,
        "time": 0.26
      },
      "name": "tests.test_version.test_setversion_negative_version_numbers",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 556,
        "message_redacted": true,
        "time": 0.265
      },
      "name": "tests.test_version.test_setversion_json_format_output",
      "status": "failure"
    },
    {
      "branch": "93267c5d44be",
      "extra": {
        "message_chars": 195,
        "message_redacted": true,
        "time": 0.302
      },
      "name": "tests.test_version.test_setversion_with_only_dots",
      "status": "failure"
    },
    {
      "branch": "ffd7bbf54795",
      "extra": {
        "message_chars": 13,
        "message_redacted": true,
        "time": 0.131
      },
      "name": "eval.tests.test_argument_parsing.TestFormatFlag.test_format_with_equals[-T-markdown]",
      "status": "failure"
    },
    {
      "branch": "ffd7bbf54795",
      "extra": {
        "message_chars": 247,
        "message_redacted": true,
        "time": 0.121
      },
      "name": "eval.tests.test_argument_parsing.TestVersionFlags.test_combined_version_flags_short",
      "status": "failure"
    },
    {
      "branch": "ffd7bbf54795",
      "extra": {
        "message_chars": 248,
        "message_redacted": true,
        "time": 0.134
      },
      "name": "eval.tests.test_argument_parsing.TestVersionFlags.test_combined_version_flags_short_three",
      "status": "failure"
    },
    {
      "branch": "ffd7bbf54795",
      "extra": {
        "message_chars": 13,
        "message_redacted": true,
        "time": 0.127
      },
      "name": "eval.tests.test_argument_parsing.TestMutuallyExclusiveFlags.test_changelog_conflicts_with_both",
      "status": "failure"
    },
    {
      "branch": "ffd7bbf54795",
      "extra": {
        "message_chars": 13,
        "message_redacted": true,
        "time": 0.094
      },
      "name": "eval.tests.test_argument_parsing.TestMutuallyExclusiveFlags.test_changelog_conflicts_with_outfile",
      "status": "failure"
    },
    {
      "branch": "ffd7bbf54795",
      "extra": {
        "message_chars": 13,
        "message_redacted": true,
        "time": 0.119
      },
      "name": "eval.tests.test_argument_parsing.TestMutuallyExclusiveFlags.test_changelog_conflicts_with_infile",
      "status": "failure"
    },
    {
      "branch": "ffd7bbf54795",
      "extra": {
        "message_chars": 13,
        "message_redacted": true,
        "time": 0.104
      },
      "name": "eval.tests.test_argument_parsing.TestMutuallyExclusiveFlags.test_from_conflicts_with_from_latest_tag_short",
      "status": "failure"
    },
    {
      "branch": "ffd7bbf54795",
      "extra": {
        "message_chars": 13,
        "message_redacted": true,
        "time": 0.133
      },
      "name": "eval.tests.test_argument_parsing.TestMutuallyExclusiveFlags.test_from_conflicts_with_from_latest_tag",
      "status": "failure"
    },
    {
      "branch": "ffd7bbf54795",
      "extra": {
        "message_chars": 13,
        "message_redacted": true,
        "time": 0.101
      },
      "name": "eval.tests.test_argument_parsing.TestExitCodes.test_conflict_exit_code",
      "status": "failure"
    },
    {
      "branch": "12b2a3e58e6b",
      "extra": {
        "message_chars": 179,
        "message_redacted": true,
        "time": 0.132
      },
      "name": "eval.tests.test_argparse_validation.test_format_option_accepts_multiple_syntaxes[args4-{\"header\"]",
      "status": "failure"
    },
    {
      "branch": "12b2a3e58e6b",
      "extra": {
        "message_chars": 184,
        "message_redacted": true,
        "time": 0.157
      },
      "name": "eval.tests.test_argparse_validation.test_format_option_accepts_multiple_syntaxes[args1-##]",
      "status": "failure"
    },
    {
      "branch": "12b2a3e58e6b",
      "extra": {
        "message_chars": 176,
        "message_redacted": true,
        "time": 0.152
      },
      "name": "eval.tests.test_argparse_validation.test_combining_version_bump_flags_with_missing_semver_is_runtime_error[args0]",
      "status": "failure"
    },
    {
      "branch": "12b2a3e58e6b",
      "extra": {
        "message_chars": 136,
        "message_redacted": true,
        "time": 0.157
      },
      "name": "eval.tests.test_argparse_validation.test_value_taking_option_cannot_be_used_multiple_times",
      "status": "failure"
    },
    {
      "branch": "1bd74ab559f5",
      "extra": {
        "message_chars": 347,
        "message_redacted": true,
        "time": 0.18
      },
      "name": "eval.tests.test_cli_help_version.test_help_exact_match",
      "status": "failure"
    }
  ],
  "test_results_omitted": 0,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}