{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "c49fad5e1f18f9be8f98afb62a734c9d57b1ee03a3211c594e9c9f9427a99a2b",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.1639431639923714
    },
    {
      "branch": "",
      "command": "find /workspace -type f -exec sha256sum {} + 2>/dev/null | grep -E \"^(2255b718c18c5e12f490a6bb66eb920122a3988f60cbd657b0468792cdf427ae)  \" | cut -c67- | xargs -I% rm -fv %",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.14243653899757192
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.16652655199868605
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.14414930700149853
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.13986595999449492
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.11480618000496179
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.8370248630089918
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1432513770123478
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12991604600392748
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12237077200552449
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11720867799886037
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 226926,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 25.45905342399783
    },
    {
      "branch": "9e91d4ba23b1",
      "command": "docker cp 28b94f167b2fa44ec283ded1648394c6f639be954145b72381abf6975b9de04b:/workspace/eval/results.xml /tmp/tmp8hyejyvi.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 220300,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.04165637999540195
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13690190500346944
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11244522799097467
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.10407864299486391
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13073006398917641
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 224081,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 23.50453968399961
    },
    {
      "branch": "7e38a5c4c8b2",
      "command": "docker cp 6ea4f96f676d275d6c93c70e26327d75b9d332f04ca7bffebe1767efccbc6e11:/workspace/eval/results.xml /tmp/tmphg1kbiay.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 198839,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03835157999128569
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13063912298821378
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12481137800205033
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12593152400222607
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12724785700265784
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 462631,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 49.284238157008076
    },
    {
      "branch": "0d82cc7b3089",
      "command": "docker cp 9370599acc7c0f95e5b20469d1ce3990f505062b2242fda532df40fd16ba75d4:/workspace/eval/results.xml /tmp/tmp3pjczujl.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 471802,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.037975975006702356
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10691211400262546
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12934925699664745
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12106672200025059
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13002932899689768
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 24284,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 7.703510450999602
    },
    {
      "branch": "dcb39b350797",
      "command": "docker cp 8d6e5f953105b737cca763506970029c31502ad188d6a718c78e541a99eea07e:/workspace/eval/results.xml /tmp/tmpc61bszui.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 17175,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.037515780990361236
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.09811457600153517
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11583154900290538
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13235254799656104
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12480806501116604
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 42769,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 7.857992074001231
    },
    {
      "branch": "3ba2eba9299f",
      "command": "docker cp 5423484bb5d0ece355445ad833e80feb7a0dfdec6f0808fe6b1fef57d72a4f59:/workspace/eval/results.xml /tmp/tmp0dvg734d.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 38981,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.0380414740066044
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11454612499801442
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1354104380006902
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1214137880015187
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13394433799840044
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2761,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.6219550910027465
    },
    {
      "branch": "16cc4aec109e",
      "command": "docker cp 272f2ad71f8779167efb3a0b99dc6fc134f0b932f7092bd6674cea36de7b3cd8:/workspace/eval/results.xml /tmp/tmpff8ni4u9.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1360,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.01994008400652092
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1320837509992998
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1299434679967817
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.10644072899594903
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13191724300850183
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 34703,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 7.727593645991874
    },
    {
      "branch": "7db22598f3f9",
      "command": "docker cp 2551b6a13aa75b29d9b586a255067eaec1be31b5be90be306d0b20a4654702cb:/workspace/eval/results.xml /tmp/tmpoi7sywmr.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 23429,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03680082999926526
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14312520899693482
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1388657140050782
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.14037481299601495
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1251577289949637
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 17574,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 5.710076115996344
    },
    {
      "branch": "ca19e5f78c56",
      "command": "docker cp dc21188ce928bed1b10e255933631da6ba05ddf1161dda9c2c8f454b269b8af2:/workspace/eval/results.xml /tmp/tmp9cel2ipk.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 15801,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03679838799871504
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11586186799104325
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1261859359947266
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11288266499468591
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11625723200268112
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 24439,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 4.842245047009783
    },
    {
      "branch": "ab8ac5490068",
      "command": "docker cp dbaecc8e0adf8c109353295cd404c62c895a286c3ad89250b23ca97c893e7f62:/workspace/eval/results.xml /tmp/tmpbg1esph9.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 21298,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.040097927005263045
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12825634899490979
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.14305643799889367
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1099745519895805
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.09343371400609612
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 156576,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 21.434730073000537
    },
    {
      "branch": "894395edf357",
      "command": "docker cp 2ec2c7999e001809e20261855920d12c2e3f3aa9f91ff8fee1f2e72d23b9f6a4:/workspace/eval/results.xml /tmp/tmpl4zus33s.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 95808,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03996575900237076
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14370980899548158
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.10989536199485883
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.09581428299134132
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.09797566500492394
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 40073,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 7.669612721001613
    },
    {
      "branch": "0c21df0dda18",
      "command": "docker cp d2f78f1df5da6204a410023afc1934bc0dc2f7b6f5fd5839e50e1cda852d288c:/workspace/eval/results.xml /tmp/tmpk12_1fa9.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 37168,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.04357230100140441
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1465291030035587
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1364162120007677
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12540378300764132
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13415805200929753
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 29711,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.660033533000387
    },
    {
      "branch": "65b3ba9bf0fd",
      "command": "docker cp b3268024d99c9e5d0d2c7fb2f9c01e1f1a0afb30a0dc03ea77838ae008562d9f:/workspace/eval/results.xml /tmp/tmp4c9gi95_.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 27082,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.037951133999740705
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 926,
    "passed": 48
  },
  "test_branch_errors": {},
  "test_branches": [
    "9e91d4ba23b1",
    "7e38a5c4c8b2",
    "0d82cc7b3089",
    "dcb39b350797",
    "3ba2eba9299f",
    "16cc4aec109e",
    "7db22598f3f9",
    "ca19e5f78c56",
    "ab8ac5490068",
    "894395edf357",
    "0c21df0dda18",
    "65b3ba9bf0fd"
  ],
  "test_records": 974,
  "test_results": [
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.083
      },
      "name": "eval.tests.test_advanced_features.test_blockquote_nested",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.082
      },
      "name": "eval.tests.test_advanced_features.test_preformatted_text",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.078
      },
      "name": "eval.tests.test_advanced_features.test_url_with_query_params",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.089
      },
      "name": "eval.tests.test_advanced_features.test_nested_lists",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.082
      },
      "name": "eval.tests.test_advanced_features.test_cite_tag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.083
      },
      "name": "eval.tests.test_advanced_features.test_relative_url_with_dots",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 373,
        "message_redacted": true,
        "time": 0.099
      },
      "name": "eval.tests.test_advanced_features.test_multiple_classes",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 378,
        "message_redacted": true,
        "time": 0.085
      },
      "name": "eval.tests.test_advanced_features.test_pseudo_class_selector",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.097
      },
      "name": "eval.tests.test_advanced_features.test_emphasis_within_strong",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.08
      },
      "name": "eval.tests.test_advanced_features.test_mark_tag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.095
      },
      "name": "eval.tests.test_advanced_features.test_address_tag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.091
      },
      "name": "eval.tests.test_advanced_features.test_sub_tag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.107
      },
      "name": "eval.tests.test_advanced_features.test_figure_and_figcaption",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.1
      },
      "name": "eval.tests.test_advanced_features.test_mailto_link",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.105
      },
      "name": "eval.tests.test_advanced_features.test_image_without_alt",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.104
      },
      "name": "eval.tests.test_advanced_features.test_empty_elements",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.064
      },
      "name": "eval.tests.test_advanced_features.test_multiple_strong_in_paragraph",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.09
      },
      "name": "eval.tests.test_advanced_features.test_link_within_strong",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.066
      },
      "name": "eval.tests.test_advanced_features.test_time_tag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 386,
        "message_redacted": true,
        "time": 0.081
      },
      "name": "eval.tests.test_advanced_features.test_attribute_selector",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.083
      },
      "name": "eval.tests.test_advanced_features.test_abbr_tag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.074
      },
      "name": "eval.tests.test_advanced_features.test_link_with_title",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.078
      },
      "name": "eval.tests.test_advanced_features.test_data_url",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.074
      },
      "name": "eval.tests.test_advanced_features.test_details_and_summary",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.079
      },
      "name": "eval.tests.test_advanced_features.test_sup_tag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.092
      },
      "name": "eval.tests.test_advanced_features.test_kbd_tag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.084
      },
      "name": "eval.tests.test_advanced_features.test_whitespace_preservation_in_code",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.088
      },
      "name": "eval.tests.test_advanced_features.test_url_with_fragment",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.084
      },
      "name": "eval.tests.test_advanced_features.test_mixed_list_types",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.095
      },
      "name": "eval.tests.test_advanced_features.test_definition_list",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.078
      },
      "name": "eval.tests.test_advanced_features.test_code_with_language",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.077
      },
      "name": "eval.tests.test_advanced_features.test_tel_link",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.077
      },
      "name": "eval.tests.test_core_conversion.test_convert_multiple_paragraphs",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 374,
        "message_redacted": true,
        "time": 0.081
      },
      "name": "eval.tests.test_domain.test_domain_relative_image",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 349,
        "message_redacted": true,
        "time": 0.097
      },
      "name": "eval.tests.test_basic_invocation.test_help_flag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.093
      },
      "name": "eval.tests.test_core_conversion.test_convert_em",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.076
      },
      "name": "eval.tests.test_domain.test_no_domain_relative_link",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.091
      },
      "name": "eval.tests.test_advanced_features.test_list_with_paragraphs",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 373,
        "message_redacted": true,
        "time": 0.059
      },
      "name": "eval.tests.test_domain.test_domain_with_http",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.09
      },
      "name": "eval.tests.test_core_conversion.test_convert_code_inline",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.084
      },
      "name": "eval.tests.test_error_handling.test_nested_unclosed_tags",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.083
      },
      "name": "eval.tests.test_core_conversion.test_convert_hr_tag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.065
      },
      "name": "eval.tests.test_basic_invocation.test_stdin_whitespace_only",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.083
      },
      "name": "eval.tests.test_core_conversion.test_convert_h3",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.077
      },
      "name": "eval.tests.test_error_handling.test_malformed_html",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.086
      },
      "name": "eval.tests.test_core_conversion.test_convert_unordered_list",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.062
      },
      "name": "eval.tests.test_error_handling.test_deeply_nested_tags",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.073
      },
      "name": "eval.tests.test_error_handling.test_style_tag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.098
      },
      "name": "eval.tests.test_core_conversion.test_convert_complex_document",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 374,
        "message_redacted": true,
        "time": 0.086
      },
      "name": "eval.tests.test_domain.test_domain_absolute_link_unchanged",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.091
      },
      "name": "eval.tests.test_core_conversion.test_convert_h1",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 379,
        "message_redacted": true,
        "time": 0.091
      },
      "name": "eval.tests.test_domain.test_domain_with_path",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 366,
        "message_redacted": true,
        "time": 0.084
      },
      "name": "eval.tests.test_domain.test_domain_without_protocol",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.086
      },
      "name": "eval.tests.test_basic_invocation.test_stdin_basic_conversion",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.087
      },
      "name": "eval.tests.test_core_conversion.test_convert_strong_default",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.083
      },
      "name": "eval.tests.test_core_conversion.test_convert_link",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.088
      },
      "name": "eval.tests.test_core_conversion.test_convert_pre_code_block",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 352,
        "message_redacted": true,
        "time": 0.085
      },
      "name": "eval.tests.test_basic_invocation.test_version_flag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 374,
        "message_redacted": true,
        "time": 0.073
      },
      "name": "eval.tests.test_domain.test_domain_relative_link",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.078
      },
      "name": "eval.tests.test_error_handling.test_special_html_entities",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.077
      },
      "name": "eval.tests.test_core_conversion.test_convert_ordered_list",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.08
      },
      "name": "eval.tests.test_error_handling.test_unicode_characters",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.071
      },
      "name": "eval.tests.test_error_handling.test_script_tag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.067
      },
      "name": "eval.tests.test_error_handling.test_comment_handling",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 683,
        "message_redacted": true,
        "time": 0.083
      },
      "name": "eval.tests.test_error_handling.test_null_bytes",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.099
      },
      "name": "eval.tests.test_core_conversion.test_convert_br_tag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.088
      },
      "name": "eval.tests.test_basic_invocation.test_stdin_empty_input",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.078
      },
      "name": "eval.tests.test_escaping.test_escape_brackets",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.062
      },
      "name": "eval.tests.test_core_conversion.test_convert_blockquote",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.102
      },
      "name": "eval.tests.test_escaping.test_escape_asterisk",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.077
      },
      "name": "eval.tests.test_core_conversion.test_convert_nested_tags",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 368,
        "message_redacted": true,
        "time": 0.098
      },
      "name": "eval.tests.test_core_conversion.test_convert_strong_underscore",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.095
      },
      "name": "eval.tests.test_core_conversion.test_convert_image",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 345,
        "message_redacted": true,
        "time": 0.074
      },
      "name": "eval.tests.test_basic_invocation.test_version_short_flag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.079
      },
      "name": "eval.tests.test_escaping.test_real_vs_fake_bold",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.088
      },
      "name": "eval.tests.test_core_conversion.test_convert_h2",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.083
      },
      "name": "eval.tests.test_error_handling.test_very_long_line",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.067
      },
      "name": "eval.tests.test_escaping.test_escape_list_markers",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.073
      },
      "name": "eval.tests.test_escaping.test_ampersand_escape",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.057
      },
      "name": "eval.tests.test_escaping.test_quotes_handling",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.082
      },
      "name": "eval.tests.test_escaping.test_backtick_in_code",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.066
      },
      "name": "eval.tests.test_escaping.test_escape_backticks",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.086
      },
      "name": "eval.tests.test_error_handling.test_binary_data",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.08
      },
      "name": "eval.tests.test_escaping.test_escape_gt_lt",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 381,
        "message_redacted": true,
        "time": 0.092
      },
      "name": "eval.tests.test_input_output.test_output_to_file",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.095
      },
      "name": "eval.tests.test_escaping.test_escape_pipe_in_text",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 423,
        "message_redacted": true,
        "time": 0.073
      },
      "name": "eval.tests.test_input_output.test_input_and_output_files",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 405,
        "message_redacted": true,
        "time": 0.078
      },
      "name": "eval.tests.test_input_output.test_output_file_with_overwrite",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.072
      },
      "name": "eval.tests.test_escaping.test_escape_hashes",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 381,
        "message_redacted": true,
        "time": 0.084
      },
      "name": "eval.tests.test_input_output.test_input_from_file",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 383,
        "message_redacted": true,
        "time": 0.083
      },
      "name": "eval.tests.test_input_output.test_input_glob_pattern",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 1405,
        "message_redacted": true,
        "time": 0.082
      },
      "name": "eval.tests.test_input_output.test_input_directory_error",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.069
      },
      "name": "eval.tests.test_escaping.test_escape_underscore",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 365,
        "message_redacted": true,
        "time": 0.073
      },
      "name": "eval.tests.test_plugins.test_strikethrough_strike_tag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 357,
        "message_redacted": true,
        "time": 0.072
      },
      "name": "eval.tests.test_plugins.test_table_with_headers",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 357,
        "message_redacted": true,
        "time": 0.068
      },
      "name": "eval.tests.test_plugins.test_table_enabled_basic",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 423,
        "message_redacted": true,
        "time": 0.107
      },
      "name": "eval.tests.test_input_output.test_input_output_round_trip",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 1411,
        "message_redacted": true,
        "time": 0.079
      },
      "name": "eval.tests.test_input_output.test_output_directory_must_end_with_slash",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.091
      },
      "name": "eval.tests.test_plugins.test_strikethrough_disabled_by_default",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 365,
        "message_redacted": true,
        "time": 0.079
      },
      "name": "eval.tests.test_plugins.test_strikethrough_enabled",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.069
      },
      "name": "eval.tests.test_plugins.test_table_disabled_by_default",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 397,
        "message_redacted": true,
        "time": 0.085
      },
      "name": "eval.tests.test_plugins.test_table_span_cell_behavior",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 365,
        "message_redacted": true,
        "time": 0.094
      },
      "name": "eval.tests.test_plugins.test_strikethrough_s_tag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 2799,
        "message_redacted": true,
        "time": 0.078
      },
      "name": "eval.tests.test_plugins.test_table_option_without_plugin",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 387,
        "message_redacted": true,
        "time": 0.088
      },
      "name": "eval.tests.test_input_output.test_multiple_files_with_glob",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 388,
        "message_redacted": true,
        "time": 0.082
      },
      "name": "eval.tests.test_plugins.test_table_skip_empty_rows",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 368,
        "message_redacted": true,
        "time": 0.087
      },
      "name": "eval.tests.test_selectors.test_exclude_selector_basic",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 389,
        "message_redacted": true,
        "time": 0.076
      },
      "name": "eval.tests.test_plugins.test_table_header_promotion",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 402,
        "message_redacted": true,
        "time": 0.077
      },
      "name": "eval.tests.test_plugins.test_table_cell_padding_behavior",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 392,
        "message_redacted": true,
        "time": 0.064
      },
      "name": "eval.tests.test_plugins.test_table_presentation_tables",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 398,
        "message_redacted": true,
        "time": 0.069
      },
      "name": "eval.tests.test_plugins.test_table_newline_behavior",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 383,
        "message_redacted": true,
        "time": 0.064
      },
      "name": "eval.tests.test_plugins.test_combined_plugins",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 395,
        "message_redacted": true,
        "time": 0.081
      },
      "name": "eval.tests.test_selectors.test_multiple_include_selectors",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 370,
        "message_redacted": true,
        "time": 0.097
      },
      "name": "eval.tests.test_selectors.test_exclude_selector_by_tag",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 370,
        "message_redacted": true,
        "time": 0.091
      },
      "name": "eval.tests.test_selectors.test_include_selector_basic",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 370,
        "message_redacted": true,
        "time": 0.092
      },
      "name": "eval.tests.test_selectors.test_include_selector_by_id",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.08
      },
      "name": "eval.tests.test_special_cases.test_div_with_nested_content",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 395,
        "message_redacted": true,
        "time": 0.084
      },
      "name": "eval.tests.test_selectors.test_multiple_exclude_selectors",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.079
      },
      "name": "eval.tests.test_special_cases.test_link_with_empty_href",
      "status": "failure"
    },
    {
      "branch": "9e91d4ba23b1",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.083
      },
      "name": "eval.tests.test_special_cases.test_span_with_class",
      "status": "failure"
    }
  ],
  "test_results_omitted": 806,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}