{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "02bafeda7ef0349a55529eaaec8572e0fbf35050a9bf6f0af80a4015ae33478c",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.1562851570051862
    },
    {
      "branch": "",
      "command": "find /workspace -type f -exec sha256sum {} + 2>/dev/null | grep -E \"^(cda9ecdd140675f8deae07452dd339da80dbd1c0d2d76c038c13e2a5e12665a8)  \" | cut -c67- | xargs -I% rm -fv %",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.12606665900966618
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.15823428699513897
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.12140680500306189
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.12625312700401992
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.13020814799529035
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6175462380051613
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1400131260015769
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11361186701105908
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13578184599464294
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.10028264801076148
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 27260,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 4.898216603993205
    },
    {
      "branch": "c4f46363268e",
      "command": "docker cp 70863e2692cf325a3e03e940aa290ed585d7669e3ad83a2fe7fa6342a0bd8ea9:/workspace/eval/results.xml /tmp/tmpkr_log6k.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 27211,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.037286302002030425
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13592811199487187
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11275499100156594
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.10708749800687656
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11232362600276247
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 16205,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.8053948979941197
    },
    {
      "branch": "00cd38f2ece5",
      "command": "docker cp 5bc2a91b47b233f53b27440f8b585355480031d9db37c5208b342240eff90598:/workspace/eval/results.xml /tmp/tmp88sgs0vz.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 10935,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.0363494740013266
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1479451089981012
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11870280699804425
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12910194399592
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1354313629999524
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 92376,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 15.717639106995193
    },
    {
      "branch": "61cbf5f33b3c",
      "command": "docker cp 58dc78c4c6f90b11a3f260ee717c3cd55eaf12bd7738cb2ac7acf5aa83788cfd:/workspace/eval/results.xml /tmp/tmpuoyom3m6.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 67564,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.037503537998418324
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1406798690004507
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11991960000887047
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12784704398654867
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13560744900314603
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 13253,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 7.705150146008236
    },
    {
      "branch": "da03f47f3ae8",
      "command": "docker cp db0c3a8a493204861a3c3314405b115d064e6861ac0e3a286aba0b09708d5fd4:/workspace/eval/results.xml /tmp/tmpeeqtnbnh.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 8941,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.035728286995436065
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13104761100839823
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1230852290027542
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11677037901245058
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12841332900279667
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3557,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.734175584002514
    },
    {
      "branch": "4648fc65a45a",
      "command": "docker cp fa57a47b1ad92fb2f5b6bf6e1804724a546d77b90c5b5b7fae3a5e7c78a8d038:/workspace/eval/results.xml /tmp/tmp0d25bsf_.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1721,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.025139481003861874
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1427016069937963
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12787163999746554
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.08269822099828161
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12519702399731614
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 20474,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.632525723005529
    },
    {
      "branch": "acb602d37a0c",
      "command": "docker cp 80bf2a69e3b3ff162a1c80f77a4d725bdbb6db769375303a46a7fa870739df7d:/workspace/eval/results.xml /tmp/tmpymdgddcz.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 13417,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03900037000130396
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13602537800034042
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12847630999749526
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1213319689995842
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12477490599849261
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5721,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 6.039861918005045
    },
    {
      "branch": "e5ea0b741776",
      "command": "docker cp 7d1929f3d2b05fd2496bae5a4d32337ee05a410d04e498586fa990fcfd3d776c:/workspace/eval/results.xml /tmp/tmpddqdxmzm.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3507,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03578032199584413
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1419414000120014
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1220036379963858
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1067598820081912
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1188403889973415
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 9278,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.4335066659987206
    },
    {
      "branch": "669d25dc68c5",
      "command": "docker cp 2aaf2d57fff06e36a41545137cc59af602e7ebf2924532c86add6aeddf83985e:/workspace/eval/results.xml /tmp/tmp9gxssd_v.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5475,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.030613841998274438
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12985528900753707
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13774777299840935
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.09775885099952575
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11671064399706665
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 17837,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.4703365900059
    },
    {
      "branch": "bcffb206da76",
      "command": "docker cp b8fcf03bad82af768d4a77ced6acc80feb2e2a1fcb3b8f6092a98ec592bdc026:/workspace/eval/results.xml /tmp/tmpcevh5out.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 13014,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.02497694500198122
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12890537099156063
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11028641600569244
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.10026549000758678
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13118787200073712
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3730,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.65204167200136
    },
    {
      "branch": "482bffc77db0",
      "command": "docker cp a35894ae2352a679448e88663cb8c763e3db64e3e18b30763aa3bac5ff081c07:/workspace/eval/results.xml /tmp/tmp52aur081.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1980,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.04008701498969458
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 63,
    "passed": 783
  },
  "test_branch_errors": {},
  "test_branches": [
    "c4f46363268e",
    "00cd38f2ece5",
    "61cbf5f33b3c",
    "da03f47f3ae8",
    "4648fc65a45a",
    "acb602d37a0c",
    "e5ea0b741776",
    "669d25dc68c5",
    "bcffb206da76",
    "482bffc77db0"
  ],
  "test_records": 846,
  "test_results": [
    {
      "branch": "c4f46363268e",
      "extra": {
        "message_chars": 1425,
        "message_redacted": true,
        "time": 0.151
      },
      "name": "eval.tests.test_basic_invocation.test_invalid_command",
      "status": "failure"
    },
    {
      "branch": "c4f46363268e",
      "extra": {
        "message_chars": 1738,
        "message_redacted": true,
        "time": 0.175
      },
      "name": "eval.tests.test_basic_invocation.test_no_arguments_shows_help",
      "status": "failure"
    },
    {
      "branch": "c4f46363268e",
      "extra": {
        "message_chars": 1373,
        "message_redacted": true,
        "time": 0.131
      },
      "name": "eval.tests.test_error_conditions.test_enter_vault_with_no_args",
      "status": "failure"
    },
    {
      "branch": "c4f46363268e",
      "extra": {
        "message_chars": 1911,
        "message_redacted": true,
        "time": 0.369
      },
      "name": "eval.tests.test_error_conditions.test_invalid_item_type",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 189,
        "message_redacted": true,
        "time": 0.129
      },
      "name": "tests.test_config_operations.test_invalid_config_type_rejected",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 260,
        "message_redacted": true,
        "time": 0.121
      },
      "name": "tests.test_config_operations.test_set_conflict_invalid_boolean_panics",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 104,
        "message_redacted": true,
        "time": 0.097
      },
      "name": "tests.test_config_operations.test_config_handles_single_quotes_in_value",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 175,
        "message_redacted": true,
        "time": 0.295
      },
      "name": "tests.test_error_handling.test_invalid_name_question_mark",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 175,
        "message_redacted": true,
        "time": 0.283
      },
      "name": "tests.test_error_handling.test_invalid_name_less_than",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 286,
        "message_redacted": true,
        "time": 0.106
      },
      "name": "tests.test_config_operations.test_config_too_many_arguments",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 175,
        "message_redacted": true,
        "time": 0.31
      },
      "name": "tests.test_error_handling.test_invalid_name_asterisk",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 328,
        "message_redacted": true,
        "time": 0.093
      },
      "name": "tests.test_error_handling.test_missing_open_note_name",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 332,
        "message_redacted": true,
        "time": 0.128
      },
      "name": "tests.test_error_handling.test_missing_vault_name_for_enter",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 210,
        "message_redacted": true,
        "time": 0.201
      },
      "name": "tests.test_error_handling.test_rename_vault_to_invalid_name",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 234,
        "message_redacted": true,
        "time": 0.776
      },
      "name": "tests.test_folder_operations.test_chdir_nested_path",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 227,
        "message_redacted": true,
        "time": 1.065
      },
      "name": "tests.test_folder_operations.test_chdir_relative_path_navigation",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 251,
        "message_redacted": true,
        "time": 1.063
      },
      "name": "tests.test_folder_operations.test_chdir_into_nested_folder_from_root",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 355,
        "message_redacted": true,
        "time": 1.212
      },
      "name": "tests.test_folder_operations.test_deeply_nested_folder_operations",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 34,
        "message_redacted": true,
        "time": 0.136
      },
      "name": "tests.test_error_handling.test_malformed_config_toml",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 74,
        "message_redacted": true,
        "time": 0.118
      },
      "name": "tests.test_config_operations.test_config_handles_newlines_in_value",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 475,
        "message_redacted": true,
        "time": 0.345
      },
      "name": "tests.test_folder_operations.test_folder_invalid_name_characters",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 179,
        "message_redacted": true,
        "time": 0.319
      },
      "name": "tests.test_error_handling.test_invalid_folder_name",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 175,
        "message_redacted": true,
        "time": 0.288
      },
      "name": "tests.test_error_handling.test_invalid_name_percent",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 175,
        "message_redacted": true,
        "time": 0.292
      },
      "name": "tests.test_error_handling.test_invalid_name_greater_than",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 175,
        "message_redacted": true,
        "time": 0.31
      },
      "name": "tests.test_error_handling.test_invalid_name_backslash",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 242,
        "message_redacted": true,
        "time": 0.658
      },
      "name": "tests.test_integration_workflows.test_complex_move_rename_chain",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 334,
        "message_redacted": true,
        "time": 0.123
      },
      "name": "tests.test_error_handling.test_missing_chdir_path",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 339,
        "message_redacted": true,
        "time": 0.108
      },
      "name": "tests.test_error_handling.test_missing_remove_arguments",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 175,
        "message_redacted": true,
        "time": 0.336
      },
      "name": "tests.test_error_handling.test_path_traversal_in_note_name",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 248,
        "message_redacted": true,
        "time": 1.212
      },
      "name": "tests.test_integration_workflows.test_interleaved_multi_vault_operations",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 243,
        "message_redacted": true,
        "time": 0.596
      },
      "name": "tests.test_integration_workflows.test_items_with_special_characters",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 34,
        "message_redacted": true,
        "time": 0.099
      },
      "name": "tests.test_error_handling.test_malformed_vaults_toml",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 219,
        "message_redacted": true,
        "time": 0.41
      },
      "name": "tests.test_error_handling.test_rename_to_same_name",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 177,
        "message_redacted": true,
        "time": 0.105
      },
      "name": "tests.test_error_handling.test_invalid_vault_name",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 175,
        "message_redacted": true,
        "time": 0.337
      },
      "name": "tests.test_error_handling.test_invalid_name_colon",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 350,
        "message_redacted": true,
        "time": 0.109
      },
      "name": "tests.test_error_handling.test_missing_rename_arguments",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 215,
        "message_redacted": true,
        "time": 0.313
      },
      "name": "tests.test_error_handling.test_invalid_name_forward_slash",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 203,
        "message_redacted": true,
        "time": 0.391
      },
      "name": "tests.test_error_handling.test_rename_note_to_invalid_name",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 406,
        "message_redacted": true,
        "time": 0.361
      },
      "name": "tests.test_folder_operations.test_rename_same_name_error",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 180,
        "message_redacted": true,
        "time": 0.294
      },
      "name": "tests.test_error_handling.test_path_traversal_in_folder_name",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 293,
        "message_redacted": true,
        "time": 1.282
      },
      "name": "tests.test_integration_workflows.test_deep_folder_hierarchy_navigation",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 333,
        "message_redacted": true,
        "time": 0.096
      },
      "name": "tests.test_error_handling.test_missing_note_name_argument",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 221,
        "message_redacted": true,
        "time": 0.435
      },
      "name": "tests.test_note_operations.test_note_rename_same_name_error",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 221,
        "message_redacted": true,
        "time": 0.406
      },
      "name": "tests.test_error_handling.test_rename_folder_to_same_name",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 350,
        "message_redacted": true,
        "time": 0.127
      },
      "name": "tests.test_error_handling.test_missing_move_arguments",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 175,
        "message_redacted": true,
        "time": 0.314
      },
      "name": "tests.test_error_handling.test_invalid_name_pipe",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 205,
        "message_redacted": true,
        "time": 0.381
      },
      "name": "tests.test_error_handling.test_rename_folder_to_invalid_name",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 147,
        "message_redacted": true,
        "time": 0.717
      },
      "name": "tests.test_integration_workflows.test_vault_creation_error_cases",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 193,
        "message_redacted": true,
        "time": 0.115
      },
      "name": "tests.test_state_persistence.test_config_invalid_key_returns_error",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 379,
        "message_redacted": true,
        "time": 0.106
      },
      "name": "tests.test_state_persistence.test_config_boolean_parsing_case_insensitive",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 136,
        "message_redacted": true,
        "time": 0.371
      },
      "name": "tests.test_vault_lifecycle.test_enter_already_in_vault_error",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 343,
        "message_redacted": true,
        "time": 0.124
      },
      "name": "tests.test_error_handling.test_missing_folder_name_argument",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 159,
        "message_redacted": true,
        "time": 0.417
      },
      "name": "tests.test_error_handling.test_invalid_name_double_quote",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 215,
        "message_redacted": true,
        "time": 0.41
      },
      "name": "tests.test_note_operations.test_note_invalid_name_with_slash",
      "status": "failure"
    },
    {
      "branch": "61cbf5f33b3c",
      "extra": {
        "message_chars": 388,
        "message_redacted": true,
        "time": 0.17
      },
      "name": "tests.test_error_handling.test_missing_vault_location_argument",
      "status": "failure"
    },
    {
      "branch": "da03f47f3ae8",
      "extra": {
        "message_chars": 323,
        "message_redacted": true,
        "time": 0.17
      },
      "name": "eval.tests.test_argparse.test_vault_combined_short_flags_repeat_is_error",
      "status": "failure"
    },
    {
      "branch": "da03f47f3ae8",
      "extra": {
        "message_chars": 329,
        "message_redacted": true,
        "time": 0.179
      },
      "name": "eval.tests.test_argparse.test_vault_dash_dash_treats_flag_like_value_then_missing_location",
      "status": "failure"
    },
    {
      "branch": "da03f47f3ae8",
      "extra": {
        "message_chars": 318,
        "message_redacted": true,
        "time": 0.173
      },
      "name": "eval.tests.test_argparse.test_vault_dash_equals_is_parsed_as_unknown_flag_cluster",
      "status": "failure"
    },
    {
      "branch": "da03f47f3ae8",
      "extra": {
        "message_chars": 320,
        "message_redacted": true,
        "time": 0.169
      },
      "name": "eval.tests.test_argparse.test_vault_flag_duplicate_is_error",
      "status": "failure"
    },
    {
      "branch": "da03f47f3ae8",
      "extra": {
        "message_chars": 176,
        "message_redacted": true,
        "time": 0.179
      },
      "name": "eval.tests.test_argparse.test_list_rejects_unknown_flag",
      "status": "failure"
    },
    {
      "branch": "4648fc65a45a",
      "extra": {
        "message_chars": 192,
        "message_redacted": true,
        "time": 0.191
      },
      "name": "eval.tests.test_config_handling.test_malformed_config_file_panics_on_load",
      "status": "failure"
    },
    {
      "branch": "acb602d37a0c",
      "extra": {
        "message_chars": 496,
        "message_redacted": true,
        "time": 0.149
      },
      "name": "eval.tests.test_help_edge_cases.test_unknown_subcommand_has_nonzero_exit_and_error_message",
      "status": "failure"
    },
    {
      "branch": "e5ea0b741776",
      "extra": {
        "message_chars": 123,
        "message_redacted": true,
        "time": 0.55
      },
      "name": "eval.tests.test_jot_io.test_list_to_closed_pipe_does_not_panic",
      "status": "failure"
    }
  ],
  "test_results_omitted": 0,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}