{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "83e1ef6d48884ca34c30652fc09939ec23da3c44913c904f477a8caf87ac8bd9",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.1604694949928671
    },
    {
      "branch": "",
      "command": "find /workspace -type f -exec sha256sum {} + 2>/dev/null | grep -E \"^(76c0c98b1ec3165b826ee40f73fbf7406f1bd8e054f48bcc139242b36b96ff6e)  \" | cut -c67- | xargs -I% rm -fv %",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.13518600500538014
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.15249639001558535
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.10387393698329106
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.12719956898945384
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.12705501000164077
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6345230120059568
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14645329301129095
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11537833500187844
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11790765001205727
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12210255800164305
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 16881,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 8.083346319006523
    },
    {
      "branch": "d3e0b8bd2908",
      "command": "docker cp 234a2cb7c293cb8fb294bd383b7d92c8026e532d67d531c3d1500d5eae5ea019:/workspace/eval/results.xml /tmp/tmp9bljkt5_.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 11418,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03508572801365517
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1412876319955103
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11967638999340124
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13154711699462496
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12464414298301563
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 30305,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 43.749610889994074
    },
    {
      "branch": "5eb07a00de06",
      "command": "docker cp 55fb97a438b27475435440095d983c5ba6e5015aa44efcfe67f9a25e45262cb0:/workspace/eval/results.xml /tmp/tmpccboejtg.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 30788,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03992898197611794
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10633004602277651
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.10376292999717407
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12599351600511
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.14133712300099432
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 10476,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.984932271996513
    },
    {
      "branch": "7264c23f9ad3",
      "command": "docker cp 8cfddf77fb1a1c0aec1fdd5ecafcc373e7d8bd62befe9c5dd14f40e17d4aa62b:/workspace/eval/results.xml /tmp/tmpsb_n1c5_.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 6032,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.037403094000183046
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1415573720005341
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12431678600842133
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13092623100965284
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13551360499695875
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 7984,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.7783815809816588
    },
    {
      "branch": "46c007125807",
      "command": "docker cp 67523ff7056d5642cb88cc9a2fc117195340d4ce367552076138c64750a4463a:/workspace/eval/results.xml /tmp/tmpmb0tpygv.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4683,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03699891600990668
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.06912623002426699
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.08043022899073549
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11251213098876178
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.07790704097715206
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 8804,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.400344931986183
    },
    {
      "branch": "11c7e7093b72",
      "command": "docker cp 3b2d86b57eefca5b647ed962c4777acf14c700cad1a220200844970dab95dfc0:/workspace/eval/results.xml /tmp/tmpmiyf20vq.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5295,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.039130225020926446
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13815270198392682
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12554574001114815
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13359370600664988
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12992952598142438
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4368,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 18.46816355001647
    },
    {
      "branch": "4b3c87660b99",
      "command": "docker cp 28e67c60387fed55c7369979662a805d3aec7a6f6267229cd39bf2a069fa53ca:/workspace/eval/results.xml /tmp/tmpezpqago0.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4697,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03951658299774863
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 11,
    "passed": 321
  },
  "test_branch_errors": {},
  "test_branches": [
    "d3e0b8bd2908",
    "5eb07a00de06",
    "7264c23f9ad3",
    "46c007125807",
    "11c7e7093b72",
    "4b3c87660b99"
  ],
  "test_records": 332,
  "test_results": [
    {
      "branch": "5eb07a00de06",
      "extra": {
        "message_chars": 627,
        "message_redacted": true,
        "time": 9.38
      },
      "name": "tests.test_keyboard_layouts.test_azerty_layout_rendering",
      "status": "failure"
    },
    {
      "branch": "5eb07a00de06",
      "extra": {
        "message_chars": 628,
        "message_redacted": true,
        "time": 9.37
      },
      "name": "tests.test_keyboard_layouts.test_qwerty_to_qwertz_emulation_workflow",
      "status": "failure"
    },
    {
      "branch": "5eb07a00de06",
      "extra": {
        "message_chars": 627,
        "message_redacted": true,
        "time": 9.35
      },
      "name": "tests.test_keyboard_layouts.test_qwerty_to_azerty_emulation_workflow",
      "status": "failure"
    },
    {
      "branch": "5eb07a00de06",
      "extra": {
        "message_chars": 620,
        "message_redacted": true,
        "time": 9.34
      },
      "name": "tests.test_keyboard_layouts.test_qwertz_distinctive_keys",
      "status": "failure"
    },
    {
      "branch": "5eb07a00de06",
      "extra": {
        "message_chars": 628,
        "message_redacted": true,
        "time": 9.419
      },
      "name": "tests.test_keyboard_layouts.test_qwertz_layout_rendering",
      "status": "failure"
    },
    {
      "branch": "5eb07a00de06",
      "extra": {
        "message_chars": 3175,
        "message_redacted": true,
        "time": 7.066
      },
      "name": "tests.test_ngrams.test_empty_file_triggers_panic",
      "status": "failure"
    },
    {
      "branch": "5eb07a00de06",
      "extra": {
        "message_chars": 1563,
        "message_redacted": true,
        "time": 9.588
      },
      "name": "tests.test_tui_typing.test_tui_typing_and_lesson_completion",
      "status": "failure"
    },
    {
      "branch": "5eb07a00de06",
      "extra": {
        "message_chars": 658,
        "message_redacted": true,
        "time": 18.788
      },
      "name": "tests.test_keyboard_layouts.test_all_layouts_have_spacebar",
      "status": "failure"
    },
    {
      "branch": "5eb07a00de06",
      "extra": {
        "message_chars": 65,
        "message_redacted": true,
        "time": 8.105
      },
      "name": "tests.test_tui_typing.test_tui_error_display_and_backspace",
      "status": "failure"
    },
    {
      "branch": "5eb07a00de06",
      "extra": {
        "message_chars": 642,
        "message_redacted": true,
        "time": 28.018
      },
      "name": "tests.test_keyboard_layouts.test_mixed_layout_pairs_coverage",
      "status": "failure"
    },
    {
      "branch": "4b3c87660b99",
      "extra": {
        "message_chars": 1558,
        "message_redacted": true,
        "time": 10.453
      },
      "name": "eval.tests.test_tui_ngrrram.test_typing_updates_counters",
      "status": "failure"
    }
  ],
  "test_results_omitted": 0,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}