{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "ff5d32ecf5a33a83376f02b99c37b3d794d61933c9ae8f2517fd4547a9e152a4",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.15146417601499707
    },
    {
      "branch": "",
      "command": "find /workspace -type f -exec sha256sum {} + 2>/dev/null | grep -E \"^(e07420f640d1e6a0c047cab49431d1b561f661241efe111a053ff5dde1119d09)  \" | cut -c67- | xargs -I% rm -fv %",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.12075578200165182
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.14547715699882247
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.13544590197852813
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.109583280980587
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.1307551469944883
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6641632090031635
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13141597399953753
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13610015899757855
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1278172130114399
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13035833998583257
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 16185,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 4.767797906999476
    },
    {
      "branch": "a7a2f5f083bf",
      "command": "docker cp 91e1ff4a9da79aaf49dab6e346c96c6b95996bc6a6d593f656d8bd6a66d7e971:/workspace/eval/results.xml /tmp/tmpaw90iy_3.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 11967,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.030992832995252684
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13432842900510877
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12570306498673744
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.10274862698861398
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1294525050034281
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 21507,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 4.726320172980195
    },
    {
      "branch": "d31025219260",
      "command": "docker cp 1c5055b6b302b275ef4c562e4d0672f4f2759a0cadec836add8a34aa9061247d:/workspace/eval/results.xml /tmp/tmplj5jbycl.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 15559,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.04124998699990101
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13693249400239438
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11013998900307342
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1287449689989444
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12478844900033437
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 6714,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.756994650000706
    },
    {
      "branch": "65af426d9fb3",
      "command": "docker cp faf1e270e3417beb3209d1230406d858fbeeb2032ad4ae6959da32ba73de6b3b:/workspace/eval/results.xml /tmp/tmpbcpnk227.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 3682,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.027359442989109084
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.144411693996517
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1403105659992434
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.14396733700414188
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11341305100359023
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1187,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.243756249983562
    },
    {
      "branch": "efa8c407dbe3",
      "command": "docker cp a110c9f49e28aca9e9cdd4f330c885c65fe3995f423e72687dd2b153b79a8619:/workspace/eval/results.xml /tmp/tmpidm8kr0v.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 487,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.0387332419923041
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12863715898129158
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13256021900451742
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12984371298807673
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12765389497508295
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 7457,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.4012693340191618
    },
    {
      "branch": "9ec117cda36f",
      "command": "docker cp 11f811aadc3a173f08d55f1cc113b78b2bc7b078443df69178995e4401a3afec:/workspace/eval/results.xml /tmp/tmpsc4bdfka.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4535,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.040213354979641736
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13539201297680847
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11107810298562981
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12257705198135227
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12160289799794555
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2596,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.453605096001411
    },
    {
      "branch": "b3b22ece5568",
      "command": "docker cp de306c51d98b1befd45dc2412a66876d706b40d3034c36b951074eb5441dee16:/workspace/eval/results.xml /tmp/tmp2bhvhyxd.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1391,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03815723498701118
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13483422601711936
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12097737001022324
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13284609699621797
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13039810897316784
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 2526,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.989291031000903
    },
    {
      "branch": "5f4759b87816",
      "command": "docker cp 82fd875cba1013e2495fc6560a00c87615fcda806833e3e0d024c0ba99e5a0e3:/workspace/eval/results.xml /tmp/tmpxgj67u0b.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 1988,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.030595196003559977
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 4,
    "passed": 343,
    "skipped": 1
  },
  "test_branch_errors": {},
  "test_branches": [
    "a7a2f5f083bf",
    "d31025219260",
    "65af426d9fb3",
    "efa8c407dbe3",
    "9ec117cda36f",
    "b3b22ece5568",
    "5f4759b87816"
  ],
  "test_records": 348,
  "test_results": [
    {
      "branch": "a7a2f5f083bf",
      "extra": {
        "message_chars": 297,
        "message_redacted": true,
        "time": 0.092
      },
      "name": "eval.tests.test_csview.test_line_numbers_seq_alias",
      "status": "failure"
    },
    {
      "branch": "a7a2f5f083bf",
      "extra": {
        "message_chars": 314,
        "message_redacted": true,
        "time": 0.257
      },
      "name": "eval.tests.test_csview.test_style_case_insensitive",
      "status": "failure"
    },
    {
      "branch": "d31025219260",
      "extra": {
        "message_chars": 397,
        "message_redacted": true,
        "time": 0.118
      },
      "name": "tests.test_input.test_emoji_wide_unicode_characters",
      "status": "failure"
    },
    {
      "branch": "65af426d9fb3",
      "extra": {
        "message_chars": 13,
        "message_redacted": true,
        "time": 0.256
      },
      "name": "eval.tests.test_argparse.test_flag_value_forms_are_equivalent_for_success",
      "status": "failure"
    },
    {
      "branch": "b3b22ece5568",
      "extra": {
        "message_chars": 53,
        "message_redacted": true,
        "time": 0.0
      },
      "name": "eval.tests.test_csview_io.test_unreadable_file_permission_denied_exit_1",
      "status": "skipped"
    }
  ],
  "test_results_omitted": 0,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}