{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "5a83c6e4a14294803acfa9576acb318d4d78a6cee45f2d7782f266d44c7c0b6b",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.24441347899846733
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.14931597700342536
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.14732630699290894
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.1279634140082635
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.12641087101656012
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6730541949800681
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11046981799881905
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1103015510016121
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.0937716179760173
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13411202098359354
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 437795,
      "output_redacted": true,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 41.60557464900194
    },
    {
      "branch": "c0523186ac1e",
      "command": "docker cp b06fb00967591dd5b281f0f35a3c726bd8cbab262e47af4ec1e3ed33d88e41a0:/workspace/eval/results.xml /tmp/tmp5m9p1zgk.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 449267,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03588000498712063
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1536014759913087
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1344048600003589
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11911801298265345
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13517145600053482
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 4043511,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 295.26118864299497
    },
    {
      "branch": "0368ca6ed669",
      "command": "docker cp 19773568ff5cd01c67f15d6630b506750c07390ef7e007b17f3a58e4991ba56a:/workspace/eval/results.xml /tmp/tmp_iw9tn4q.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 5623868,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.05729447101475671
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 2357,
    "passed": 108,
    "skipped": 114
  },
  "test_branch_errors": {},
  "test_branches": [
    "c0523186ac1e",
    "0368ca6ed669"
  ],
  "test_records": 2579,
  "test_results": [
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 464,
        "message_redacted": true,
        "time": 0.124
      },
      "name": "eval.tests.test_coverage.test_verilog_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 465,
        "message_redacted": true,
        "time": 0.136
      },
      "name": "eval.tests.test_coverage.test_gdscript_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 466,
        "message_redacted": true,
        "time": 0.117
      },
      "name": "eval.tests.test_coverage.test_ada_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 464,
        "message_redacted": true,
        "time": 0.12
      },
      "name": "eval.tests.test_coverage.test_objc_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 467,
        "message_redacted": true,
        "time": 0.112
      },
      "name": "eval.tests.test_coverage.test_bats_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 466,
        "message_redacted": true,
        "time": 0.131
      },
      "name": "eval.tests.test_coverage.test_org_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 466,
        "message_redacted": true,
        "time": 0.143
      },
      "name": "eval.tests.test_coverage.test_asm_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 465,
        "message_redacted": true,
        "time": 0.135
      },
      "name": "eval.tests.test_coverage.test_kotlin_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 478,
        "message_redacted": true,
        "time": 0.108
      },
      "name": "eval.tests.test_ctags.test_sort_yes",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 468,
        "message_redacted": true,
        "time": 0.156
      },
      "name": "eval.tests.test_coverage.test_protobuf_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 412,
        "message_redacted": true,
        "time": 0.101
      },
      "name": "eval.tests.test_ctags.test_help_full",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 470,
        "message_redacted": true,
        "time": 0.124
      },
      "name": "eval.tests.test_ctags.test_output_format_xref",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 448,
        "message_redacted": true,
        "time": 0.148
      },
      "name": "eval.tests.test_ctags.test_L_option_stdin",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 464,
        "message_redacted": true,
        "time": 0.148
      },
      "name": "eval.tests.test_ctags.test_basic_c_file",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 410,
        "message_redacted": true,
        "time": 0.125
      },
      "name": "eval.tests.test_ctags.test_version_output",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 460,
        "message_redacted": true,
        "time": 0.128
      },
      "name": "eval.tests.test_ctags.test_nonexistent_file_warning",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 465,
        "message_redacted": true,
        "time": 0.099
      },
      "name": "eval.tests.test_coverage.test_julia_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 467,
        "message_redacted": true,
        "time": 0.139
      },
      "name": "eval.tests.test_coverage.test_asciidoc_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 466,
        "message_redacted": true,
        "time": 0.134
      },
      "name": "eval.tests.test_coverage.test_powershell_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 465,
        "message_redacted": true,
        "time": 0.115
      },
      "name": "eval.tests.test_coverage.test_haxe_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 464,
        "message_redacted": true,
        "time": 0.141
      },
      "name": "eval.tests.test_coverage.test_eiffel_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 472,
        "message_redacted": true,
        "time": 0.108
      },
      "name": "eval.tests.test_coverage.test_cmake_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 465,
        "message_redacted": true,
        "time": 0.116
      },
      "name": "eval.tests.test_coverage.test_csharp_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 466,
        "message_redacted": true,
        "time": 0.138
      },
      "name": "eval.tests.test_coverage.test_clojure_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 470,
        "message_redacted": true,
        "time": 0.149
      },
      "name": "eval.tests.test_coverage.test_autoconf_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 979,
        "message_redacted": true,
        "time": 0.124
      },
      "name": "eval.tests.test_ctags.test_no_args_exit_code",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 470,
        "message_redacted": true,
        "time": 0.095
      },
      "name": "eval.tests.test_ctags.test_output_format_etags",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 410,
        "message_redacted": true,
        "time": 0.105
      },
      "name": "eval.tests.test_ctags.test_version_output_format",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 478,
        "message_redacted": true,
        "time": 0.176
      },
      "name": "eval.tests.test_ctags.test_sort_no",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 449,
        "message_redacted": true,
        "time": 0.082
      },
      "name": "eval.tests.test_ctags.test_filter_mode_reads_stdin",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 484,
        "message_redacted": true,
        "time": 0.09
      },
      "name": "eval.tests.test_ctags.test_output_to_file",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 465,
        "message_redacted": true,
        "time": 0.099
      },
      "name": "eval.tests.test_ctags.test_basic_python_file",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 464,
        "message_redacted": true,
        "time": 0.112
      },
      "name": "eval.tests.test_coverage.test_d_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 468,
        "message_redacted": true,
        "time": 0.109
      },
      "name": "eval.tests.test_coverage.test_scala_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 464,
        "message_redacted": true,
        "time": 0.157
      },
      "name": "eval.tests.test_coverage.test_r_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 466,
        "message_redacted": true,
        "time": 0.087
      },
      "name": "eval.tests.test_coverage.test_vhdl_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 466,
        "message_redacted": true,
        "time": 0.132
      },
      "name": "eval.tests.test_coverage.test_scheme_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 467,
        "message_redacted": true,
        "time": 0.107
      },
      "name": "eval.tests.test_coverage.test_html_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 465,
        "message_redacted": true,
        "time": 0.116
      },
      "name": "eval.tests.test_coverage.test_elixir_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 475,
        "message_redacted": true,
        "time": 0.101
      },
      "name": "eval.tests.test_coverage.test_systemd_unit_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 503,
        "message_redacted": true,
        "time": 0.115
      },
      "name": "eval.tests.test_coverage.test_json_fields_output",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 1020,
        "message_redacted": true,
        "time": 0.083
      },
      "name": "eval.tests.test_ctags.test_filter_mode_warning_on_output_file",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 410,
        "message_redacted": true,
        "time": 0.078
      },
      "name": "eval.tests.test_ctags.test_license_output",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 34,
        "message_redacted": true,
        "time": 0.157
      },
      "name": "eval.tests.test_ctags.test_output_format_ectags_mode_marker",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 464,
        "message_redacted": true,
        "time": 0.077
      },
      "name": "eval.tests.test_ctags.test_output_to_stdout",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 407,
        "message_redacted": true,
        "time": 0.097
      },
      "name": "eval.tests.test_ctags.test_help_output",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 465,
        "message_redacted": true,
        "time": 0.101
      },
      "name": "eval.tests.test_ctags.test_basic_js_file",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 483,
        "message_redacted": true,
        "time": 0.071
      },
      "name": "eval.tests.test_ctags.test_sort_foldcase",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 466,
        "message_redacted": true,
        "time": 0.149
      },
      "name": "eval.tests.test_coverage.test_pascal_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 466,
        "message_redacted": true,
        "time": 0.11
      },
      "name": "eval.tests.test_coverage.test_iniconf_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 419,
        "message_redacted": true,
        "time": 0.129
      },
      "name": "eval.tests.test_coverage.test_ctags_self_test_language",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 505,
        "message_redacted": true,
        "time": 0.133
      },
      "name": "eval.tests.test_coverage.test_python_entry_points_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 466,
        "message_redacted": true,
        "time": 0.114
      },
      "name": "eval.tests.test_coverage.test_tex_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 466,
        "message_redacted": true,
        "time": 0.104
      },
      "name": "eval.tests.test_coverage.test_elm_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 468,
        "message_redacted": true,
        "time": 0.145
      },
      "name": "eval.tests.test_coverage.test_swift_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 467,
        "message_redacted": true,
        "time": 0.113
      },
      "name": "eval.tests.test_coverage.test_lisp_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 465,
        "message_redacted": true,
        "time": 0.114
      },
      "name": "eval.tests.test_coverage.test_ocaml_parsing",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 476,
        "message_redacted": true,
        "time": 0.086
      },
      "name": "eval.tests.test_ctags.test_L_option_reads_file_list",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 1007,
        "message_redacted": true,
        "time": 0.094
      },
      "name": "eval.tests.test_ctags.test_invalid_option",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 488,
        "message_redacted": true,
        "time": 0.094
      },
      "name": "eval.tests.test_ctags.test_output_format_json",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 403,
        "message_redacted": true,
        "time": 0.07
      },
      "name": "eval.tests.test_ctags.test_help_short",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 492,
        "message_redacted": true,
        "time": 0.077
      },
      "name": "eval.tests.test_ctags.test_multiple_files",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 470,
        "message_redacted": true,
        "time": 0.157
      },
      "name": "eval.tests.test_ctags.test_u_flag_equivalent_to_sort_no",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 478,
        "message_redacted": true,
        "time": 0.069
      },
      "name": "eval.tests.test_ctags.test_format_1",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 414,
        "message_redacted": true,
        "time": 0.135
      },
      "name": "eval.tests.test_ctags.test_list_fields",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 504,
        "message_redacted": true,
        "time": 0.101
      },
      "name": "eval.tests.test_ctags.test_fields_E_shows_extras",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 470,
        "message_redacted": true,
        "time": 0.128
      },
      "name": "eval.tests.test_ctags.test_backward_search_pattern",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 415,
        "message_redacted": true,
        "time": 0.129
      },
      "name": "eval.tests.test_ctags.test_list_kinds_C",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 479,
        "message_redacted": true,
        "time": 0.15
      },
      "name": "eval.tests.test_ctags.test_fields_plus_n",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 498,
        "message_redacted": true,
        "time": 0.15
      },
      "name": "eval.tests.test_ctags.test_pseudo_tags_disabled",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 484,
        "message_redacted": true,
        "time": 0.147
      },
      "name": "eval.tests.test_ctags.test_guess_language_eagerly",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 414,
        "message_redacted": true,
        "time": 0.148
      },
      "name": "eval.tests.test_ctags.test_list_extras",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 429,
        "message_redacted": true,
        "time": 0.146
      },
      "name": "eval.tests.test_ctags.test_list_map_extensions",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 416,
        "message_redacted": true,
        "time": 0.124
      },
      "name": "eval.tests.test_ctags.test_list_excludes",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 425,
        "message_redacted": true,
        "time": 0.128
      },
      "name": "eval.tests.test_ctags.test_list_mline_regex_flags",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 476,
        "message_redacted": true,
        "time": 0.105
      },
      "name": "eval.tests.test_ctags.test_totals_output",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 422,
        "message_redacted": true,
        "time": 0.077
      },
      "name": "eval.tests.test_ctags.test_describe_language",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 416,
        "message_redacted": true,
        "time": 0.137
      },
      "name": "eval.tests.test_ctags.test_with_list_header_no",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 540,
        "message_redacted": true,
        "time": 0.081
      },
      "name": "eval.tests.test_ctags.test_regex_lang_definition",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 485,
        "message_redacted": true,
        "time": 0.071
      },
      "name": "eval.tests.test_ctags.test_D_macro_definition",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 416,
        "message_redacted": true,
        "time": 0.117
      },
      "name": "eval.tests.test_ctags.test_list_features",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 427,
        "message_redacted": true,
        "time": 0.14
      },
      "name": "eval.tests.test_ctags.test_describe_language_python",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 470,
        "message_redacted": true,
        "time": 0.141
      },
      "name": "eval.tests.test_ctags.test_forward_search_pattern",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 479,
        "message_redacted": true,
        "time": 0.131
      },
      "name": "eval.tests.test_ctags.test_fields_plus_l",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 509,
        "message_redacted": true,
        "time": 0.137
      },
      "name": "eval.tests.test_ctags.test_languages_filter",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 416,
        "message_redacted": true,
        "time": 0.111
      },
      "name": "eval.tests.test_ctags.test_list_params",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 422,
        "message_redacted": true,
        "time": 0.133
      },
      "name": "eval.tests.test_ctags.test_list_output_formats",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 416,
        "message_redacted": true,
        "time": 0.129
      },
      "name": "eval.tests.test_ctags.test_list_fields_for_language",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 503,
        "message_redacted": true,
        "time": 0.107
      },
      "name": "eval.tests.test_ctags.test_pseudo_tags_all",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 417,
        "message_redacted": true,
        "time": 0.125
      },
      "name": "eval.tests.test_ctags.test_list_kinds_all",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 459,
        "message_redacted": true,
        "time": 0.125
      },
      "name": "eval.tests.test_ctags.test_V_equivalent_to_verbose",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 489,
        "message_redacted": true,
        "time": 0.127
      },
      "name": "eval.tests.test_ctags.test_extras_anonymous",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 425,
        "message_redacted": true,
        "time": 0.12
      },
      "name": "eval.tests.test_ctags.test_list_map_patterns",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 415,
        "message_redacted": true,
        "time": 0.12
      },
      "name": "eval.tests.test_ctags.test_kinddef_new_kind",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 430,
        "message_redacted": true,
        "time": 0.096
      },
      "name": "eval.tests.test_ctags.test_machinable_list_extras",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 483,
        "message_redacted": true,
        "time": 0.071
      },
      "name": "eval.tests.test_ctags.test_I_ignore_tokens",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 417,
        "message_redacted": true,
        "time": 0.133
      },
      "name": "eval.tests.test_ctags.test_list_languages",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 501,
        "message_redacted": true,
        "time": 0.134
      },
      "name": "eval.tests.test_ctags.test_put_field_prefix",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 419,
        "message_redacted": true,
        "time": 0.132
      },
      "name": "eval.tests.test_ctags.test_list_pseudo_tags",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 420,
        "message_redacted": true,
        "time": 0.11
      },
      "name": "eval.tests.test_ctags.test_list_kinds_full",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 482,
        "message_redacted": true,
        "time": 0.134
      },
      "name": "eval.tests.test_ctags.test_excmd_number",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 425,
        "message_redacted": true,
        "time": 0.133
      },
      "name": "eval.tests.test_ctags.test_list_map_rexprs",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 414,
        "message_redacted": true,
        "time": 0.124
      },
      "name": "eval.tests.test_ctags.test_list_maps",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 464,
        "message_redacted": true,
        "time": 0.138
      },
      "name": "eval.tests.test_ctags.test_if0_default_disabled",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 415,
        "message_redacted": true,
        "time": 0.134
      },
      "name": "eval.tests.test_ctags.test_list_roles",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 466,
        "message_redacted": true,
        "time": 0.142
      },
      "name": "eval.tests.test_ctags.test_verbose_mode",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 509,
        "message_redacted": true,
        "time": 0.121
      },
      "name": "eval.tests.test_ctags.test_exclude_pattern",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 480,
        "message_redacted": true,
        "time": 0.127
      },
      "name": "eval.tests.test_ctags.test_extras_qualified",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 480,
        "message_redacted": true,
        "time": 0.132
      },
      "name": "eval.tests.test_ctags.test_fields_plus_kz",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 478,
        "message_redacted": true,
        "time": 0.129
      },
      "name": "eval.tests.test_ctags.test_fields_star",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 590,
        "message_redacted": true,
        "time": 0.107
      },
      "name": "eval.tests.test_ctags.test_langdef_custom_language",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 12,
        "message_redacted": true,
        "time": 0.183
      },
      "name": "eval.tests.test_ctags.test_append_mode",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 419,
        "message_redacted": true,
        "time": 0.117
      },
      "name": "eval.tests.test_ctags.test_list_regex_flags",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 432,
        "message_redacted": true,
        "time": 0.123
      },
      "name": "eval.tests.test_ctags.test_machinable_output",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 422,
        "message_redacted": true,
        "time": 0.124
      },
      "name": "eval.tests.test_ctags.test_list_aliases",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 464,
        "message_redacted": true,
        "time": 0.101
      },
      "name": "eval.tests.test_ctags.test_quiet_mode",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 549,
        "message_redacted": true,
        "time": 0.134
      },
      "name": "eval.tests.test_ctags.test_exclude_exception",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 483,
        "message_redacted": true,
        "time": 0.128
      },
      "name": "eval.tests.test_ctags.test_excmd_pattern",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 463,
        "message_redacted": true,
        "time": 0.127
      },
      "name": "eval.tests.test_ctags.test_R_equivalent_to_recurse",
      "status": "failure"
    },
    {
      "branch": "c0523186ac1e",
      "extra": {
        "message_chars": 489,
        "message_redacted": true,
        "time": 0.095
      },
      "name": "eval.tests.test_ctags.test_extras_inputFile_tag",
      "status": "failure"
    }
  ],
  "test_results_omitted": 2351,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}