{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "evaluator_log_steps": [
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.16998458199668676
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.12781798200740013
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.1683488330018008
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.13278577000892255
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.14013919100398198
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.13982197100995108
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6274329589941772
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13523616500606295
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.08006041101180017
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12274816799617838
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13290551099635195
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 4.476281440991443
    },
    {
      "branch": "34775c99d1b3",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.022415651998016983
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13564333399699535
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1412953370017931
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1269512619910529
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11053568001079839
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.515257309001754
    },
    {
      "branch": "5389d4487a4f",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.038307807000819594
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12144174899731297
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1300644719885895
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11307635399862193
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.09211336199950892
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 1965.8129094079923
    },
    {
      "branch": "adfa09238fcb",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.35529621499881614
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.17592516299919225
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11696059499809053
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12891867199505214
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12079242800245993
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 10.05052628100384
    },
    {
      "branch": "4755328c3247",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.039546631000121124
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.16706378699745983
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13220677599019837
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13373656899784692
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12724111700663343
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.749514373994316
    },
    {
      "branch": "e691f4eccbfb",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.05094940299750306
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11112947500078008
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11463651999656577
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12755278500844724
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13241062298766337
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.9537274790054653
    },
    {
      "branch": "ea19aed40d43",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03296157000295352
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.15338647400494665
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13369634599075653
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12737971400201786
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1305428199993912
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.955709639005363
    },
    {
      "branch": "923fa802fa04",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.0369057219941169
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1699355199962156
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13378309400286525
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1289269060071092
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12985724200552795
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.7788752790074795
    },
    {
      "branch": "8a8cc3a8aeb3",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.028361517004668713
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12593689700588584
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12094613000226673
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12633863600785844
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13377244501316454
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.810193748999154
    },
    {
      "branch": "76aa74f92a9e",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03844241400656756
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1480734910001047
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1362502720003249
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1253094089915976
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11403967899968848
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.2594712029967923
    },
    {
      "branch": "a7cf48110fef",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03908799099735916
    }
  ],
  "evaluator_log_steps_omitted": 0,
  "executable_hash": "007ae244f35dd61b45f339f553eb26de0b53a877eca6406c9f8ee1b5b4d7f8c2",
  "failed_tests": [
    {
      "branch": "34775c99d1b3",
      "message_chars": 194,
      "message_redacted": true,
      "name": "eval.tests.test_errors.test_duplicate_array_flags_error",
      "status": "failure"
    },
    {
      "branch": "34775c99d1b3",
      "message_chars": 233,
      "message_redacted": true,
      "name": "eval.tests.test_errors.test_duplicate_format_flags_error",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 1317,
      "message_redacted": true,
      "name": "tests.test_errors.test_cols_overflow_value",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 284,
      "message_redacted": true,
      "name": "tests.test_exotic_formats.test_format_flag_empty_string_rejected",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 241,
      "message_redacted": true,
      "name": "tests.test_interactions.test_multiple_format_flags_rejected",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 231,
      "message_redacted": true,
      "name": "tests.test_interactions.test_multiple_column_flags_rejected",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 217,
      "message_redacted": true,
      "name": "tests.test_interactions.test_multiple_array_flags_rejected",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 135,
      "message_redacted": true,
      "name": "tests.test_pipe_handling.test_broken_pipe_array_c_format",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 135,
      "message_redacted": true,
      "name": "tests.test_pipe_handling.test_broken_pipe_with_column_limit",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 172,
      "message_redacted": true,
      "name": "tests.test_pipe_handling.test_broken_pipe_with_head_exits_zero",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 135,
      "message_redacted": true,
      "name": "tests.test_pipe_handling.test_broken_pipe_with_grep_early_match",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 135,
      "message_redacted": true,
      "name": "tests.test_pipe_handling.test_broken_pipe_very_small_consumer_limit",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 135,
      "message_redacted": true,
      "name": "tests.test_pipe_handling.test_broken_pipe_octal_format",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 135,
      "message_redacted": true,
      "name": "tests.test_pipe_handling.test_broken_pipe_upperhex_format",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 250,
      "message_redacted": true,
      "name": "tests.test_errors.test_len_overflow_value",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 135,
      "message_redacted": true,
      "name": "tests.test_pipe_handling.test_broken_pipe_with_color_enabled",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 135,
      "message_redacted": true,
      "name": "tests.test_pipe_handling.test_broken_pipe_array_output_rust_format",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 135,
      "message_redacted": true,
      "name": "tests.test_pipe_handling.test_broken_pipe_binary_format_output",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 135,
      "message_redacted": true,
      "name": "tests.test_pipe_handling.test_broken_pipe_with_prefix_disabled",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 135,
      "message_redacted": true,
      "name": "tests.test_pipe_handling.test_broken_pipe_array_python_format",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 135,
      "message_redacted": true,
      "name": "tests.test_pipe_handling.test_broken_pipe_combined_column_and_format",
      "status": "failure"
    },
    {
      "branch": "adfa09238fcb",
      "message_chars": 174,
      "message_redacted": true,
      "name": "tests.test_errors.test_func_overflow_value",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 157,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args5]",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 117,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args0]",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 157,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args4]",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 157,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args2]",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 157,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args6]",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 157,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args7]",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 157,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args3]",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 117,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args1]",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 157,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args8]",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 120,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args11]",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 157,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args9]",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 157,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args14]",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 157,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args15]",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 140,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args13]",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 140,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args12]",
      "status": "failure"
    },
    {
      "branch": "4755328c3247",
      "message_chars": 120,
      "message_redacted": true,
      "name": "eval.tests.test_argparse_validation.test_value_options_cannot_be_repeated[flag_args10]",
      "status": "failure"
    }
  ],
  "failed_tests_omitted": 0,
  "status_counts": {
    "failure": 38,
    "passed": 839
  },
  "test_branch_errors": {},
  "test_branches": [
    "34775c99d1b3",
    "5389d4487a4f",
    "adfa09238fcb",
    "4755328c3247",
    "e691f4eccbfb",
    "ea19aed40d43",
    "923fa802fa04",
    "8a8cc3a8aeb3",
    "76aa74f92a9e",
    "a7cf48110fef"
  ],
  "test_records": 877,
  "warnings": []
}