{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "evaluator_log_steps": [
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.1399456250073854
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.1125761830189731
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.14018929499434307
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.13382026899489574
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.13737129099899903
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.1361124670074787
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6283669369877316
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12912579497788101
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13312341400887817
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.127350415015826
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13715061100083403
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 4.342067558987765
    },
    {
      "branch": "974a321c7e65",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.021763402997748926
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10961026098811999
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13844545499887317
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12358173399115913
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.14298636102466844
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 10.256776719994377
    },
    {
      "branch": "25bb8e14ea1f",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.039224648993695155
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13751509098801762
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11734345799777657
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12092307899729349
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1328227850026451
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 7.761258120008279
    },
    {
      "branch": "f55d757f742b",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.045792903983965516
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11616598500404507
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.10980887900223024
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.13607461799983867
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.10837666699080728
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.4816577390010934
    },
    {
      "branch": "9c51f40d6bb3",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.034367796004517004
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11894961100188084
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.145773515017936
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12372176500502974
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12662023899611086
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 7.6476433420029934
    },
    {
      "branch": "44af1a981b68",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.02677030602353625
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12411980301840231
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.10569695199956186
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12760410201735795
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12357115600025281
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.5498876170022413
    },
    {
      "branch": "68c15b2cd662",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03744785799062811
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13341661199228838
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12631501100258902
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11367798800347373
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.08613014800357632
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 5.534562694985652
    },
    {
      "branch": "b614d8a71e4f",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.021586254006251693
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10644449800020084
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.07642246902105398
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12836823001271114
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12745501499739476
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 5.594566376996227
    },
    {
      "branch": "67117444b255",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.0334499099990353
    }
  ],
  "evaluator_log_steps_omitted": 0,
  "executable_hash": "d5c3ed277cb8862253ee4b6fa99d34a61bbfc5136b96cdd797462ffd105d8e02",
  "failed_tests": [
    {
      "branch": "974a321c7e65",
      "message_chars": 302,
      "message_redacted": true,
      "name": "eval.tests.test_encoding.test_encoding_case_insensitive_mixed",
      "status": "failure"
    },
    {
      "branch": "974a321c7e65",
      "message_chars": 2376,
      "message_redacted": true,
      "name": "eval.tests.test_completion.test_completion_help",
      "status": "failure"
    },
    {
      "branch": "974a321c7e65",
      "message_chars": 131,
      "message_redacted": true,
      "name": "eval.tests.test_edge_cases.test_broken_pipe_handling",
      "status": "failure"
    },
    {
      "branch": "974a321c7e65",
      "message_chars": 314,
      "message_redacted": true,
      "name": "eval.tests.test_encoding.test_encoding_case_insensitive_upper",
      "status": "failure"
    },
    {
      "branch": "25bb8e14ea1f",
      "message_chars": 740,
      "message_redacted": true,
      "name": "tests.test_completion.test_completion_help_shows_usage",
      "status": "failure"
    },
    {
      "branch": "25bb8e14ea1f",
      "message_chars": 194,
      "message_redacted": true,
      "name": "tests.test_completion.test_completion_subcommand_with_extra_flag_after_shell",
      "status": "failure"
    },
    {
      "branch": "25bb8e14ea1f",
      "message_chars": 122,
      "message_redacted": true,
      "name": "tests.test_errors.test_no_input_when_stdin_is_tty",
      "status": "failure"
    },
    {
      "branch": "25bb8e14ea1f",
      "message_chars": 265,
      "message_redacted": true,
      "name": "tests.test_errors.test_multiple_file_arguments",
      "status": "failure"
    },
    {
      "branch": "25bb8e14ea1f",
      "message_chars": 354,
      "message_redacted": true,
      "name": "tests.test_options.test_encoding_case_insensitive",
      "status": "failure"
    },
    {
      "branch": "25bb8e14ea1f",
      "message_chars": 110,
      "message_redacted": true,
      "name": "tests.test_scaling.test_combined_scale_both_0_5",
      "status": "failure"
    },
    {
      "branch": "25bb8e14ea1f",
      "message_chars": 773,
      "message_redacted": true,
      "name": "tests.test_completion.test_completion_help_lists_all_shells",
      "status": "failure"
    },
    {
      "branch": "f55d757f742b",
      "message_chars": 13,
      "message_redacted": true,
      "name": "eval.tests.test_args_parsing.test_single_dash_is_treated_as_file_and_fails_with_os_error",
      "status": "failure"
    },
    {
      "branch": "f55d757f742b",
      "message_chars": 204,
      "message_redacted": true,
      "name": "eval.tests.test_args_parsing.test_combined_short_flags_not_supported_for_value_options[args0]",
      "status": "failure"
    },
    {
      "branch": "f55d757f742b",
      "message_chars": 207,
      "message_redacted": true,
      "name": "eval.tests.test_args_parsing.test_combined_short_flags_not_supported_for_value_options[args2]",
      "status": "failure"
    },
    {
      "branch": "f55d757f742b",
      "message_chars": 207,
      "message_redacted": true,
      "name": "eval.tests.test_args_parsing.test_combined_short_flags_not_supported_for_value_options[args1]",
      "status": "failure"
    },
    {
      "branch": "44af1a981b68",
      "message_chars": 294,
      "message_redacted": true,
      "name": "eval.tests.test_help_exact_baselines.test_baseline_completion_help_normalized",
      "status": "failure"
    },
    {
      "branch": "44af1a981b68",
      "message_chars": 776,
      "message_redacted": true,
      "name": "eval.tests.test_help_subcommands.test_completion_help_documents_shell_arg_and_values",
      "status": "failure"
    },
    {
      "branch": "44af1a981b68",
      "message_chars": 295,
      "message_redacted": true,
      "name": "eval.tests.test_help_subcommands.test_completion_help_has_help_option_only",
      "status": "failure"
    },
    {
      "branch": "44af1a981b68",
      "message_chars": 805,
      "message_redacted": true,
      "name": "eval.tests.test_help_subcommands.test_completion_help_usage_line",
      "status": "failure"
    },
    {
      "branch": "b614d8a71e4f",
      "message_chars": 895,
      "message_redacted": true,
      "name": "eval.tests.test_subcommands.test_subcommand_recognized_via_help_flag[help]",
      "status": "failure"
    },
    {
      "branch": "b614d8a71e4f",
      "message_chars": 2439,
      "message_redacted": true,
      "name": "eval.tests.test_subcommands.test_global_flag_rejected_when_after_subcommand",
      "status": "failure"
    },
    {
      "branch": "b614d8a71e4f",
      "message_chars": 2404,
      "message_redacted": true,
      "name": "eval.tests.test_subcommands.test_help_flag_after_subcommand_name_is_routed_to_subcommand_help",
      "status": "failure"
    },
    {
      "branch": "b614d8a71e4f",
      "message_chars": 3114,
      "message_redacted": true,
      "name": "eval.tests.test_subcommands.test_subcommand_help_differs_from_main_help_for_completion",
      "status": "failure"
    },
    {
      "branch": "67117444b255",
      "message_chars": 773,
      "message_redacted": true,
      "name": "eval.tests.test_help_smoke.test_completion_help_mentions_shell_values",
      "status": "failure"
    }
  ],
  "failed_tests_omitted": 0,
  "status_counts": {
    "failure": 24,
    "passed": 345
  },
  "test_branch_errors": {},
  "test_branches": [
    "974a321c7e65",
    "25bb8e14ea1f",
    "f55d757f742b",
    "9c51f40d6bb3",
    "44af1a981b68",
    "68c15b2cd662",
    "b614d8a71e4f",
    "67117444b255"
  ],
  "test_records": 369,
  "warnings": []
}