{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "evaluator_log_steps": [
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.16140253499906976
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.1856551239907276
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.11923561200092081
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.10113576900039334
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.12131131198839284
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.645046438992722
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.138866194000002
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12857291399268433
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11530969500017818
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1082553710002685
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 21.05464163000579
    },
    {
      "branch": "f8d762f9e408",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03838826600986067
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10226386299473234
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12409934400056954
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1300081250083167
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.13378566200844944
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 51.724259596012416
    },
    {
      "branch": "be0550a7a9ee",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.04229956200288143
    }
  ],
  "evaluator_log_steps_omitted": 0,
  "executable_hash": "9f69a1d20a88066560a43c5f7a7bf6ae2e1fc89e91324d6e9bfc5940a9a34bd8",
  "failed_tests": [
    {
      "branch": "f8d762f9e408",
      "message_chars": 4570,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestBasicInvocation.test_help_mentions_coalesced_options",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 902,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestFileDecompression.test_decompress_precompressed_alice",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 868,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestFileCompression.test_compress_large_text_file_smaller",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 896,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestFileDecompression.test_decompress_precompressed_zeros",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 873,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestFileCompression.test_compress_file_with_keep_flag",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 867,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestFileCompression.test_compress_file_creates_valid_br",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 2281,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestBasicInvocation.test_help_short_goes_to_stderr",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 850,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestStdinInput.test_decompress_empty_compressed_stdin",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 599,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestRoundTrip.test_roundtrip_all_quality_levels",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 838,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestBasicInvocation.test_no_args_with_piped_stdin",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 2316,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestFileDecompression.test_decompress_suffix_mismatch_error",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 139,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestRoundTrip.test_roundtrip_text",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 115,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestStdinInput.test_compress_from_explicit_dash_stdin",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 167,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestRoundTrip.test_roundtrip_known_test_files",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 109,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestOutputOptions.test_stdout_flag_long",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 909,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestOutputOptions.test_decompress_to_output_file",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 4580,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestBasicInvocation.test_help_mentions_stdin_dash",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 2285,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestBasicInvocation.test_help_long_goes_to_stderr",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 898,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestFileCompression.test_compress_multiple_files_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 896,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestFileDecompression.test_decompress_precompressed_empty",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 51,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestStdinInput.test_compress_from_stdin_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 867,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestFileCompression.test_compress_keeps_source_by_default",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 872,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestFileDecompression.test_decompress_keeps_compressed_by_default",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 2326,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestFileDecompression.test_decompress_nonexistent_file",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 844,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestStdinInput.test_compress_empty_stdin",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 460,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestStdinInput.test_stdin_binary_data_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 494,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestCompressionQuality.test_best_flag_Z_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 99,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestOutputOptions.test_stdout_flag_short",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 462,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestRoundTrip.test_roundtrip_binary",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 494,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestCompressionQuality.test_quality_short_q_11_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 904,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestOutputOptions.test_output_file_flag_short",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 494,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestCompressionQuality.test_quality_short_q_0_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 873,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestRemoveSource.test_compress_rm_removes_source_and_valid_output",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 494,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestWindowSize.test_large_window_flag_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 494,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestWindowSize.test_lgwin_10_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 875,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestRemoveSource.test_compress_rm_long_flag",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 494,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestWindowSize.test_lgwin_0_auto_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 858,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestSquash.test_squash_with_stdout_is_invalid",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 873,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestSquash.test_squash_removes_output_if_larger",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 918,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestNoCopyStat.test_no_copy_stat_long",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 873,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestForceOverwrite.test_compress_force_overwrite",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 907,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestOutputOptions.test_output_file_flag_long_equals",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 494,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestCompressionQuality.test_quality_long_flag_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 2321,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestForceOverwrite.test_compress_fails_if_output_exists",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 878,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestForceOverwrite.test_decompress_force_overwrite",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 522,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestCompressionQuality.test_quality_short_flag_0_to_9",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 848,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestIntegrity.test_test_integrity_long_flag",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 873,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestSquash.test_squash_large_compressible_data_keeps_output",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 906,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestNoCopyStat.test_no_copy_stat_short",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 878,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestRemoveSource.test_decompress_rm_removes_source",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 3171,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestSuffix.test_suffix_with_path_separator_fails",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 896,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestIntegrity.test_test_integrity_precompressed_file",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 2326,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestForceOverwrite.test_decompress_fails_if_output_exists",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 872,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestIntegrity.test_test_integrity_valid_file",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 494,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestWindowSize.test_lgwin_24_max_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 2316,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestIntegrity.test_test_integrity_corrupt_file",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 494,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestWindowSize.test_lgwin_long_flag_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 897,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestSuffix.test_custom_suffix_decompress",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 844,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestIntegrity.test_test_integrity_from_stdin",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 878,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestForceOverwrite.test_force_flag_long_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 101,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestComment.test_compress_with_comment_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 2322,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestComment.test_comment_too_much_padding",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 113,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestComment.test_comment_no_padding_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 3881,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestComment.test_comment_length_mismatch_fails",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 887,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestSuffix.test_custom_suffix_long_flag",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 105,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestSuffix.test_default_suffix_is_br",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 884,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestSuffix.test_custom_suffix_compress",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 850,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestComment.test_decompress_ignores_comment_without_flag",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 868,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestComment.test_decompress_with_matching_comment",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 3171,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestSuffix.test_suffix_with_backslash_fails",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 856,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestConcatenated.test_concatenated_flag_K_allows_multi_stream",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 868,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestConcatenated.test_concatenated_long_flag",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 873,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestDoubleDash.test_double_dash_treats_args_as_files",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 7781,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestComment.test_decompress_with_mismatched_comment_fails",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 2291,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestConcatenated.test_decompress_concatenated_fails_without_flag",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 856,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestConcatenated.test_concatenated_three_streams",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 846,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestCoalescedOptions.test_coalesced_dKc",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 917,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestMultipleInputFiles.test_multiple_files_roundtrip",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 2291,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestErrorHandling.test_corrupt_brotli_data",
      "status": "failure"
    },
    {
      "branch": "f8d762f9e408",
      "message_chars": 111,
      "message_redacted": true,
      "name": "eval.tests.test_brotli.TestComment.test_comment_long_flag_roundtrip",
      "status": "failure"
    }
  ],
  "failed_tests_omitted": 378,
  "status_counts": {
    "failure": 458,
    "passed": 148
  },
  "test_branch_errors": {},
  "test_branches": [
    "f8d762f9e408",
    "be0550a7a9ee"
  ],
  "test_records": 606,
  "warnings": []
}