{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "evaluator_log_steps": [
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.14419631800046773
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "remove_hashed_files",
      "wall_time": 0.10947368899905996
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.14432680800018716
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.13105095199898642
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.13482900599956338
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.1146791179999127
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6779564459993708
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12571642599868937
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12173759200049972
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12035429699972156
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12237786900004721
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 4.448385327001233
    },
    {
      "branch": "27501e100bd6",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.024270702999274363
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.10015783999915584
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12448992099962197
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.10660222400110797
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12510937899969576
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 1,
      "step": "run_tests",
      "wall_time": 8.895680589001131
    },
    {
      "branch": "0d7f74667e0b",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.019204184000045643
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.09868662699955166
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12322968800071976
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1193338750017574
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.10216794199914148
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.540847155998563
    },
    {
      "branch": "edbc2e256c13",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03224399099963193
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1332410400009394
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11540570099896286
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.131428546999814
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.10478777299977082
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.39069759099948
    },
    {
      "branch": "3fe36788fd45",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.038507337998453295
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12673598699984723
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11617446899981587
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.09283135200166726
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12992940399999497
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.47665307600073
    },
    {
      "branch": "e0bceb24b9d4",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.02573229800145782
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12850974100001622
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12472131600043213
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.1282599609985482
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12468970699956117
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 3.3736171109994757
    },
    {
      "branch": "6ae19a5ce7b4",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.03542670000024373
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.11228078599924629
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.11849993100076972
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11951748799947381
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12407094899936055
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 2.14623359199868
    },
    {
      "branch": "b72326e507b1",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.030695228999320534
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.09912568899926555
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12252285700014909
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.12212378799995349
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12371882099978393
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.649959898999441
    },
    {
      "branch": "498523513ae5",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.035888287999114254
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.1372861300005752
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12335803499991016
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11458080900047207
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.1275054280013137
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.519467293999696
    },
    {
      "branch": "8bcffc633c6d",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.02372390599884966
    }
  ],
  "evaluator_log_steps_omitted": 0,
  "executable_hash": "0e4d47d427b4afb255d7bae579b6ab297c148eacfae3c706eced312ddc992b92",
  "failed_tests": [
    {
      "branch": "27501e100bd6",
      "message_chars": 1243,
      "message_redacted": true,
      "name": "tests.test_complex_scenarios.test_local_replace",
      "status": "failure"
    },
    {
      "branch": "27501e100bd6",
      "message_chars": 710,
      "message_redacted": true,
      "name": "tests.test_error_handling.test_unexpected_eof",
      "status": "failure"
    },
    {
      "branch": "27501e100bd6",
      "message_chars": 714,
      "message_redacted": true,
      "name": "tests.test_error_handling.test_invalid_json_syntax",
      "status": "failure"
    },
    {
      "branch": "27501e100bd6",
      "message_chars": 1229,
      "message_redacted": true,
      "name": "tests.test_replace_handling.test_replace_uses_replaced_version",
      "status": "failure"
    },
    {
      "branch": "27501e100bd6",
      "message_chars": 1229,
      "message_redacted": true,
      "name": "tests.test_replace_handling.test_replace_without_update",
      "status": "failure"
    },
    {
      "branch": "27501e100bd6",
      "message_chars": 1229,
      "message_redacted": true,
      "name": "tests.test_replace_handling.test_replace_with_update",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 563,
      "message_redacted": true,
      "name": "tests.test_core.test_invalid_json_produces_error_message",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 652,
      "message_redacted": true,
      "name": "tests.test_core.test_module_with_replace_field_uses_replace_version",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 527,
      "message_redacted": true,
      "name": "tests.test_errors.test_incomplete_json_truncated_mid_object",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 474,
      "message_redacted": true,
      "name": "tests.test_errors.test_invalid_timestamp_format",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 408,
      "message_redacted": true,
      "name": "tests.test_errors.test_trailing_comma_in_json",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 511,
      "message_redacted": true,
      "name": "tests.test_errors.test_emoji_in_module_path",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 513,
      "message_redacted": true,
      "name": "tests.test_errors.test_unicode_chinese_in_module_path",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 684,
      "message_redacted": true,
      "name": "tests.test_errors.test_module_with_all_fields_populated",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 694,
      "message_redacted": true,
      "name": "tests.test_errors.test_replace_uses_replacement_version_not_original",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 716,
      "message_redacted": true,
      "name": "tests.test_errors.test_replace_with_update_shows_replacement_update",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 753,
      "message_redacted": true,
      "name": "tests.test_harvest.test_run_explicit_default_style",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 385,
      "message_redacted": true,
      "name": "tests.test_errors.test_control_characters_in_path_rejected_by_json",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 753,
      "message_redacted": true,
      "name": "tests.test_harvest.test_run_default_style",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 753,
      "message_redacted": true,
      "name": "tests.test_harvest.test_run_nonexistent_style_fallback",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 703,
      "message_redacted": true,
      "name": "tests.test_harvest.test_invalid_timestamp_with_replace",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 656,
      "message_redacted": true,
      "name": "tests.test_harvest.test_new_version_with_replace",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 396,
      "message_redacted": true,
      "name": "tests.test_core.test_malformed_json_object",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 660,
      "message_redacted": true,
      "name": "tests.test_replace_timestamps.test_replace_no_update",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 684,
      "message_redacted": true,
      "name": "tests.test_replace_timestamps.test_replace_with_update_on_replace",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 753,
      "message_redacted": true,
      "name": "tests.test_harvest.test_run_markdown_style",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 697,
      "message_redacted": true,
      "name": "tests.test_replace_timestamps.test_replace_both_updates_replace_takes_precedence",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 358,
      "message_redacted": true,
      "name": "tests.test_harvest.test_run_with_malformed_json",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 727,
      "message_redacted": true,
      "name": "tests.test_replace_timestamps.test_replace_with_invalid_timestamp",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 682,
      "message_redacted": true,
      "name": "tests.test_replace_timestamps.test_multiple_modules_mixed_replace_scenarios",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 684,
      "message_redacted": true,
      "name": "tests.test_replace_timestamps.test_replace_indirect_module",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 518,
      "message_redacted": true,
      "name": "tests.test_replace_timestamps.test_markdown_style_with_replace",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 684,
      "message_redacted": true,
      "name": "tests.test_replace_timestamps.test_replace_no_timestamps_at_all",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 684,
      "message_redacted": true,
      "name": "tests.test_replace_timestamps.test_replace_different_path_fork_scenario",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 637,
      "message_redacted": true,
      "name": "tests.test_harvest.test_current_version_with_replace",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 708,
      "message_redacted": true,
      "name": "tests.test_replace_timestamps.test_direct_flag_filters_indirect_with_replace",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 742,
      "message_redacted": true,
      "name": "tests.test_replace_timestamps.test_update_flag_filter_with_replace_inconsistency",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 665,
      "message_redacted": true,
      "name": "tests.test_replace_timestamps.test_multiple_replace_modules_mixed_timestamp_validity",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 664,
      "message_redacted": true,
      "name": "tests.test_replace_timestamps.test_replace_with_local_path_empty_version",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 697,
      "message_redacted": true,
      "name": "tests.test_replace_timestamps.test_replace_with_update_only_on_main_ignored",
      "status": "failure"
    },
    {
      "branch": "0d7f74667e0b",
      "message_chars": 684,
      "message_redacted": true,
      "name": "tests.test_replace_timestamps.test_replace_missing_current_timestamp_has_update_timestamp",
      "status": "failure"
    },
    {
      "branch": "498523513ae5",
      "message_chars": 105,
      "message_redacted": true,
      "name": "eval.tests.test_executable_behavior.test_filters_exact_output[args2-default_table_direct_only.txt]",
      "status": "skipped"
    },
    {
      "branch": "498523513ae5",
      "message_chars": 83,
      "message_redacted": true,
      "name": "eval.tests.test_executable_behavior.test_unknown_style_falls_back_to_default",
      "status": "skipped"
    },
    {
      "branch": "498523513ae5",
      "message_chars": 105,
      "message_redacted": true,
      "name": "eval.tests.test_executable_behavior.test_filters_exact_output[args1-default_table_direct_only.txt]",
      "status": "skipped"
    },
    {
      "branch": "498523513ae5",
      "message_chars": 105,
      "message_redacted": true,
      "name": "eval.tests.test_executable_behavior.test_filters_exact_output[args0-default_table_update_only.txt]",
      "status": "skipped"
    },
    {
      "branch": "498523513ae5",
      "message_chars": 75,
      "message_redacted": true,
      "name": "eval.tests.test_executable_behavior.test_markdown_style_output_exact",
      "status": "skipped"
    },
    {
      "branch": "498523513ae5",
      "message_chars": 13,
      "message_redacted": true,
      "name": "eval.tests.test_executable_behavior.test_malformed_json_errors_nonzero",
      "status": "failure"
    },
    {
      "branch": "498523513ae5",
      "message_chars": 321,
      "message_redacted": true,
      "name": "eval.tests.test_executable_behavior.test_help_output_exact_match",
      "status": "failure"
    },
    {
      "branch": "498523513ae5",
      "message_chars": 13,
      "message_redacted": true,
      "name": "eval.tests.test_executable_behavior.test_unknown_flag_prints_usage_and_flag_error",
      "status": "failure"
    },
    {
      "branch": "8bcffc633c6d",
      "message_chars": 363,
      "message_redacted": true,
      "name": "eval.tests.test_externalized.test_ext_run_with_error_logs_unexpected_eof_and_still_exits_zero",
      "status": "failure"
    },
    {
      "branch": "8bcffc633c6d",
      "message_chars": 462,
      "message_redacted": true,
      "name": "eval.tests.test_externalized.test_ext_run_outputs_markdown_table",
      "status": "failure"
    },
    {
      "branch": "8bcffc633c6d",
      "message_chars": 462,
      "message_redacted": true,
      "name": "eval.tests.test_externalized.test_ext_run_outputs_default_table_default_style",
      "status": "failure"
    },
    {
      "branch": "8bcffc633c6d",
      "message_chars": 462,
      "message_redacted": true,
      "name": "eval.tests.test_externalized.test_ext_run_outputs_default_table_nil_style_equivalent",
      "status": "failure"
    },
    {
      "branch": "8bcffc633c6d",
      "message_chars": 462,
      "message_redacted": true,
      "name": "eval.tests.test_externalized.test_ext_run_outputs_default_table_nonexistent_style_falls_back",
      "status": "failure"
    }
  ],
  "failed_tests_omitted": 0,
  "status_counts": {
    "failure": 49,
    "passed": 288,
    "skipped": 5
  },
  "test_branch_errors": {},
  "test_branches": [
    "27501e100bd6",
    "0d7f74667e0b",
    "edbc2e256c13",
    "3fe36788fd45",
    "e0bceb24b9d4",
    "6ae19a5ce7b4",
    "b72326e507b1",
    "498523513ae5",
    "8bcffc633c6d"
  ],
  "test_records": 342,
  "warnings": []
}