{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "evaluator_log_steps": [
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.13858021699707024
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.15532033899216913
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.13118696099263616
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.13718998699914664
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.1097019049921073
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6673535640002228
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.13611908299208153
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.13097622500208672
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11940542800584808
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.12629690101312008
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 5.186499420990003
    },
    {
      "branch": "2172e38aeb0e",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.037484789005247876
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.14497417699021753
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.1341631799878087
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.14389061299152672
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.14034872200863902
    },
    {
      "branch": "",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 55.858723223995185
    },
    {
      "branch": "8ebac061b133",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.029757839991361834
    }
  ],
  "evaluator_log_steps_omitted": 0,
  "executable_hash": "6ca338817a20eb5185f4a7d789760aad8d44cee789b70ae4f33cc1f4db6a5ceb",
  "failed_tests": [
    {
      "branch": "2172e38aeb0e",
      "message_chars": 1255,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.TestAutotune.test_autotune_invalid_metric",
      "status": "failure"
    },
    {
      "branch": "2172e38aeb0e",
      "message_chars": 407,
      "message_redacted": true,
      "name": "eval.tests.test_error_handling.TestArgumentValidation.test_quantize_with_unsupported_model_type",
      "status": "failure"
    },
    {
      "branch": "2172e38aeb0e",
      "message_chars": 754,
      "message_redacted": true,
      "name": "eval.tests.test_training.TestSupervisedTraining.test_supervised_invalid_loss_shows_error",
      "status": "failure"
    },
    {
      "branch": "2172e38aeb0e",
      "message_chars": 495,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.TestQuantize.test_quantize_creates_ftz_file",
      "status": "failure"
    },
    {
      "branch": "2172e38aeb0e",
      "message_chars": 7063,
      "message_redacted": true,
      "name": "eval.tests.test_training.TestSupervisedTraining.test_supervised_help_flag",
      "status": "failure"
    },
    {
      "branch": "2172e38aeb0e",
      "message_chars": 604,
      "message_redacted": true,
      "name": "eval.tests.test_training.TestTrainingArguments.test_ova_loss_function",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 34,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_with_f1_label_metric_optimizes_per_label",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 612,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_invalid_metric_name_fails_with_clear_error",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 630,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_invalid_label_in_metric_fails_at_runtime",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 220,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_with_different_modelsize_units",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 545,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_missing_validation_file_fails_with_clear_error",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 34,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_with_label_precision_at_recall_metric",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 568,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_zero_duration_fails_with_timeout_error",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 171,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_modelsize_constraint_creates_quantized_ftz_model",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 618,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_invalid_modelsize_format_fails_at_parse_time",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 34,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_with_predictions_parameter_affects_evaluation",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 34,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_precision_at_recall_metric_optimizes_correctly",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 436,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_empty_validation_file_produces_nan_scores",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 34,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_recall_at_precision_metric_works",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 228,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_basic_f1_metric_runs_trials_and_creates_model",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 295,
      "message_redacted": true,
      "name": "eval.tests.test_errors_edge_cases.test_empty_training_file_throws_empty_vocabulary_error",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 86,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_best_score_improves_or_stays_constant",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 330,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_with_wordngrams_flag_warns_not_optimized",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 274,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_creates_final_model_with_best_hyperparameters",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 96,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_progress_percentage_increases_monotonically",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 86,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_trial_count_increases_during_optimization",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 326,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_manually_set_hyperparameters_not_optimized",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 614,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_very_small_modelsize_constraint_fails_or_times_out",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 320,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_with_epoch_flag_warns_not_optimized",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 466,
      "message_redacted": true,
      "name": "eval.tests.test_errors_edge_cases.test_read_only_output_directory_error",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 77,
      "message_redacted": true,
      "name": "eval.tests.test_autotune.test_autotune_with_multiple_metrics_produces_different_models",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 158,
      "message_redacted": true,
      "name": "eval.tests.test_errors_edge_cases.test_test_command_with_mismatched_labels",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 608,
      "message_redacted": true,
      "name": "eval.tests.test_harvest.test_quantization_workflow",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 77,
      "message_redacted": true,
      "name": "eval.tests.test_model_io.test_bin_file_is_binary_format",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 456,
      "message_redacted": true,
      "name": "eval.tests.test_errors_edge_cases.test_invalid_loss_function_name_shows_valid_options",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 396,
      "message_redacted": true,
      "name": "eval.tests.test_errors_edge_cases.test_dimension_zero_trains_without_crash",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 449,
      "message_redacted": true,
      "name": "eval.tests.test_predict_eval.test_test_label_per_label_metrics",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 396,
      "message_redacted": true,
      "name": "eval.tests.test_errors_edge_cases.test_learning_rate_zero_trains_without_crash",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 48,
      "message_redacted": true,
      "name": "eval.tests.test_predict_eval.test_predict_prob_k_parameter",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 596,
      "message_redacted": true,
      "name": "eval.tests.test_errors_edge_cases.test_nn_command_with_zero_k_parameter",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 413,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_quantized_model_still_makes_predictions",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 388,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_dump_args_shows_training_configuration",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 389,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_dump_input_shows_input_matrix",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 389,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_dump_quantized_model_input_not_supported",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 530,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_quantize_basic_creates_ftz_file",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 390,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_dump_quantized_model_output_not_supported",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 380,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_quantize_with_qnorm_option",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 179,
      "message_redacted": true,
      "name": "eval.tests.test_model_io.test_model_metadata_in_dump_output",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 388,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_dump_dict_format_with_special_token",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 384,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_quantize_different_dsub_values",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 417,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_quantize_retrain_option_with_cutoff",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 389,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_quantize_cutoff_option_prunes_vocabulary",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 364,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_quantize_preserves_model_args",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 388,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_dump_dict_shows_vocabulary",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 390,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_dump_output_shows_output_matrix",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 388,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_dump_quantized_model_args_dict_work",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 364,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_quantize_creates_new_file_preserves_original",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 118,
      "message_redacted": true,
      "name": "eval.tests.test_training.test_skipgram_with_character_ngrams",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 71,
      "message_redacted": true,
      "name": "eval.tests.test_vectors_queries.test_analogies_basic@analogies_serial",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 118,
      "message_redacted": true,
      "name": "eval.tests.test_training.test_cbow_with_character_ngrams",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 397,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_quantize_with_qnorm_and_dsub",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 572,
      "message_redacted": true,
      "name": "eval.tests.test_vectors_queries.test_print_word_vectors_oov_word",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 35,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_dump_output_matrix_row_count_matches_labels",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 592,
      "message_redacted": true,
      "name": "eval.tests.test_vectors_queries.test_print_ngrams_basic",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 111,
      "message_redacted": true,
      "name": "eval.tests.test_training.test_supervised_with_learning_rate_variations",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 576,
      "message_redacted": true,
      "name": "eval.tests.test_vectors_queries.test_print_sentence_vectors_single_word",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 576,
      "message_redacted": true,
      "name": "eval.tests.test_vectors_queries.test_print_sentence_vectors_basic",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 725,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_quantize_only_works_on_supervised_models",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 35,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_dump_input_matrix_dimensions_match_words",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 435,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_dump_all_options_on_bin_model",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 364,
      "message_redacted": true,
      "name": "eval.tests.test_quantize_dump.test_quantize_then_test_accuracy_reasonable",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 572,
      "message_redacted": true,
      "name": "eval.tests.test_vectors_queries.test_print_word_vectors_multiword_input",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 576,
      "message_redacted": true,
      "name": "eval.tests.test_vectors_queries.test_print_sentence_vectors_empty_line",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 561,
      "message_redacted": true,
      "name": "eval.tests.test_vectors_queries.test_nn_oov_query",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 207,
      "message_redacted": true,
      "name": "eval.tests.test_vectors_queries.test_analogies_invalid_k",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 384,
      "message_redacted": true,
      "name": "eval.tests.test_vectors_queries.test_print_word_vectors_empty_input",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 556,
      "message_redacted": true,
      "name": "eval.tests.test_vectors_queries.test_nn_default_k",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 590,
      "message_redacted": true,
      "name": "eval.tests.test_vectors_queries.test_print_ngrams_short_word",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 561,
      "message_redacted": true,
      "name": "eval.tests.test_vectors_queries.test_nn_basic_query",
      "status": "failure"
    },
    {
      "branch": "8ebac061b133",
      "message_chars": 572,
      "message_redacted": true,
      "name": "eval.tests.test_vectors_queries.test_print_word_vectors_basic",
      "status": "failure"
    }
  ],
  "failed_tests_omitted": 6,
  "status_counts": {
    "failure": 86,
    "passed": 266
  },
  "test_branch_errors": {},
  "test_branches": [
    "2172e38aeb0e",
    "8ebac061b133"
  ],
  "test_records": 352,
  "warnings": []
}