{ "error_code": null, "error_details_chars": 0, "error_details_redacted": false, "executable_hash": "e7166be4c885a6c0ec43eb7793162c9e2dd63467392748f34ccada255618f664", "log": [ { "branch": "", "command": "rm -rf /workspace/* /workspace/.[!.]*", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 0, "output_redacted": true, "returncode": 0, "step": "wipe_workspace", "wall_time": 0.18764825598918833 }, { "branch": "", "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 0, "output_redacted": true, "returncode": 0, "step": "seed_git", "wall_time": 0.15799238299950957 }, { "branch": "", "command": "chmod +x ./compile.sh && ./compile.sh", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 0, "output_redacted": true, "returncode": 0, "step": "compile", "wall_time": 0.12663414899725467 }, { "branch": "", "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 0, "output_redacted": true, "returncode": 0, "step": "copy_executable", "wall_time": 0.11720249100471847 }, { "branch": "", "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 117, "output_redacted": true, "returncode": 0, "step": "hash_executable", "wall_time": 0.12653907999629155 }, { "branch": "", "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 222, "output_redacted": true, "returncode": 0, "step": "install_rerunfailures", "wall_time": 0.6679453989927424 }, { "branch": "", "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 0, "output_redacted": true, "returncode": 0, "step": "restore_executable", "wall_time": 0.11906542199722026 }, { "branch": "", "command": "sha256sum ./executable", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 79, "output_redacted": true, "returncode": 0, "step": "verify_executable_hash", "wall_time": 0.11504885400063358 }, { "branch": "", "command": "rm -f eval/results.xml results.xml", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 0, "output_redacted": true, "returncode": 0, "step": "clean_stale_results", "wall_time": 0.11960919901321176 }, { "branch": "", "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 0, "output_redacted": true, "returncode": 0, "step": "patch_timeout_method", "wall_time": 0.09924415400018916 }, { "branch": "", "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh", "exception_info_chars": 29, "exception_info_redacted": true, "output_original_chars": 0, "output_redacted": true, "returncode": -1, "step": "run_tests", "wall_time": 3600.1092659889982 }, { "branch": "", "command": "docker cp b1237cffdabee3bf080a1653d9e294f31243105090ca730ed4f294e0e514b844:/workspace/eval/results.xml /tmp/tmp1qk_1lfs.xml", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 157, "output_redacted": true, "returncode": 1, "step": "results_read", "wall_time": 0.15066064799611922 }, { "branch": "", "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 0, "output_redacted": true, "returncode": 0, "step": "restore_executable", "wall_time": 0.1376823839964345 }, { "branch": "", "command": "sha256sum ./executable", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 79, "output_redacted": true, "returncode": 0, "step": "verify_executable_hash", "wall_time": 0.13192725800035987 }, { "branch": "", "command": "rm -f eval/results.xml results.xml", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 0, "output_redacted": true, "returncode": 0, "step": "clean_stale_results", "wall_time": 0.12611405400093645 }, { "branch": "", "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 0, "output_redacted": true, "returncode": 0, "step": "patch_timeout_method", "wall_time": 0.13085700299416203 }, { "branch": "", "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh", "exception_info_chars": 29, "exception_info_redacted": true, "output_original_chars": 0, "output_redacted": true, "returncode": -1, "step": "run_tests", "wall_time": 3600.107202995001 }, { "branch": "", "command": "docker cp cd21a1b0a058bd1861b9a23256bb97131e331d62223f5d60411ee6464359385a:/workspace/eval/results.xml /tmp/tmpa74vvrn0.xml", "exception_info_chars": 0, "exception_info_redacted": false, "output_original_chars": 157, "output_redacted": true, "returncode": 1, "step": "results_read", "wall_time": 0.06011360899719875 } ], "log_entries_omitted": 0, "public_redactions": { "extra_text": "redacted", "log_output": "redacted", "long_extra_text": "truncated to 2000 chars" }, "solution_branch": "submission", "status_counts": { "not_run": 298 }, "test_branch_errors": { "bc90bea37ab6": [ { "error_code": "results_read_failed", "error_details": "Error response from daemon: Could not find the file /workspace/eval/results.xml in container cd21a1b0a058bd1861b9a23256bb97131e331d62223f5d60411ee6464359385a" } ] }, "test_branches": [ "bc90bea37ab6" ], "test_records": 298, "test_results": [ { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_command_name_count_mismatch_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_command_with_backslash_escape", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_command_with_double_quotes_preserved", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_command_with_multiple_pipes_and_redirects", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_command_with_shell_metacharacters", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_duplicate_parameter_names_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_empty_parameter_list_produces_no_benchmarks", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_multiple_commands_with_parameters", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_parameter_list_cartesian_product", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_parameter_scan_decimal_steps", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_parameter_scan_with_command_name_substitution", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_parameter_substitution_no_overlapping", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_parameter_substitution_with_unicode", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_parameter_with_special_shell_chars", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_shell_none_mode_direct_execution", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_timer_captures_cpu_time_and_memory", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_timer_reports_exit_codes", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_command_timer.test_timer_wall_clock_vs_cpu_time_difference", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_all_export_formats_with_comparison", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_asciidoc_export_format", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_comparison_formatting_with_high_variance", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_comparison_with_parameters", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_compute_relative_speed_with_zero_mean_time", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_compute_with_check_returns_none_for_zero_time", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_csv_export_with_comparison_columns", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_custom_reference_command_selection", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_faster_reference_ordering", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_fastest_of_function_correctness", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_json_export_structure_completeness", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_markdown_export_format", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_mean_time_ordering_consistency", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_orgmode_export_format", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_reference_equal_ordering_case", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_relative_ordering_field_values", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_relative_speed_calculation_structural_correctness", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_single_benchmark_no_comparison_summary", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_slower_reference_ordering", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_sort_order_command_vs_mean_time_structure", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_stddev_propagation_in_relative_speed", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_vastly_different_execution_times", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_comparison.test_zero_time_edge_case_handling", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_edge_cases.test_decimal_parameter_precision", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_edge_cases.test_empty_parameter_range_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_edge_cases.test_large_but_valid_parameter_range", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_edge_cases.test_parameter_range_single_value", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_edge_cases.test_parameter_range_with_negative_values", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_edge_cases.test_parameter_step_larger_than_range", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_edge_cases.test_parsedecimal_error_in_parameter_scan", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_edge_cases.test_parseint_error_in_parameter_scan", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_edge_cases.test_step_required_for_decimal_range", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_edge_cases.test_too_large_parameter_range_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_edge_cases.test_valid_decimal_range_with_step", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_edge_cases.test_zero_step_parameter_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_command_failure_without_ignore_flag", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_command_name_too_many_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_command_not_found_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_command_with_missing_parameter_substitution", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_command_with_special_shell_characters", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_conclude_command_count_mismatch_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_conflicting_runs_options_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_duplicate_parameter_names_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_empty_command_string", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_empty_export_path_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_empty_shell_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_export_file_invalid_path_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_fast_execution_time_warning", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_ignore_failure_invalid_exit_code_parsing", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_input_file_not_exist_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_invalid_min_benchmarking_time_float_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_invalid_option_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_invalid_output_policy_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_invalid_runs_integer_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_invalid_shell_parse_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_invalid_sort_method_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_invalid_style_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_invalid_time_unit_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_invalid_warmup_integer_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_long_command_line_handling", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_min_runs_greater_than_max_runs_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_no_command_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_nonzero_exit_code_warning", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_output_policy_count_mismatch_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_parameter_list_substitution", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_parameter_range_too_large_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_parameter_scan_empty_range_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_parameter_scan_floating_point_no_step_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_parameter_step_zero_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_prepare_command_count_mismatch_error", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_prepare_command_execution", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_errors_warnings.test_show_output_with_failing_command", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_asciidoc_export_no_divider_row", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_asciidoc_export_table_structure", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_asciidoc_export_time_unit", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_csv_export_always_uses_seconds", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_csv_export_basic_structure", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_csv_export_no_runs_field", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_csv_export_parameter_scan_range", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_csv_export_proper_escaping", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_csv_export_with_parameters", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_csv_multiple_commands_order", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_export_with_reference_command", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_json_export_command_with_unused_parameters_field", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_json_export_exit_codes", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_json_export_large_run_count", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_json_export_memory_usage", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_json_export_structure_validation", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_json_export_with_parameters", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_json_stddev_present", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_markdown_export_auto_time_unit_detection", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_markdown_export_pipe_escaping", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_markdown_export_relative_speed_display", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_markdown_export_sort_order", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_markdown_export_table_structure", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_markdown_export_time_unit_control", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_markdown_with_parameter_list", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_multiple_exports_same_run", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_orgmode_export_first_column_spacing", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_orgmode_export_table_structure", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_exports.test_orgmode_export_time_unit", "status": "not_run" }, { "branch": "bc90bea37ab6", "extra": {}, "name": "tests.test_harvest.test_can_ignore_multiple_exit_codes", "status": "not_run" } ], "test_results_omitted": 178, "test_results_policy": "non-passed records only, capped for public artifact size", "warnings": [] }