{ "agent_trace": { "codex_logs_available_local_only": true, "raw_logs_published": false }, "collected_at": "2026-05-18T13:25:15.317805+00:00", "docker_cpus": 16, "docker_memory": "30g", "eval": { "error_code": null, "failed_tests": [ "tests.test_advanced_features.test_generate_with_complex_grammar", "tests.test_build.test_build_output_flag", "tests.test_build.test_build_verbose_flag", "tests.test_build.test_build_with_path_arg", "tests.test_advanced_features.test_build_with_different_flags", "tests.test_build.test_build_with_generated_parser", "tests.test_advanced_features.test_generate_disable_optimizations_detailed", "tests.test_integration.test_generate_error_invalid_grammar", "tests.test_integration.test_error_handling_missing_files", "tests.test_other_commands.test_tags_missing_file", "tests.test_other_commands.test_highlight_missing_file", "tests.test_parse.test_parse_dot_flag", "tests.test_build.test_build_debug_flag", "tests.test_build.test_build_reuse_allocator", "tests.test_end_to_end_coverage.test_generate_produces_all_files", "tests.test_extensive_generate.test_generate_report_all_states", "tests.test_extensive_generate.test_generate_complex_grammar", "tests.test_generate.test_generate_invalid_grammar", "tests.test_highlight_improved.test_highlight_missing_file", "tests.test_highlight_tags_deep.test_highlight_multiple_files", "tests.test_integration.test_full_workflow_generate_build_parse", "tests.test_main_cli_paths.test_parse_alias", "tests.test_integration.test_build_with_flags", "tests.test_main_cli_paths.test_command_aliases", "tests.test_tags_comprehensive.test_tags_missing_source_file", "tests.test_tags_improved.test_tags_missing_source_file", "tests.test_extensive_generate.test_generate_disable_optimizations", "tests.test_highlight_tags_deep.test_highlight_with_html_output", "tests.test_main_cli_paths.test_init_alias", "tests.test_fuzz.test_fuzz_log_graphs_env_enables_graph_logging", "tests.test_fuzz.test_fuzz_log_graphs_flag_enables_graph_logging", "tests.test_fuzz.test_fuzz_invalid_regex_include", "tests.test_dump_and_complete.test_dump_languages_nonexistent_config", "tests.test_fuzz.test_fuzz_exclude_filter_omits_specified_tests", "tests.test_harvest.test_corpus[reserved_words]", "tests.test_harvest.test_corpus[get_col_should_hang_not_crash]", "tests.test_build.test_build_simple_valid_grammar", "tests.test_generate_js_edge_cases.test_exports_property_assignment", "tests.test_harvest.test_corpus_expected_error[partially_resolved_conflict]", "tests.test_generate_gaps.test_generate_inline_rules_optimization", "tests.test_harvest.test_corpus_epsilon_external_tokens", "tests.test_generate_js_edge_cases.test_js_syntax_error_in_grammar", "tests.test_build.test_build_preserves_existing_files", "tests.test_harvest.test_corpus[anonymous_tokens_with_escaped_chars]", "tests.test_generate.test_generate_abi_version_14_default", "tests.test_generate.test_generate_with_log_flag_produces_debug_output", "tests.test_generate.test_generate_simple_grammar_creates_all_files", "tests.test_dump_and_complete.test_dump_languages_multiple_grammars_single_dir", "tests.test_fuzz.test_fuzz_invalid_regex_exclude", "tests.test_fuzz.test_fuzz_skip_flag_marks_test_as_skipped", "tests.test_harvest.test_corpus[start_rule_is_blank]", "tests.test_harvest.test_corpus[immediate_tokens]", "tests.test_harvest.test_corpus[associativity_left]", "tests.test_generate_gaps.test_generate_extras_whitespace_and_comments", "tests.test_generate_js_edge_cases.test_console_log_nested_arrays", "tests.test_harvest.test_corpus_external_and_internal_anonymous_tokens", "tests.test_generate_js_edge_cases.test_javascript_closures_and_scope", "tests.test_build.test_build_output_to_absolute_path", "tests.test_harvest.test_corpus_expected_error[precedence_on_single_child_missing]", "tests.test_generate.test_generate_multiple_rules_same_name_error", "tests.test_generate.test_generate_parser_c_has_expected_structure", "tests.test_build.test_build_creates_shared_library", "tests.test_generate.test_generate_combined_output_and_no_parser", "tests.test_dump_and_complete.test_complete_alias_comp", "tests.test_fuzz.test_fuzz_minimal_iteration_count", "tests.test_fuzz.test_fuzz_multiple_skip_entries", "tests.test_harvest.test_corpus[start_rule_is_token]", "tests.test_generate_gaps.test_generate_word_token_for_keyword_extraction", "tests.test_harvest.test_corpus_external_and_internal_tokens", "tests.test_harvest.test_corpus[inline_rules]", "tests.test_harvest.test_corpus[associativity_right]", "tests.test_dump_and_complete.test_complete_bash", "tests.test_generate_js_edge_cases.test_javascript_array_methods", "tests.test_generate_js_edge_cases.test_console_log_bigint", "tests.test_harvest.test_corpus_expected_error[terminal_supertype]", "tests.test_generate.test_generate_very_long_rule_name", "tests.test_generate.test_generate_grammar_json_is_valid_json", "tests.test_build.test_build_with_custom_output_path", "tests.test_generate.test_generate_grammar_with_inline_rules", "tests.test_dump_and_complete.test_complete_bash_contains_all_subcommands" ], "failed_tests_omitted": 731, "public_eval_path": "eval.json", "summary_path": "eval-summary.json", "test_branch_errors": {}, "test_records": 1690, "warnings": [] }, "host_machine": "x86_64", "host_system": "Linux", "inference_mode": "mini-swe-compatible-nointernet", "instance_id": "tree-sitter__tree-sitter.5e23cca", "metrics": { "almost_resolved": "False", "cached_input_tokens": "6169856", "calls": "106", "created_at": "2026-05-17T16:16:22.979347+00:00", "docker_cpus": "16", "docker_memory": "30g", "error_code": "", "estimated_cost_usd": "5.5094", "evaluator_problem": "False", "host_machine": "x86_64", "host_system": "Linux", "inference_mode": "mini-swe-compatible-nointernet", "input_tokens": "6324049", "instance_id": "tree-sitter__tree-sitter.5e23cca", "long_context_calls": "0", "model": "gpt-5.5", "n_resolved_tests": "516", "n_system_errors": "0", "n_tests": "1232", "n_warnings": "0", "output_tokens": "55116", "paper_compliant": "False", "pricing_source": "https://developers.openai.com/api/docs/models/gpt-5.5", "reasoning_effort": "xhigh", "reasoning_output_tokens": "15017", "resolved": "False", "run_name": "cpx62-miniswecompat-xhigh-20260517T094700Z-tree-sitter-tree-sitter", "run_version": "20260517T094700Z", "score": "0.41883116883116883", "session_ended_at": "2026-05-17T16:33:50.100Z", "session_started_at": "2026-05-17T16:16:26.041Z", "test_branch_errors": "{}", "total_tokens": "6379165", "wall_clock_seconds": "1044" }, "model": "gpt-5.5", "package": { "contents": [ ".", "./compile.sh", "./executable", "./tree_sitter_cli.py" ], "contents_omitted": 0, "submission_available_local_only": true }, "paper_compliant": false, "paper_mode": false, "reasoning_effort": "xhigh", "run_name": "cpx62-miniswecompat-xhigh-20260517T094700Z-tree-sitter-tree-sitter", "run_version": "20260517T094700Z", "usage_audit": { "available_local_artifact": true, "public_path": "usage-audit.json" } }