{ "agent_trace": { "codex_logs_available_local_only": true, "raw_logs_published": false }, "collected_at": "2026-05-18T13:25:14.667684+00:00", "docker_cpus": 16, "docker_memory": "30g", "eval": { "error_code": null, "failed_tests": [ "eval.tests.test_errors_only.test_errors_only_schema_validation", "eval.tests.test_http_options.test_http_options_with_structured_output", "eval.tests.test_output_formats.test_schema_validation_structured_output", "eval.tests.test_schema_validation.test_schema_validation_flag_output", "eval.tests.test_schema_validation.test_schema_validation_hierarchical_output", "eval.tests.test_schema_validation.test_invalid_schema_flag_output", "eval.tests.test_schema_validation.test_schema_validation_list_output", "tests.test_content.test_json_unquoted_key_invalid", "tests.test_content.test_base16_encoding_invalid", "tests.test_content.test_base64_with_whitespace_invalid", "tests.test_additional_properties.test_output_hierarchical_format", "tests.test_additional_properties_gap.test_pattern_properties_ap_false_multiple_unexpected_list", "tests.test_contains.test_contains_hierarchical_annotations_single_match", "tests.test_cli_args.test_invalid_json_in_schema", "tests.test_content.test_base64_encoding_invalid", "tests.test_cli_args.test_no_instances_list_output_valid", "tests.test_additional_properties.test_output_list_format_structured", "tests.test_errors.test_deeply_nested_json_recursion_limit", "tests.test_error_paths.test_special_characters_in_property_names", "tests.test_format_validation.test_ipv6_invalid_non_hex", "tests.test_content.test_json_single_quotes_invalid", "tests.test_content.test_base64_with_newline_invalid", "tests.test_content.test_content_media_type_json_invalid", "tests.test_draft4_minmax.test_error_list_output_includes_details", "tests.test_additional_properties_gap.test_pattern_properties_ap_false_hierarchical_annotations", "tests.test_ecma_regex.test_bell_character_rejection", "tests.test_additional_properties_gap.test_additional_properties_false_no_properties_defined_text", "tests.test_contains.test_contains_hierarchical_annotations_multiple_matches", "tests.test_cli_args.test_invalid_json_in_instance", "tests.test_format_validation.test_uri_invalid_spaces", "tests.test_content.test_base64url_encoding_rejects_base64_chars", "tests.test_cli_args.test_no_instances_hierarchical_output_valid", "tests.test_error_paths.test_unevaluated_properties_error_paths", "tests.test_format_validation.test_json_pointer_invalid_no_leading_slash", "tests.test_content.test_draft_6_applies_content_encoding", "tests.test_content.test_combined_encoding_invalid_then_media_type_not_checked", "tests.test_harvest.test_output_text_multiple_errors", "tests.test_harvest.test_instance_validation_with_invalid_schema_hierarchical_output", "tests.test_additional_properties_gap.test_properties_pattern_additional_properties_hierarchical_annotations", "tests.test_additional_properties_gap.test_additional_properties_false_no_properties_defined_list", "tests.test_format_validation.test_idn_email_invalid_format", "tests.test_contains.test_contains_hierarchical_non_array_empty_annotations", "tests.test_ecma_regex.test_hierarchical_output_with_pattern_error", "tests.test_cli_args.test_no_instances_list_output_invalid", "tests.test_content.test_base32_encoding_invalid", "tests.test_format_validation.test_date_time_with_lowercase_z", "tests.test_cli_args.test_no_instances_with_flag_output_valid", "tests.test_items.test_items_list_output_format_structure", "tests.test_content.test_draft_6_applies_content_media_type", "tests.test_content.test_combined_encoding_valid_media_type_invalid", "tests.test_format_validation.test_json_pointer_invalid_bad_escape", "tests.test_harvest.test_validate_invalid_schema_list_output", "tests.test_additional_properties_gap.test_additional_properties_false_no_properties_defined_hierarchical", "tests.test_format_validation.test_ipv6_invalid_too_many_colons", "tests.test_additional_properties_gap.test_properties_pattern_ap_false_hierarchical", "tests.test_ecma_regex.test_mixed_control_chars_and_classes", "tests.test_harvest.test_instance_validation_with_invalid_schema_structured_output", "tests.test_format_validation.test_uri_reference_valid_relative", "tests.test_cli_args.test_no_instances_hierarchical_output_invalid", "tests.test_content.test_base32hex_encoding_invalid", "tests.test_cli_args.test_no_instances_with_flag_output_invalid", "tests.test_items_specialized.test_items_integer_type_invalid_hierarchical_output", "tests.test_content.test_base16_odd_length_invalid", "tests.test_harvest.test_validate_invalid_schema_hierarchical_output", "tests.test_content.test_empty_string_invalid_json", "tests.test_additional_properties_gap.test_additional_properties_false_no_properties_multiple_props_text", "tests.test_format_validation.test_time_valid_leap_second", "tests.test_format_validation.test_duration_invalid_missing_p", "tests.test_harvest.test_instance_validation_with_invalid_schema_list_output", "tests.test_additional_properties_gap.test_overlapping_properties_pattern_violate_pattern_list", "tests.test_format_validation.test_uri_reference_valid_fragment_only", "tests.test_items.test_items_integer_draft4_strict_semantics", "tests.test_multiple_of.test_bigint_error_message_preserves_precision", "tests.test_error_paths.test_conditional_if_then_else_paths", "tests.test_ecma_regex.test_digit_class_translation_nomatch", "tests.test_draft4_minmax.test_draft4_integer_type_rejects_float_with_zero_fraction", "tests.test_items_specialized.test_items_integer_type_draft4_invalid_hierarchical_output", "tests.test_harvest.test_validate_schema_with_json_parse_error", "tests.test_additional_properties_gap.test_additional_properties_false_no_properties_multiple_props_hierarchical", "tests.test_format_validation.test_duration_invalid_empty" ], "failed_tests_omitted": 1223, "public_eval_path": "eval.json", "summary_path": "eval-summary.json", "test_branch_errors": {}, "test_records": 3005, "warnings": [] }, "host_machine": "x86_64", "host_system": "Linux", "inference_mode": "mini-swe-compatible-nointernet", "instance_id": "stranger6667__jsonschema.d52e881", "metrics": { "almost_resolved": "False", "cached_input_tokens": "9814912", "calls": "149", "created_at": "2026-05-17T15:49:18.373558+00:00", "docker_cpus": "16", "docker_memory": "30g", "error_code": "", "estimated_cost_usd": "8.1863", "evaluator_problem": "False", "host_machine": "x86_64", "host_system": "Linux", "inference_mode": "mini-swe-compatible-nointernet", "input_tokens": "10065083", "instance_id": "stranger6667__jsonschema.d52e881", "long_context_calls": "0", "model": "gpt-5.5", "n_resolved_tests": "1634", "n_system_errors": "0", "n_tests": "2933", "n_warnings": "0", "output_tokens": "67599", "paper_compliant": "False", "pricing_source": "https://developers.openai.com/api/docs/models/gpt-5.5", "reasoning_effort": "xhigh", "reasoning_output_tokens": "17404", "resolved": "False", "run_name": "cpx62-miniswecompat-xhigh-20260517T094700Z-stranger6667-jsonschema", "run_version": "20260517T094700Z", "score": "0.557108762359359", "session_ended_at": "2026-05-17T16:11:24.723Z", "session_started_at": "2026-05-17T15:49:21.424Z", "test_branch_errors": "{}", "total_tokens": "10132682", "wall_clock_seconds": "1323" }, "model": "gpt-5.5", "package": { "contents": [ ".", "./compile.sh", "./executable", "./jsonschema_cli.py" ], "contents_omitted": 0, "submission_available_local_only": true }, "paper_compliant": false, "paper_mode": false, "reasoning_effort": "xhigh", "run_name": "cpx62-miniswecompat-xhigh-20260517T094700Z-stranger6667-jsonschema", "run_version": "20260517T094700Z", "usage_audit": { "available_local_artifact": true, "public_path": "usage-audit.json" } }