{ "agent_trace": { "codex_logs_available_local_only": true, "raw_logs_published": false, "summary_path": "agent-summary.json" }, "codex_version": "codex-cli 0.131.0", "collected_at": "2026-05-19T20:18:00.009534+00:00", "docker_cpus": 16, "docker_memory": "30g", "eval": { "error_code": null, "failed_tests": [ "eval.tests.test_01_basic_invocation.TestBasicInvocation.test_no_arguments", "eval.tests.test_01_basic_invocation.TestBasicInvocation.test_invalid_flag", "eval.tests.test_01_basic_invocation.TestBasicInvocation.test_unknown_command", "eval.tests.test_01_basic_invocation.TestHelpSubcommand.test_help_command", "eval.tests.test_01_basic_invocation.TestHelpSubcommand.test_help_local", "eval.tests.test_01_basic_invocation.TestHelpSubcommand.test_help_server", "eval.tests.test_01_basic_invocation.TestLocalCommandHelp.test_local_help_flag", "eval.tests.test_01_basic_invocation.TestLocalCommandHelp.test_local_help_short_flag", "eval.tests.test_01_basic_invocation.TestServerCommandHelp.test_server_help_flag", "eval.tests.test_01_basic_invocation.TestServerCommandHelp.test_server_help_short_flag", "eval.tests.test_02_local_command_validation.TestLocalCommandOptionalArgs.test_local_with_local_host_flag", "eval.tests.test_02_local_command_validation.TestLocalCommandOptionalArgs.test_local_with_local_host_short_flag", "eval.tests.test_02_local_command_validation.TestLocalCommandOptionalArgs.test_local_with_port_flag", "eval.tests.test_02_local_command_validation.TestLocalCommandOptionalArgs.test_local_with_port_short_flag", "eval.tests.test_02_local_command_validation.TestLocalCommandOptionalArgs.test_local_with_secret_flag", "eval.tests.test_02_local_command_validation.TestLocalCommandOptionalArgs.test_local_with_secret_short_flag", "eval.tests.test_02_local_command_validation.TestLocalCommandPortValidation.test_local_invalid_port_negative", "eval.tests.test_02_local_command_validation.TestLocalCommandPortValidation.test_local_invalid_port_non_numeric", "eval.tests.test_02_local_command_validation.TestLocalCommandPortValidation.test_local_invalid_port_too_large", "eval.tests.test_02_local_command_validation.TestLocalCommandPortValidation.test_local_invalid_port_zero", "eval.tests.test_02_local_command_validation.TestLocalCommandPortValidation.test_local_valid_port_formats", "eval.tests.test_02_local_command_validation.TestLocalCommandRequired.test_local_missing_port_shows_error", "eval.tests.test_02_local_command_validation.TestLocalCommandRequired.test_local_missing_to_flag", "eval.tests.test_02_local_command_validation.TestLocalCommandRequired.test_local_no_args_shows_error", "eval.tests.test_02_local_command_validation.TestLocalCommandServerValidation.test_local_empty_server_address", "eval.tests.test_02_local_command_validation.TestLocalCommandServerValidation.test_local_invalid_server_connection", "eval.tests.test_02_local_command_validation.TestLocalCommandToShortFlag.test_local_with_to_short_flag", "eval.tests.test_03_server_command_validation.TestServerCommandBasic.test_server_no_args_attempts_start", "eval.tests.test_03_server_command_validation.TestServerCommandBindAddress.test_server_bind_addr_and_tunnels", "eval.tests.test_03_server_command_validation.TestServerCommandBindAddress.test_server_bind_addr_flag", "eval.tests.test_03_server_command_validation.TestServerCommandBindAddress.test_server_bind_addr_invalid_format", "eval.tests.test_03_server_command_validation.TestServerCommandBindAddress.test_server_bind_addr_ipv6", "eval.tests.test_03_server_command_validation.TestServerCommandBindAddress.test_server_bind_tunnels_flag", "eval.tests.test_03_server_command_validation.TestServerCommandCombinations.test_server_all_flags_together", "eval.tests.test_03_server_command_validation.TestServerCommandPortRange.test_server_both_port_flags", "eval.tests.test_03_server_command_validation.TestServerCommandPortRange.test_server_invalid_max_port_too_large", "eval.tests.test_03_server_command_validation.TestServerCommandPortRange.test_server_invalid_min_port_negative", "eval.tests.test_03_server_command_validation.TestServerCommandPortRange.test_server_invalid_port_range_reversed", "eval.tests.test_03_server_command_validation.TestServerCommandPortRange.test_server_max_port_flag", "eval.tests.test_03_server_command_validation.TestServerCommandPortRange.test_server_min_port_flag", "eval.tests.test_03_server_command_validation.TestServerCommandPortRange.test_server_port_range_equal", "eval.tests.test_03_server_command_validation.TestServerCommandSecret.test_server_with_empty_secret", "eval.tests.test_03_server_command_validation.TestServerCommandSecret.test_server_with_secret_flag", "eval.tests.test_03_server_command_validation.TestServerCommandSecret.test_server_with_secret_short_flag", "eval.tests.test_04_environment_variables.TestEnvironmentVariableValidation.test_local_invalid_port_in_env_var", "eval.tests.test_04_environment_variables.TestEnvironmentVariableValidation.test_server_empty_port_range_from_env_vars", "eval.tests.test_04_environment_variables.TestEnvironmentVariableValidation.test_server_invalid_min_port_in_env_var", "eval.tests.test_04_environment_variables.TestLocalEnvironmentVariables.test_local_all_env_vars", "eval.tests.test_04_environment_variables.TestLocalEnvironmentVariables.test_local_env_var_overridden_by_flag", "eval.tests.test_04_environment_variables.TestLocalEnvironmentVariables.test_local_port_from_env_var", "eval.tests.test_04_environment_variables.TestLocalEnvironmentVariables.test_local_secret_from_env_var", "eval.tests.test_04_environment_variables.TestLocalEnvironmentVariables.test_local_server_from_env_var", "eval.tests.test_04_environment_variables.TestServerEnvironmentVariables.test_server_all_env_vars", "eval.tests.test_04_environment_variables.TestServerEnvironmentVariables.test_server_env_var_overridden_by_flag", "eval.tests.test_04_environment_variables.TestServerEnvironmentVariables.test_server_max_port_from_env_var", "eval.tests.test_04_environment_variables.TestServerEnvironmentVariables.test_server_min_port_from_env_var", "eval.tests.test_04_environment_variables.TestServerEnvironmentVariables.test_server_secret_from_env_var", "eval.tests.test_05_server_client_integration.TestClientServerBasicInteraction.test_client_attempts_connection_to_server", "eval.tests.test_05_server_client_integration.TestClientServerWithAuthentication.test_client_no_secret_server_with_secret", "eval.tests.test_05_server_client_integration.TestClientServerWithAuthentication.test_client_server_matching_secrets", "eval.tests.test_05_server_client_integration.TestClientServerWithAuthentication.test_client_server_mismatched_secrets", "eval.tests.test_05_server_client_integration.TestClientServerWithAuthentication.test_client_with_secret_server_without_secret", "eval.tests.test_05_server_client_integration.TestServerPortAllocation.test_server_allocates_port_in_range", "eval.tests.test_05_server_client_integration.TestServerStartup.test_server_starts_successfully", "eval.tests.test_05_server_client_integration.TestServerStartup.test_server_with_bind_addr", "eval.tests.test_05_server_client_integration.TestServerStartup.test_server_with_custom_port_range", "eval.tests.test_05_server_client_integration.TestServerStartup.test_server_with_secret", "eval.tests.test_06_full_integration.TestClientErrorHandling.test_client_fails_on_invalid_local_port", "eval.tests.test_06_full_integration.TestClientErrorHandling.test_client_fails_on_unreachable_server", "eval.tests.test_06_full_integration.TestFullServerClientFlow.test_end_to_end_data_transfer", "eval.tests.test_06_full_integration.TestFullServerClientFlow.test_server_accepts_client_hello", "eval.tests.test_06_full_integration.TestMultipleClients.test_server_handles_multiple_sequential_clients", "eval.tests.test_06_full_integration.TestServerErrorHandling.test_server_accepts_client_with_correct_secret", "eval.tests.test_06_full_integration.TestServerErrorHandling.test_server_rejects_client_with_wrong_secret", "eval.tests.test_07_deep_server_coverage.TestAuthenticationCodePaths.test_client_with_various_secrets", "eval.tests.test_07_deep_server_coverage.TestAuthenticationCodePaths.test_server_with_various_secrets", "eval.tests.test_07_deep_server_coverage.TestClientCodePaths.test_client_connection_attempts", "eval.tests.test_07_deep_server_coverage.TestClientCodePaths.test_client_with_different_local_hosts", "eval.tests.test_07_deep_server_coverage.TestClientCodePaths.test_client_with_port_selection", "eval.tests.test_07_deep_server_coverage.TestServerCodePaths.test_server_new_and_configuration" ], "failed_tests_omitted": 214, "public_eval_path": "eval.json", "summary_path": "eval-summary.json", "test_branch_errors": { "183567ae769e": [ { "error_code": "results_read_failed", "error_details": "Error response from daemon: Could not find the file /workspace/eval/results.xml in container 9a73cea1164a1fc04e8a87d35cf6acba72d63dbd72a0d1d1bfae949abafd2d81" } ] }, "test_records": 450, "warnings": [] }, "host_machine": "x86_64", "host_system": "Linux", "inference_mode": "paper-prompt-nointernet", "instance_id": "ekzhang__bore.8e059cd", "metrics": { "almost_resolved": "False", "cached_input_tokens": "3102336", "calls": "75", "codex_version": "codex-cli 0.131.0", "created_at": "2026-05-19T01:09:04.434421+00:00", "docker_cpus": "16", "docker_memory": "30g", "error_code": "", "estimated_cost_usd": "3.4388", "evaluator_problem": "True", "host_machine": "x86_64", "host_system": "Linux", "inference_mode": "paper-prompt-nointernet", "input_tokens": "3272562", "instance_id": "ekzhang__bore.8e059cd", "long_context_calls": "0", "model": "gpt-5.5", "n_resolved_tests": "120", "n_system_errors": "0", "n_tests": "394", "n_warnings": "0", "output_tokens": "34550", "paper_compliant": "False", "pricing_source": "https://developers.openai.com/api/docs/models/gpt-5.5", "reasoning_effort": "xhigh", "reasoning_output_tokens": "9974", "resolved": "False", "run_name": "cpx62-paperprompt-xhigh-shard-0-of-5-20260518T232006Z-ekzhang-bore", "run_version": "20260518T232006Z", "score": "0.30456852791878175", "session_ended_at": "2026-05-19T01:19:31.944Z", "session_started_at": "2026-05-19T01:09:12.380Z", "test_branch_errors": "{\"183567ae769e\": [\"results_read_failed\"]}", "total_tokens": "3307112", "wall_clock_seconds": "619" }, "model": "gpt-5.5", "package": { "contents": [ ".", "./.git", "./.git/COMMIT_EDITMSG", "./.git/HEAD", "./.git/branches", "./.git/config", "./.git/description", "./.git/hooks", "./.git/hooks/applypatch-msg.sample", "./.git/hooks/commit-msg.sample", "./.git/hooks/fsmonitor-watchman.sample", "./.git/hooks/post-update.sample", "./.git/hooks/pre-applypatch.sample", "./.git/hooks/pre-commit.sample", "./.git/hooks/pre-merge-commit.sample", "./.git/hooks/pre-push.sample", "./.git/hooks/pre-rebase.sample", "./.git/hooks/pre-receive.sample", "./.git/hooks/prepare-commit-msg.sample", "./.git/hooks/push-to-checkout.sample", "./.git/hooks/update.sample", "./.git/index", "./.git/info", "./.git/info/exclude", "./.git/logs", "./.git/logs/HEAD", "./.git/logs/refs", "./.git/logs/refs/heads", "./.git/logs/refs/heads/master", "./.git/objects", "./.git/objects/19", "./.git/objects/19/2b2f12785e6cf4f8887727b84653a430cb70a1", "./.git/objects/29", "./.git/objects/29/ca7620bc2f4757f56f1a4ae10b3c13e4e7bc8f", "./.git/objects/64", "./.git/objects/64/29f4caa6155de7a10ecc57da5a37edcd31edd6", "./.git/objects/6b", "./.git/objects/6b/0a21a3ed9b3903de50333452da813cac4eefde", "./.git/objects/99", "./.git/objects/99/e546f741960ca8d00d1bf656b09e9b6613ccaf", "./.git/objects/a8", "./.git/objects/a8/08c9a808624e42480848fc927558bbf005d576", "./.git/objects/bb", "./.git/objects/bb/ac1843e77966dee9a474857b69e2f8800cd041", "./.git/objects/d8", "./.git/objects/d8/22cd56f8b53a3b74590fbf19563ec10e6c6f9f", "./.git/objects/fa", "./.git/objects/fa/df0af89979b02cf781c46a635d220c4452df1b", "./.git/objects/info", "./.git/objects/pack", "./.git/refs", "./.git/refs/heads", "./.git/refs/heads/master", "./.git/refs/tags", "./LICENSE", "./README.md", "./bore_cli.py", "./ci", "./ci/build.bash", "./ci/common.bash", "./ci/set_rust_version.bash", "./ci/test.bash", "./compile.sh", "./executable" ], "contents_omitted": 0, "submission_available_local_only": true }, "paper_compliant": false, "paper_mode": true, "reasoning_effort": "xhigh", "run_name": "cpx62-paperprompt-xhigh-shard-0-of-5-20260518T232006Z-ekzhang-bore", "run_version": "20260518T232006Z", "usage_audit": { "available_local_artifact": true, "public_path": "usage-audit.json" } }