{
  "error_code": null,
  "error_details_chars": 0,
  "error_details_redacted": false,
  "executable_hash": "1e2f2c2fe806d11a0b998ea547e3cbd6be8e21807f8d283526203c6dc31b10e0",
  "log": [
    {
      "branch": "",
      "command": "rm -rf /workspace/* /workspace/.[!.]*",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "wipe_workspace",
      "wall_time": 0.1995650500030024
    },
    {
      "branch": "",
      "command": "if [ ! -d .git ]; then GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c init.defaultBranch=gold init -q && git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false add -A && GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' git -c user.email=gold@local -c user.name=gold -c commit.gpgsign=false commit -q --allow-empty -m gold; fi",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "seed_git",
      "wall_time": 0.15703837499313522
    },
    {
      "branch": "",
      "command": "chmod +x ./compile.sh && ./compile.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "compile",
      "wall_time": 0.13983230899611954
    },
    {
      "branch": "",
      "command": "mv ./executable /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "copy_executable",
      "wall_time": 0.1301302299980307
    },
    {
      "branch": "",
      "command": "sha256sum /opt/programbench-stashed-executable-do-not-modify",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 117,
      "output_redacted": true,
      "returncode": 0,
      "step": "hash_executable",
      "wall_time": 0.11730037399684079
    },
    {
      "branch": "",
      "command": "pip3 install -q --disable-pip-version-check pytest-rerunfailures",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 222,
      "output_redacted": true,
      "returncode": 0,
      "step": "install_rerunfailures",
      "wall_time": 0.6293714649946196
    },
    {
      "branch": "",
      "command": "rm -f ./executable && mv /opt/programbench-stashed-executable-do-not-modify ./executable && chmod +x ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "restore_executable",
      "wall_time": 0.12647361800190993
    },
    {
      "branch": "",
      "command": "sha256sum ./executable",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 79,
      "output_redacted": true,
      "returncode": 0,
      "step": "verify_executable_hash",
      "wall_time": 0.12570352999318857
    },
    {
      "branch": "",
      "command": "rm -f eval/results.xml results.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "clean_stale_results",
      "wall_time": 0.11990595801034942
    },
    {
      "branch": "",
      "command": "test -f eval/run.sh && sed -i 's/--timeout-method=thread/--timeout-method=signal/g' eval/run.sh || true",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 0,
      "output_redacted": true,
      "returncode": 0,
      "step": "patch_timeout_method",
      "wall_time": 0.11704766600450967
    },
    {
      "branch": "",
      "command": "export PYTEST_ADDOPTS=\"$PYTEST_ADDOPTS --reruns=2 --reruns-delay=1\" && chmod +x ./eval/run.sh && ./eval/run.sh",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 860759,
      "output_redacted": true,
      "returncode": 0,
      "step": "run_tests",
      "wall_time": 102.46044465899467
    },
    {
      "branch": "5946c18eda4c",
      "command": "docker cp 8fe86fcd468d97fbeb39da7d9ad26ff33ac2b07c982df9f8ccf82310c2091bf1:/workspace/eval/results.xml /tmp/tmps3zcd5nt.xml",
      "exception_info_chars": 0,
      "exception_info_redacted": false,
      "output_original_chars": 797202,
      "output_redacted": true,
      "returncode": 0,
      "step": "results_read",
      "wall_time": 0.029051336998236366
    }
  ],
  "log_entries_omitted": 0,
  "public_redactions": {
    "extra_text": "redacted",
    "log_output": "redacted",
    "long_extra_text": "truncated to 2000 chars"
  },
  "solution_branch": "submission",
  "status_counts": {
    "failure": 670,
    "passed": 225
  },
  "test_branch_errors": {},
  "test_branches": [
    "5946c18eda4c"
  ],
  "test_records": 895,
  "test_results": [
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 41,
        "message_redacted": true,
        "time": 0.216
      },
      "name": "eval.tests.test_debug_query.test_debug_query_ast_without_color",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 41,
        "message_redacted": true,
        "time": 0.24
      },
      "name": "eval.tests.test_debug_query.test_debug_query_ast_go",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 473,
        "message_redacted": true,
        "time": 0.19
      },
      "name": "eval.tests.test_config_validation.test_undefined_metavar_in_transform_section",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 318,
        "message_redacted": true,
        "time": 0.2
      },
      "name": "eval.tests.test_errors.test_missing_required_rewrite",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 298,
        "message_redacted": true,
        "time": 0.212
      },
      "name": "eval.tests.test_error_diagnostics.test_parse_test_error",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 400,
        "message_redacted": true,
        "time": 0.231
      },
      "name": "eval.tests.test_completions.test_bash_completions_exact_golden",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 328,
        "message_redacted": true,
        "time": 0.218
      },
      "name": "eval.tests.test_fixer_expand.test_expand_with_stop_rule",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 111,
        "message_redacted": true,
        "time": 0.242
      },
      "name": "eval.tests.test_config_validation.test_update_all_with_multiple_files_atomic_updates",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 355,
        "message_redacted": true,
        "time": 0.229
      },
      "name": "eval.tests.test_harvest.test_file",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 324,
        "message_redacted": true,
        "time": 0.222
      },
      "name": "eval.tests.test_errors.test_invalid_strictness_value",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 259,
        "message_redacted": true,
        "time": 0.217
      },
      "name": "eval.tests.test_debug_query.test_debug_query_ast_var_decl",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 35,
        "message_redacted": true,
        "time": 0.179
      },
      "name": "eval.tests.test_custom_languages.test_custom_language_multiple_extensions",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 323,
        "message_redacted": true,
        "time": 0.181
      },
      "name": "eval.tests.test_harvest.test_scan_duplicate_default_ids",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 531,
        "message_redacted": true,
        "time": 0.157
      },
      "name": "eval.tests.test_file_filtering.test_glob_include_pattern",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 192,
        "message_redacted": true,
        "time": 0.153
      },
      "name": "eval.tests.test_harvest.test_sg_scan_html",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 276,
        "message_redacted": true,
        "time": 0.187
      },
      "name": "eval.tests.test_harvest.test_js_in_html",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 324,
        "message_redacted": true,
        "time": 0.199
      },
      "name": "eval.tests.test_fixer_expand.test_expand_no_matching_sibling",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 309,
        "message_redacted": true,
        "time": 0.239
      },
      "name": "eval.tests.test_error_diagnostics.test_rule_not_specified_stdin_scan",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 285,
        "message_redacted": true,
        "time": 0.215
      },
      "name": "eval.tests.test_errors.test_circular_symlink",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 479,
        "message_redacted": true,
        "time": 0.235
      },
      "name": "eval.tests.test_config_validation.test_duplicate_transform_variable_name",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 42,
        "message_redacted": true,
        "time": 0.206
      },
      "name": "eval.tests.test_debug_query.test_debug_query_sexp_typescript",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 291,
        "message_redacted": true,
        "time": 0.236
      },
      "name": "eval.tests.test_completions.test_zsh_completions_exact_golden",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 307,
        "message_redacted": true,
        "time": 0.212
      },
      "name": "eval.tests.test_debug_query.test_debug_query_cst_var_decl",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 1133,
        "message_redacted": true,
        "time": 0.216
      },
      "name": "eval.tests.test_harvest.test_max_diagnostics_shown",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 183,
        "message_redacted": true,
        "time": 0.209
      },
      "name": "eval.tests.test_core_edge_cases.test_kind_matcher_error_nodes",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 325,
        "message_redacted": true,
        "time": 0.172
      },
      "name": "eval.tests.test_errors.test_conflicting_flags_json_interactive",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 2439,
        "message_redacted": true,
        "time": 0.205
      },
      "name": "eval.tests.test_custom_languages.test_custom_language_malformed_yaml_config",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 60,
        "message_redacted": true,
        "time": 0.174
      },
      "name": "eval.tests.test_debug_query.test_debug_query_invalid_format_error",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 398,
        "message_redacted": true,
        "time": 0.182
      },
      "name": "eval.tests.test_file_filtering.test_glob_precedence_later_wins",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 205,
        "message_redacted": true,
        "time": 0.154
      },
      "name": "eval.tests.test_harvest.test_scan_unused_suppression",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 374,
        "message_redacted": true,
        "time": 0.17
      },
      "name": "eval.tests.test_harvest.test_sg_test_error",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 289,
        "message_redacted": true,
        "time": 0.162
      },
      "name": "eval.tests.test_harvest.test_inspect",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 374,
        "message_redacted": true,
        "time": 0.176
      },
      "name": "eval.tests.test_fixer_expand.test_expand_to_document_bounds",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 352,
        "message_redacted": true,
        "time": 0.189
      },
      "name": "eval.tests.test_errors.test_no_command_provided",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 41,
        "message_redacted": true,
        "time": 0.199
      },
      "name": "eval.tests.test_debug_query.test_debug_query_cst_python_indentation",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 211,
        "message_redacted": true,
        "time": 0.195
      },
      "name": "eval.tests.test_debug_query.test_debug_query_sexp_var_decl",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 372,
        "message_redacted": true,
        "time": 0.211
      },
      "name": "eval.tests.test_error_diagnostics.test_cannot_infer_shell_error",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 330,
        "message_redacted": true,
        "time": 0.218
      },
      "name": "eval.tests.test_core_edge_cases.test_kind_matcher_invalid_kind_name",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 426,
        "message_redacted": true,
        "time": 0.229
      },
      "name": "eval.tests.test_completions.test_fish_completions_exact_golden",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 479,
        "message_redacted": true,
        "time": 0.204
      },
      "name": "eval.tests.test_config_validation.test_fixer_list_missing_title_rejected",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 1176,
        "message_redacted": true,
        "time": 0.226
      },
      "name": "eval.tests.test_harvest.test_max_diagnostics_shown_single_file",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 98,
        "message_redacted": true,
        "time": 0.207
      },
      "name": "eval.tests.test_debug_query.test_debug_query_colored_output",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 147,
        "message_redacted": true,
        "time": 0.196
      },
      "name": "eval.tests.test_custom_languages.test_custom_language_missing_required_field",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 317,
        "message_redacted": true,
        "time": 0.182
      },
      "name": "eval.tests.test_file_filtering.test_follow_with_no_ignore_hidden_combines",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 2499,
        "message_redacted": true,
        "time": 0.201
      },
      "name": "eval.tests.test_errors.test_scan_with_malformed_yaml",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 410,
        "message_redacted": true,
        "time": 0.172
      },
      "name": "eval.tests.test_harvest.test_sg_test_filter",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 254,
        "message_redacted": true,
        "time": 0.174
      },
      "name": "eval.tests.test_harvest.test_debug_query",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 159,
        "message_redacted": true,
        "time": 0.217
      },
      "name": "eval.tests.test_harvest.test_label",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 247,
        "message_redacted": true,
        "time": 0.241
      },
      "name": "eval.tests.test_fixer_expand.test_expand_with_regex_matcher",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 404,
        "message_redacted": true,
        "time": 0.216
      },
      "name": "eval.tests.test_errors.test_invalid_config_file_path",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 41,
        "message_redacted": true,
        "time": 0.216
      },
      "name": "eval.tests.test_debug_query.test_debug_query_ast_java",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 288,
        "message_redacted": true,
        "time": 0.219
      },
      "name": "eval.tests.test_error_diagnostics.test_test_fail_with_details",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 98,
        "message_redacted": true,
        "time": 0.234
      },
      "name": "eval.tests.test_debug_query.test_debug_query_pattern_simple_metavar",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 475,
        "message_redacted": true,
        "time": 0.23
      },
      "name": "eval.tests.test_config_validation.test_rewriter_without_fix_rejected",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 498,
        "message_redacted": true,
        "time": 0.256
      },
      "name": "eval.tests.test_completions.test_powershell_completions_exact_golden",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 147,
        "message_redacted": true,
        "time": 0.192
      },
      "name": "eval.tests.test_custom_languages.test_custom_language_unknown_language_in_rule",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 234,
        "message_redacted": true,
        "time": 0.23
      },
      "name": "eval.tests.test_core_edge_cases.test_ellipsis_leading_captures_prefix",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 214,
        "message_redacted": true,
        "time": 0.211
      },
      "name": "eval.tests.test_debug_query.test_debug_query_default_format",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 199,
        "message_redacted": true,
        "time": 0.167
      },
      "name": "eval.tests.test_errors.test_scan_with_nonexistent_rule_directory",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 1162,
        "message_redacted": true,
        "time": 0.199
      },
      "name": "eval.tests.test_harvest.test_max_diagnostics_shown_stdin",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 302,
        "message_redacted": true,
        "time": 0.328
      },
      "name": "eval.tests.test_file_filtering.test_scan_command_respects_follow_flag",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 424,
        "message_redacted": true,
        "time": 0.348
      },
      "name": "eval.tests.test_harvest.test_sg_test_off_rule",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 199,
        "message_redacted": true,
        "time": 0.158
      },
      "name": "eval.tests.test_harvest.test_unsupport_config_arg",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 257,
        "message_redacted": true,
        "time": 0.196
      },
      "name": "eval.tests.test_language_features.test_css_in_html_style_tags",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 331,
        "message_redacted": true,
        "time": 0.18
      },
      "name": "eval.tests.test_fixer_expand.test_expand_start_on_first_item",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 2504,
        "message_redacted": true,
        "time": 0.198
      },
      "name": "eval.tests.test_errors.test_scan_with_malformed_rule_yaml",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 2498,
        "message_redacted": true,
        "time": 0.211
      },
      "name": "eval.tests.test_error_diagnostics.test_error_chain_formatting",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 195,
        "message_redacted": true,
        "time": 0.155
      },
      "name": "eval.tests.test_debug_query.test_debug_query_pattern_complex_metavars",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 45,
        "message_redacted": true,
        "time": 0.213
      },
      "name": "eval.tests.test_debug_query.test_debug_query_pattern_regex_metavar",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 433,
        "message_redacted": true,
        "time": 0.185
      },
      "name": "eval.tests.test_errors.test_missing_required_pattern",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 35,
        "message_redacted": true,
        "time": 0.215
      },
      "name": "eval.tests.test_custom_languages.test_custom_language_file_type_detection",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 227,
        "message_redacted": true,
        "time": 0.212
      },
      "name": "eval.tests.test_config_validation.test_update_all_modifies_files_in_place",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 251,
        "message_redacted": true,
        "time": 0.219
      },
      "name": "eval.tests.test_core_edge_cases.test_metavar_named_requires_argument",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 370,
        "message_redacted": true,
        "time": 0.205
      },
      "name": "eval.tests.test_completions.test_elvish_completions_exact_golden",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 41,
        "message_redacted": true,
        "time": 0.211
      },
      "name": "eval.tests.test_debug_query.test_debug_query_cst_go",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 355,
        "message_redacted": true,
        "time": 0.182
      },
      "name": "eval.tests.test_harvest.test_yaml_sgconfig_extension",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 235,
        "message_redacted": true,
        "time": 0.158
      },
      "name": "eval.tests.test_harvest.test_trace_default_project",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 307,
        "message_redacted": true,
        "time": 0.207
      },
      "name": "eval.tests.test_injection.test_static_injection_single_css_match",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 211,
        "message_redacted": true,
        "time": 0.182
      },
      "name": "eval.tests.test_file_filtering.test_nested_symlinks_no_follow_only_real_path",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 246,
        "message_redacted": true,
        "time": 0.195
      },
      "name": "eval.tests.test_language_features.test_typescript_in_html_with_lang_attribute",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 328,
        "message_redacted": true,
        "time": 0.208
      },
      "name": "eval.tests.test_fixer_expand.test_expand_end_on_last_item",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 290,
        "message_redacted": true,
        "time": 0.238
      },
      "name": "eval.tests.test_file_filtering.test_default_no_follow_symlinks",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 245,
        "message_redacted": true,
        "time": 0.226
      },
      "name": "eval.tests.test_debug_query.test_debug_query_pattern_multi_capture",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 324,
        "message_redacted": true,
        "time": 0.222
      },
      "name": "eval.tests.test_errors.test_pattern_that_doesnt_parse_in_target_language",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 201,
        "message_redacted": true,
        "time": 0.219
      },
      "name": "eval.tests.test_error_diagnostics.test_duplicate_rule_id_error",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 41,
        "message_redacted": true,
        "time": 0.159
      },
      "name": "eval.tests.test_debug_query.test_debug_query_cst_cpp_template",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 189,
        "message_redacted": true,
        "time": 0.208
      },
      "name": "eval.tests.test_core_edge_cases.test_metavar_optional_matches_empty",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 772,
        "message_redacted": true,
        "time": 0.205
      },
      "name": "eval.tests.test_config_validation.test_json_output_includes_fix_replacement_fields",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 433,
        "message_redacted": true,
        "time": 0.191
      },
      "name": "eval.tests.test_debug_query.test_debug_query_pattern_with_selector",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 451,
        "message_redacted": true,
        "time": 0.188
      },
      "name": "eval.tests.test_completions.test_completions_help_text",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 2334,
        "message_redacted": true,
        "time": 0.19
      },
      "name": "eval.tests.test_harvest.test_sg_scan_sarif_with_fixes",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 35,
        "message_redacted": true,
        "time": 0.185
      },
      "name": "eval.tests.test_custom_languages.test_custom_language_metavar_char_configuration",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 243,
        "message_redacted": true,
        "time": 0.173
      },
      "name": "eval.tests.test_harvest.test_trace_project",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 394,
        "message_redacted": true,
        "time": 0.18
      },
      "name": "eval.tests.test_fixer_expand.test_expand_start_neighbor",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 310,
        "message_redacted": true,
        "time": 0.165
      },
      "name": "eval.tests.test_injection.test_static_injection_multiple_css_matches",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 182,
        "message_redacted": true,
        "time": 0.208
      },
      "name": "eval.tests.test_language_features.test_languageglobs_eslintrc_as_js",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 316,
        "message_redacted": true,
        "time": 0.221
      },
      "name": "eval.tests.test_fixer_expand.test_expand_with_comma_and_whitespace",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 42,
        "message_redacted": true,
        "time": 0.223
      },
      "name": "eval.tests.test_debug_query.test_debug_query_sexp_jsx",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 274,
        "message_redacted": true,
        "time": 0.255
      },
      "name": "eval.tests.test_core_edge_cases.test_ellipsis_captures_multiline_arguments",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 293,
        "message_redacted": true,
        "time": 0.227
      },
      "name": "eval.tests.test_debug_query.test_debug_query_ast_python",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 41,
        "message_redacted": true,
        "time": 0.218
      },
      "name": "eval.tests.test_debug_query.test_debug_query_ast_python_multiline",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 770,
        "message_redacted": true,
        "time": 0.212
      },
      "name": "eval.tests.test_config_validation.test_sarif_output_includes_fix_artifacts",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 378,
        "message_redacted": true,
        "time": 0.204
      },
      "name": "eval.tests.test_completions.test_no_shell_argument_no_env_fails",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 546,
        "message_redacted": true,
        "time": 0.418
      },
      "name": "eval.tests.test_file_filtering.test_follow_flag_traverses_symlinked_directories",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 455,
        "message_redacted": true,
        "time": 0.17
      },
      "name": "eval.tests.test_errors.test_empty_pattern",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 171,
        "message_redacted": true,
        "time": 0.171
      },
      "name": "eval.tests.test_custom_languages.test_custom_language_case_sensitive_names",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 312,
        "message_redacted": true,
        "time": 0.179
      },
      "name": "eval.tests.test_errors.test_invalid_json_format",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 326,
        "message_redacted": true,
        "time": 0.156
      },
      "name": "eval.tests.test_injection.test_dynamic_injection_css_language_detection",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 337,
        "message_redacted": true,
        "time": 0.163
      },
      "name": "eval.tests.test_fixer_expand.test_expand_end_neighbor",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 428,
        "message_redacted": true,
        "time": 0.408
      },
      "name": "eval.tests.test_harvest.test_sg_scan",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 6961,
        "message_redacted": true,
        "time": 0.164
      },
      "name": "eval.tests.test_harvest.test_scan_multi_rule_file_auto_numbered_ids",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 191,
        "message_redacted": true,
        "time": 0.221
      },
      "name": "eval.tests.test_language_features.test_languageglobs_vue_file_with_js_extraction",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 362,
        "message_redacted": true,
        "time": 0.213
      },
      "name": "eval.tests.test_fixer_expand.test_expand_nested_functions",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 241,
        "message_redacted": true,
        "time": 0.241
      },
      "name": "eval.tests.test_error_diagnostics.test_build_globs_error",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 42,
        "message_redacted": true,
        "time": 0.207
      },
      "name": "eval.tests.test_debug_query.test_debug_query_sexp_nested_calls",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 82,
        "message_redacted": true,
        "time": 0.209
      },
      "name": "eval.tests.test_config_validation.test_fix_with_transform_applies_transformation",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 154,
        "message_redacted": true,
        "time": 0.192
      },
      "name": "eval.tests.test_debug_query.test_debug_query_sexp_rust",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 400,
        "message_redacted": true,
        "time": 0.237
      },
      "name": "eval.tests.test_completions.test_shell_env_var_bash_inference",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 296,
        "message_redacted": true,
        "time": 0.242
      },
      "name": "eval.tests.test_core_edge_cases.test_template_replacement_with_metavars",
      "status": "failure"
    },
    {
      "branch": "5946c18eda4c",
      "extra": {
        "message_chars": 263,
        "message_redacted": true,
        "time": 0.165
      },
      "name": "eval.tests.test_debug_query.test_debug_query_pattern_console_log",
      "status": "failure"
    }
  ],
  "test_results_omitted": 550,
  "test_results_policy": "non-passed records only, capped for public artifact size",
  "warnings": []
}