{ "run_id": "brave--brave-search-mcp-server-claude-haiku-5-5-1792808070937972573", "date": "2026-10-24T02:19:48.941216+00:00", "repo": "brave/brave-search-mcp-server", "kind": "mcp", "source": "https://github.com/brave/brave-search-mcp-server", "commit": "af08e0d48e812a3966ea2211854db5f0e8eee32d", "model": "claude-haiku-5-5", "model_version": "claude-haiku-5-5", "test_version": "readme-bdc1b9b3c0d2673aef87e076d14084a8123ddcf09d520c1ee2e364a37c3f4bb9", "pass": null, "outcome": "needs_credentials", "time_to_working_s": null, "total_s": 318.0032136440277, "steps": [ "cd && pwd && ls -la && id -u && cat /etc/os-release | head -3 && python3 --version && (git -C . rev-parse 2>&1 | head -1) && ls /inputs 2>&1 | head; ls 2>&1", "cd && cat README.md", "cd && (node --version; npm --version) 2>&1; ls -la .coldpass secrets; cat package.json; cat server.json | head -40; echo ---; cat | head -80", "cd && git init -q && git remote add origin https://github.com/brave/brave-search-mcp-server && git fetch --depth 1 origin af08e0d48e812a3966ea2211854db5f0e8eee32d 2>&1 | tail -3; git reset --mixed FETCH_HEAD 2>&1; git log --oneline -1; git status --short | head -20", "cd /tmp && export NODE_USE_ENV_PROXY=1; (curl -fsSL -o /tmp/node22.tar.xz https://nodejs.org/dist/latest-v22.x/node-v22.22.0-linux-x64.tar.xz || curl -fsSL -o /tmp/node22.tar.xz https://nodejs.org/dist/latest-v22.x/SHASUMS256.txt) 2>&1 | tail -3; ls -la /tmp/node22.tar.xz 2>&1; curl -fsSL https://nodejs.org/dist/latest-v22.x/SHASUMS256.txt 2>&1 | grep linux-x64.tar.xz", "mkdir -p /tmp/nodeinst && cd /tmp/nodeinst && export NODE_USE_ENV_PROXY=1 && curl -fsSL -o node.tar.xz https://nodejs.org/dist/latest-v22.x/node-v22.23.4-linux-x64.tar.xz && curl -fsSL -o SHASUMS256.txt https://nodejs.org/dist/latest-v22.x/SHASUMS256.txt && grep 'linux-x64.tar.xz' SHASUMS256.txt | sed 's/node-v22.23.4-linux-x64.tar.xz/node.tar.xz/' | sha256sum -c - && tar -xJf node.tar.xz && export PATH=/tmp/nodeinst/node-v22.23.4-linux-x64/bin:$PATH && node --version && npm --version", "sudo -n ln -sf /tmp/nodeinst/node-v22.23.4-linux-x64/bin/node /usr/local/bin/node && sudo -n ln -sf /tmp/nodeinst/node-v22.23.4-linux-x64/bin/npm /usr/local/bin/npm && sudo -n ln -sf /tmp/nodeinst/node-v22.23.4-linux-x64/bin/npx /usr/local/bin/npx && node --version && cd && export NODE_USE_ENV_PROXY=1 && npm ci 2>&1 | tail -15", "cd && npm run build 2>&1 | tail -5 && ls dist | head && npm test 2>&1 | tail -15", "cd && env -u BRAVE_API_KEY timeout 10 node dist/index.js &1 | head -30", "cd && grep -n \"credentials_error\\|'checks'\\|\\\"checks\\\"\\|def main\\|--validate\\|--replay\\|expected_file\\|mcp_http\" check.py | head -80", "cd && sed -n 800,860p check.py; sed -n 1165,1200p check.py; sed -n 1228,1260p check.py", "cd && grep -rniE \"setInterval|cron|schedule|webhook|--once|--now|setTimeout\" src --include=*.ts | grep -v test | head; echo \"---\"; ls .coldpass; grep -n \"Available Scripts\" -A 12 README.md | head -14", "cd && env | grep -c BRAVE_API || true; cat > probe.json <<'EOF'\n{\"checks\":[\n {\"check\":\"install\",\"kind\":\"cli\",\"argv\":[\"npm\",\"run\",\"build\"],\"cwd\":\"\",\"expected_returncode\":0,\"expected_text\":\"tsc\"},\n {\"check\":\"credentials_error\",\"kind\":\"cli\",\"argv\":[\"node\",\"dist/index.js\"],\"cwd\":\"\",\"stdin\":\"\",\"expected_returncode\":1,\"expected_text\":\"A Brave API key is required\"}\n]}\nEOF\ncat > claims.json <<'EOF'\n[\n {\"kind\":\"claim\",\"claim\":\"The README says the test suite is run with npm test; the suite passes on the pinned commit\",\"source\":\"- `npm test`: Run the test suite\",\"input\":\"npm test\",\"expected\":\"all tests pass, 0 failures\",\"probe_kind\":\"cli\",\"argv\":[\"npm\",\"test\"],\"cwd\":\"\",\"expected_text\":\"# fail 0\"},\n {\"kind\":\"claim\",\"claim\":\"README says npm run build builds the TypeScript project\",\"source\":\"- `npm run build`: Build the TypeScript project\",\"input\":\"npm run build\",\"expected\":\"TypeScript compiles to dist\",\"probe_kind\":\"cli\",\"argv\":[\"npm\",\"run\",\"build\"],\"cwd\":\"\",\"expected_text\":\"shx chmod +x dist/*.js\"},\n {\"kind\":\"claim\",\"claim\":\"The server needs a Brave Search API key to run\",\"source\":\"- `BRAVE_API_KEY`: Your Brave Search API key (required unless `BRAVE_API_KEY_FILE` is set)\",\"testable\":false,\"reason\":\"No Brave Search API key is available in this guest; obtaining one would require credentials, which were not requested. Startup without a key was observed to exit 1 with 'A Brave API key is required' (see probe credentials_error check).\"}\n]\nEOF\ncat > examples.json <<'EOF'\n[\n {\"kind\":\"example\",\"claim\":\"Run the documented test suite\",\"source\":\"README.md Development > Available Scripts\",\"input\":\"npm test\",\"expected\":\"78 tests, 0 failures\",\"argv\":[\"npm\",\"test\"],\"cwd\":\"\",\"env\":{},\"expected_text\":\"# fail 0\"},\n {\"kind\":\"example\",\"claim\":\"Local HTTP serving needs an API key; the documented serve:http script was not run because it requires BRAVE_API_KEY\",\"source\":\"README.md Development > Testing via MCP Inspector\",\"reason\":\"Brave Search API key is required by the server (BRAVE_API_KEY) and none was available; the documented npm run serve:http and MCP Inspector steps were not run. Startup without a key was checked instead (probe credentials_error).\"}\n]\nEOF\ncat > cycles.json <<'EOF'\n{\"detected\":false,\"reason\":\"Entry points inspected: README and package.json scripts (build, test, serve:http, inspector) and src/*.ts for setInterval/cron/schedule/webhook/--once/--now. The only timer is a poll delay (setTimeout pollInterval) inside the summarizer request in src/tools/summarizer/index.ts, which is not a scheduled cycle. The server runs on demand over stdio or HTTP.\",\"checks\":[]}\nEOF\npython3 -I -m json.tool probe.json >/dev/null && python3 -I -m json.tool claims.json >/dev/null && python3 -I -m json.tool examples.json >/dev/null && python3 -I -m json.tool cycles.json >/dev/null && echo OK", "cd && = python3 -I --validate --replay 2>&1 | head -c 6000", "cd && cat > claims.json <<'EOF'\n[\n {\"kind\":\"claim\",\"claim\":\"README says npm run build builds the TypeScript project\",\"source\":\"- `npm run build`: Build the TypeScript project\",\"input\":\"npm run build\",\"expected\":\"TypeScript compiles to dist\",\"probe_kind\":\"cli\",\"argv\":[\"npm\",\"run\",\"build\"],\"cwd\":\"\",\"expected_text\":\"shx chmod +x dist/*.js\"},\n {\"kind\":\"claim\",\"claim\":\"The server needs a Brave Search API key to run\",\"source\":\"- `BRAVE_API_KEY`: Your Brave Search API key (required unless `BRAVE_API_KEY_FILE` is set)\",\"testable\":false,\"reason\":\"No Brave Search API key is available in this guest; obtaining one would require credentials, which were not requested. Startup without a key was observed to exit 1 with 'A Brave API key is required' (see probe credentials_error check).\"}\n]\nEOF\npython3 -I -m json.tool claims.json >/dev/null && cd && = python3 -I --validate --replay > /tmp/validate1.json 2>&1; python3 -I -c \"\nimport json,sys\nd=json.load(open('/tmp/validate1.json'))\nprint('valid:',d['valid']); print('problems:',d['problems'])\nr=d.get('replay',{})\nprint('replay status:',r.get('status'), 'strength:', r.get('details',{}).get('working'), r.get('evidence_strength'))\n\" ", "cd && cat > report.json <<'EOF'\n{\"outcome\":\"needs_credentials\",\n \"fault_owner\":\"prerequisite\",\n \"operation\":\"Build the Brave Search MCP server from the README and run its stdio MCP server, then call a search tool (e.g. brave_web_search).\",\n \"operation_scope\":\"Pinned commit af08e0d48e812a3966ea2211854db5f0e8eee32d on Ubuntu 24.04 (node 22.23.4, npm 10.9.9). The build and unit tests were run. A real tool call was NOT made because the server requires a Brave Search API key (BRAVE_API_KEY), which is not available in this guest and was not obtained.\",\n \"steps\":[\n \"git init; git remote add origin https://github.com/brave/brave-search-mcp-server; git fetch --depth 1 origin af08e0d48e812a3966ea2211854db5f0e8eee32d; git reset --mixed FETCH_HEAD\",\n \"Installed Node.js 22.23.4 (README requires Node 22.x) from nodejs.org tarball, SHA256 verified against SHASUMS256.txt; linked into /usr/local/bin\",\n \"npm ci (runs prepare: format + build)\",\n \"npm run build (README Local Build) - succeeded\",\n \"npm test (README Development > Available Scripts) - 78 tests, 0 failures\",\n \"node dist/index.js (README Command Line Options / stdio default) with no API key - exits 1 with 'A Brave API key is required via --brave-api-key, BRAVE_API_KEY, ...'\"\n ],\n \"deviations\":[\n \"node/npm were not installed in the guest; installed Node 22.23.4 from the official nodejs.org tarball instead of the README (which gives no installer). Node 22.x is the README-required version.\",\n \"The Brave Search API key was not available, so the documented tool calls, serve:http, and MCP Inspector steps were not run. Only the missing-key startup error was checked.\",\n \"git init and fetch of the pin were run because the snapshot had no git metadata.\"\n ],\n \"stuck_step\":\"Running the server and calling a tool: node dist/index.js requires BRAVE_API_KEY (or --brave-api-key / BRAVE_API_KEY_FILE).\",\n \"main_blocker\":\"No Brave Search API key is available in this guest; the server refuses to start without one.\",\n \"plain_failure\":\"The server builds and its 78 tests pass, but it cannot actually search without a Brave Search API key. That key was not provided, so no search was run and the product's real output was not observed.\",\n \"suggested_fix\":\"Re-run with a Brave Search API key supplied as BRAVE_API_KEY (or BRAVE_API_KEY_FILE) in the guest, then run probe/claims for a real brave_web_search call. No change to the repository is needed.\"\n}\nEOF\npython3 -I -m json.tool report.json >/dev/null && echo OK && cd && = python3 -I --validate --replay 2>&1 | python3 -c \"import json,sys; d=json.load(sys.stdin); print(d['valid'], d['problems'])\"" ], "stuck_step": "Running the server and calling a tool: node dist/index.js requires BRAVE_API_KEY (or --brave-api-key / BRAVE_API_KEY_FILE).", "error_excerpt": "No Brave Search API key is available in this guest; the server refuses to start without one.", "what_it_did": "Build the Brave Search MCP server from the README and run its stdio MCP server, then call a search tool (e.g. brave_web_search).", "suggested_fix": "Re-run with a Brave Search API key supplied as BRAVE_API_KEY (or BRAVE_API_KEY_FILE) in the guest, then run probe/claims for a real brave_web_search call. No change to the repository is needed.", "tokens_in": 432344, "tokens_out": 10910, "cost_usd": 0.0172354, "usage_complete": true, "checks": [ { "status": "PASS", "details": { "working": false, "argv": [ "npm", "run", "build" ], "kind": "cli", "returncode": 0, "stdout": "\n> @brave/brave-search-mcp-server@2.1.4 build\n> shx rm -rf dist && tsc && shx chmod +x dist/*.js\n", "stderr": "", "needs_credentials": false, "needs_browser": false, "scope": "partial: install" }, "check": "install", "optional": false }, { "status": "PASS", "details": { "working": false, "argv": [ "node", "dist/index.js" ], "kind": "cli", "returncode": 1, "stdout": "", "stderr": "Error: A Brave API key is required via --brave-api-key, BRAVE_API_KEY, --brave-api-key-file, or BRAVE_API_KEY_FILE. You can get one at https://brave.com/search/api/.\nInvalid configuration", "needs_credentials": true, "needs_browser": false, "scope": "partial: missing credentials" }, "check": "credentials_error", "optional": false } ], "safety": { "version": "z2x-1", "rules_version": "z2x-1", "summary": "Review signals: 13 known vulnerability.", "findings": [ { "kind": "known-vulnerability", "count": 13, "note": "OSV reports a known vulnerability in this dependency version.", "location_count": 6 } ], "context": [], "limitations": [ "A short run cannot establish that code is safe.", "Connections show IP addresses; proxies can hide the final hostname.", "Tracing covers dispatched commands and their children, not the whole OS; guest root can interfere with tracing." ], "static": { "dependency_note": "OSV scanned supported lockfiles; unpinned dependencies and unsupported formats are outside this check." }, "evidence_url": "/safety/?run=brave--brave-search-mcp-server-claude-haiku-5-5-1792808070937972573", "evidence_file": "/data/safety/brave--brave-search-mcp-server-claude-haiku-5-5-1792808070937972573.json.gz" }, "agent_execution": "native_agent", "agent_runtime": "claude-code", "agent_runtime_version": "2.1.300 (Claude Code)", "tool_transport": "mcp-assigned-vm", "platform": "linux", "reasoning": "low", "service_tier": null, "resources": { "cpus": 2, "memory_mib": 4096, "disk_mib": 32768 }, "native_tool_calls": 16, "native_bundle": null, "architecture": null, "backend": "hyperv", "platform_details": { "backend": "hyperv", "os": "linux" }, "platform_conflicts": null, "sandbox_source": null, "source_pin": { "commit": "af08e0d48e812a3966ea2211854db5f0e8eee32d", "url": "https://github.com/brave/brave-search-mcp-server" }, "purpose": "test_rerun", "profile": { "model": "claude-haiku-5-5", "reasoning": "low" }, "fault": { "owner": "prerequisite", "status": "diagnosed", "reason": "No Brave Search API key is available in this guest; the server refuses to start without one." }, "fix_id": null, "verifies_run_id": "brave--brave-search-mcp-server-claude-haiku-5-5-1792805811632003343", "hosting": null, "operation_scope": "partial checks", "tool_calls": 16, "guest_setup": null, "runtime": null, "runtime_version": null, "runtime_revision": null, "source_revision": null, "evidence_version": null, "failure_origin": null, "evidence_strength": "partly_checked", "works": "partly", "install_s": null, "first_use_s": null, "mcp": "unknown", "needs": [ "credentials" ], "needs_which": "an account or key", "agent_interface": [], "license": "MIT", "repo_url": "https://github.com/brave/brave-search-mcp-server", "cost_to_try": "needs an account or key", "hardware": null, "what_failed": "No Brave Search API key is available in this guest; the server refuses to start without one.", "local_or_hosted": "both", "cycle_checks": [ { "kind": "cycle", "claim": "Scheduled or looping main behavior", "source": "repository entry points", "input": "source inspection", "expected": "identify the product trigger", "verdict": "yes", "outcome": "Entry points inspected: README and package.json scripts (build, test, serve:http, inspector) and src/*.ts for setInterval/cron/schedule/webhook/--once/--now. The only timer is a poll delay (setTimeout pollInterval) inside the summarizer request in src/tools/summarizer/index.ts, which is not a scheduled cycle. The server runs on demand over stdio or HTTP.", "evidence": "Entry points inspected: README and package.json scripts (build, test, serve:http, inspector) and src/*.ts for setInterval/cron/schedule/webhook/--once/--now. The only timer is a poll delay (setTimeout pollInterval) inside the summarizer request in src/tools/summarizer/index.ts, which is not a scheduled cycle. The server runs on demand over stdio or HTTP.", "command": "" } ], "claims": [ { "kind": "claim", "claim": "README says npm run build builds the TypeScript project", "source": "- `npm run build`: Build the TypeScript project", "input": "npm run build", "expected": "TypeScript compiles to dist", "command": "npm run build", "verdict": "yes", "outcome": "Real input produced the expected result: shx chmod +x dist/*.js", "evidence": "shx chmod +x dist/*.js\n\n", "returncode": 0, "source_checked": true, "source_file": "README.md" }, { "kind": "claim", "claim": "The server needs a Brave Search API key to run", "source": "- `BRAVE_API_KEY`: Your Brave Search API key (required unless `BRAVE_API_KEY_FILE` is set)", "input": "", "expected": "", "command": "", "verdict": "blocked", "outcome": "No Brave Search API key is available in this guest; obtaining one would require credentials, which were not requested. Startup without a key was observed to exit 1 with 'A Brave API key is required' (see probe credentials_error check).", "evidence": "No Brave Search API key is available in this guest; obtaining one would require credentials, which were not requested. Startup without a key was observed to exit 1 with 'A Brave API key is required' (see probe credentials_error check).", "source_checked": true, "source_file": "README.md" } ], "claims_note": null, "example_checks": [ { "kind": "example", "claim": "Run the documented test suite", "source": "README.md Development > Available Scripts", "input": "npm test", "command": "npm test", "evidence": "rom a file\n ---\n duration_ms: 3.037651\n type: 'test'\n ...\n # Subtest: rejects empty files\n ok 2 - rejects empty files\n ---\n duration_ms: 2.905552\n type: 'test'\n ...\n # Subtest: reports missing files with the path and reason\n ok 3 - reports missing files with the path and reason\n ---\n duration_ms: 0.247068\n type: 'test'\n ...\n 1..3\nok 25 - readBraveApiKeyFromFile\n ---\n duration_ms: 6.289499\n type: 'suite'\n ...\n1..25\n# tests 78\n# suites 26\n# pass 78\n# fail 0\n# cancelled 0\n# skipped 0\n# todo 0\n# duration_ms 25363.950692", "deferred": false, "expected": "78 tests, 0 failures", "returncode": 0, "verdict": "yes", "outcome": "command completed successfully" }, { "kind": "example", "claim": "Local HTTP serving needs an API key; the documented serve:http script was not run because it requires BRAVE_API_KEY", "source": "README.md Development > Testing via MCP Inspector", "input": "", "command": "", "evidence": "Brave Search API key is required by the server (BRAVE_API_KEY) and none was available; the documented npm run serve:http and MCP Inspector steps were not run. Startup without a key was checked instead (probe credentials_error).", "deferred": false, "expected": "documented command completes successfully", "verdict": "blocked", "outcome": "Brave Search API key is required by the server (BRAVE_API_KEY) and none was available; the documented npm run serve:http and MCP Inspector steps were not run. Startup without a key was checked instead (probe credentials_error)." } ], "does_what_it_says": "partly", "maintenance": { "archived": false, "commits_90d": 30, "last_commit_at": "2026-10-21T14:51:38Z", "last_release_at": "2026-09-17T14:47:14Z", "median_reply_hours": 404.8, "merged_prs_90d": 15, "note": "actively maintained: last commit 2 days ago; 30 commits and 15 merged PRs in 90 days; 25 open issues, 21 open PRs; 7 of 32 recent threads got a maintainer reply, median first reply 404.8h; latest release 2026-09-17", "open_issues": 25, "open_prs": 21, "reply_share": 0.22, "reply_threads": 32, "verdict": "maintained" }, "maintenance_verdict": "maintained", "maintenance_note": "actively maintained: last commit 2 days ago; 30 commits and 15 merged PRs in 90 days; 25 open issues, 21 open PRs; 7 of 32 recent threads got a maintainer reply, median first reply 404.8h; latest release 2026-09-17", "maintained": "slow", "maintained_note": "last commit 1 days ago; 5 open issues, 0 answered or closed", "last_commit_at": "2026-10-22T12:18:01Z", "last_release_at": "2026-09-17T14:47:14Z", "open_issues": 5, "answered_issues": 0, "platforms_claimed": [ "linux" ], "topic": "mcp-server", "agent_interfaces": [ { "interface": "mcp", "evidence": "README metadata", "observed": false, "operation_checked": false }, { "interface": "cli", "evidence": "independent checker", "observed": true, "operation_checked": false } ], "agent_interfaces_note": "README metadata is a discovery hint, not pinned documentation proof. Observed interfaces were exercised in this run. An empty list means unknown.", "mcp_context": null, "tested_at": "2026-10-24T02:19:48.941216+00:00", "tested_commit": "af08e0d48e812a3966ea2211854db5f0e8eee32d", "tested_with": { "agent": "claude-code", "agent_version": "2.1.300 (Claude Code)", "model": "claude-haiku-5-5", "model_route": "claude-haiku-5-5", "reasoning": "low", "os": "linux", "backend": "hyperv", "transport": "mcp-assigned-vm" }, "setup_identity": { "repo": "brave/brave-search-mcp-server", "source": "https://github.com/brave/brave-search-mcp-server", "commit": "af08e0d48e812a3966ea2211854db5f0e8eee32d", "source_revision": null, "os": "linux", "architecture": null, "backend": "hyperv", "model": "claude-haiku-5-5", "model_version": "claude-haiku-5-5", "agent_execution": "native_agent", "agent_runtime": "claude-code", "agent_runtime_version": "2.1.300 (Claude Code)", "tool_transport": "mcp-assigned-vm", "runtime": null, "runtime_version": null, "runtime_revision": null, "sandbox_source": null, "guest_setup": null, "source_pin": { "commit": "af08e0d48e812a3966ea2211854db5f0e8eee32d", "url": "https://github.com/brave/brave-search-mcp-server" }, "platform_details": { "os": "linux", "backend": "hyperv" }, "platform_conflicts": null, "reasoning": "low", "service_tier": null, "resources": { "cpus": 2, "memory_mib": 4096, "disk_mib": 32768 }, "operation": "Build the Brave Search MCP server from the README and run its stdio MCP server, then call a search tool (e.g. brave_web_search).", "operation_scope": "partial checks", "hosting": null, "kind": "mcp", "test_version": "readme-bdc1b9b3c0d2673aef87e076d14084a8123ddcf09d520c1ee2e364a37c3f4bb9", "evidence_version": null }, "transcript_url": null, "report_url": "/reports/brave--brave-search-mcp-server-claude-haiku-5-5-1792808070937972573.json" }