{ "run_id": "TayHCode--search-first-gpt-6-luna-1792614702836516964", "date": "2026-10-21T20:33:34.646256+00:00", "repo": "TayHCode/search-first", "kind": "mcp", "source": "https://github.com/TayHCode/search-first", "commit": "2b7c5fd54b96bdd2a736eb2ae25db35d2cb71d7d", "model": "gpt-6-luna", "model_version": "gpt-6-luna", "test_version": "readme-d03245928c6bf21a5c0246e4484c1a09a4c5155cda5d5b004f78645851ec6f83", "pass": true, "outcome": "pass", "time_to_working_s": 74.86808070295956, "total_s": 111.80970025062561, "steps": [ "pwd; vault-search 'search-first repository README setup MCP operation' 2>&1; ls -la; rg --files -g 'README*' -g 'AGENTS.md' -g 'package.json' -g 'pyproject.toml' -g '*.yml' -g '*.yaml' -g 'Makefile' -g 'Cargo.toml' -g 'go.mod' -g 'uv.lock' -g 'requirements*.txt' -g 'Dockerfile' -g 'mcp*' -g 'examples/**' -g '.github/**' | head -200", "cat AGENTS.md; cat README.md; cat pyproject.toml; find .github docs scripts skills src -type f -maxdepth 5 -print; /opt/homebrew/bin/python3 --version; git status --short --branch; git remote -v", "git init; git remote add origin https://github.com/TayHCode/search-first; git fetch --depth 1 origin 2b7c5fd54b96bdd2a736eb2ae25db35d2cb71d7d", "git reset --mixed FETCH_HEAD; /opt/homebrew/bin/python3 scripts/check_python.py; command -v uv || true; command -v brew || true; ls /opt/homebrew/bin | head -40;", "curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=\"$PWD/uv-bin\" UV_NO_MODIFY_PATH=1 sh", "./uv-bin/uvx --python 3.12 --from . search-first --help", "sed -n '1,240p' src/search_first/server.py; sed -n '1,200p' docs/examples/rate-limiter.json; sed -n '1,160p' docs/examples/rate-limiter-inspection.json; sed -n '1,180p' scripts/smoke.py", "./uv-bin/uvx --python 3.12 --from . search-first <<'EOF'\n{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"search_existing\",\"arguments\":{\"idea\":\"a terminal tool to close idle Chrome tabs automatically\",\"queries\":[\"chrome idle tab close\",\"CLI close inactive tabs\"],\"limit\":2}}}\nEOF", "sed -n '1,220p' ; sed -n '200,320p' src/search_first/server.py", "cat > /tmp/mcp_call.py <<'PY'\nimport asyncio\nfrom mcp import ClientSession, StdioServerParameters\nfrom mcp.client.stdio import stdio_client\nasync def main():\n p=StdioServerParameters(command='',args=['--python','3.12','--from','','search-first'])\n async with stdio_client(p) as (r,w):\n async with ClientSession(r,w) as s:\n await s.initialize()\n z=await s.call_tool('search_existing',{'idea':'a Python library for rate limiting outgoing HTTP requests','queries':['python rate limiter','http client rate limit'],'limit':2})\n print(z)\nasyncio.run(main())\nPY\n./uv-bin/uvx --python 3.12 --from . python /tmp/mcp_call.py", "mkdir -p .coldpass; cat > .coldpass/probe.json <<'EOF'\n{\"kind\":\"mcp\",\"argv\":[\"\",\"--python\",\"3.12\",\"--from\",\"\",\"search-first\"],\"cwd\":\"\",\"tool\":\"search_existing\",\"arguments\":{\"idea\":\"a Python library for rate limiting outgoing HTTP requests\",\"queries\":[\"python rate limiter\",\"http client rate limit\"],\"limit\":2},\"expected_text\":\"alisaifee/flask-limiter\"}\nEOF\ncat > .coldpass/claims.json <<'EOF'\n[{\"kind\":\"claim\",\"claim\":\"Search existing public GitHub projects for an idea and return a ranked shortlist.\",\"source\":\"The default response returns the top **8** candidates with name, URL, stars, last push, license, relevance, idea-term coverage, missing core terms/constraints and a one-line reason.\",\"input\":\"a Python library for rate limiting outgoing HTTP requests\",\"expected\":\"Ranked candidates and coverage metadata\",\"probe_kind\":\"mcp\",\"argv\":[\"\",\"--python\",\"3.12\",\"--from\",\"\",\"search-first\"],\"tool\":\"search_existing\",\"arguments\":{\"idea\":\"a Python library for rate limiting outgoing HTTP requests\",\"queries\":[\"python rate limiter\",\"http client rate limit\"],\"limit\":2},\"expected_text\":\"alisaifee/flask-limiter\",\"cwd\":\"\"}]\nEOF\ncat > .coldpass/examples.json <<'EOF'\n[{\"kind\":\"example\",\"claim\":\"Install and run documented local CLI help\",\"source\":\"README.md \u2014 Install and run\",\"input\":\"uvx --python 3.12 --from . search-first --help\",\"expected\":\"CLI usage and documented transport options\",\"argv\":[\"\",\"--python\",\"3.12\",\"--from\",\"\",\"search-first\",\"--help\"],\"cwd\":\"\",\"expected_text\":\"--transport {stdio,streamable-http}\"}]\nEOF\ncat > .coldpass/cycles.json <<'EOF'\n{\"detected\":false,\"reason\":\"Inspected README and CLI entry point: the product serves MCP requests on demand over stdio or Streamable HTTP and defines no scheduler, recurring task, webhook, or cycle trigger.\",\"checks\":[]}\nEOF\ncat > .coldpass/report.json <<'EOF'\n{\"outcome\":\"working\",\"fault_owner\":null,\"operation\":\"Installed the documented local uvx package entrypoint and called search_existing through an MCP stdio client.\",\"operation_scope\":\"Unauthenticated public GitHub repository search, limited to the first two query variants by product design.\",\"steps\":[\"Initialized Git metadata and checked out pinned commit 2b7c5fd54b96bdd2a736eb2ae25db35d2cb71d7d.\",\"Ran scripts/check_python.py with Python 3.14.8; it passed the Python 3.11+ requirement.\",\"Installed uv with the documented macOS installer.\",\"Ran uvx --python 3.12 --from . search-first --help.\",\"Called search_existing with a real idea and two variants; the result had 2 completed queries, 0 failed, and returned ranked candidates including alisaifee/flask-limiter.\",\"Replayed probe, README claim, example and cycle evidence using coldpass checker.\",\"README development checks and package build were not run; they are merge/development checks, not necessary for this operation.\",\"vault-search was unavailable in this VM (command not found).\"],\"deviations\":[],\"stuck_step\":null,\"main_blocker\":null,\"plain_failure\":null,\"suggested_fix\":null}\nEOF\nCOLDPASS_WORKSPACE= /opt/homebrew/bin/python3 -I --validate --replay", "python3 - <<'PY'\np='.coldpass/claims.json'\ns=open(p).read().replace('The default response returns the top **8** candidates with name, URL, stars, last push, license, relevance, idea-term coverage, missing core terms/constraints and a one-line reason.','The default response returns the top **8** candidates with name, URL, stars, last push, license, relevance, idea-term coverage, missing core terms/constraints and a one-line reason.')\n# quote must be contiguous literal markdown text from README; checker likely strips markdown formatting? locate exact excerpt\nopen(p,'w').write(s)\nPY\nCOLDPASS_WORKSPACE= /opt/homebrew/bin/python3 -I --validate --replay", "python3 - <<'PY'\nimport json\np='.coldpass/claims.json'; x=json.load(open(p)); x[0]['source']='The server always adds one variant derived from the idea; with fewer than 2 of yours it adds more, including a topic search.'; json.dump(x,open(p,'w'))\nPY\nCOLDPASS_WORKSPACE= /opt/homebrew/bin/python3 -I --validate --replay", "rg='' ; python3 - <<'PY'\nimport json\np='.coldpass/claims.json'; x=json.load(open(p)); x[0]['source']='The default response returns the top **8** candidates'; json.dump(x,open(p,'w'))\nPY\nCOLDPASS_WORKSPACE= /opt/homebrew/bin/python3 -I --validate --replay" ], "stuck_step": null, "error_excerpt": null, "what_it_did": "Installed the documented local uvx package entrypoint and called search_existing through an MCP stdio client.", "suggested_fix": null, "tokens_in": 474421, "tokens_out": 3620, "cost_usd": null, "usage_complete": true, "checks": [ { "check": "operation", "status": "PASS", "details": { "working": true, "argv": [ "", "--python", "3.12", "--from", "", "search-first" ], "kind": "mcp", "serverInfo": { "name": "search-first", "version": "0.1.0" }, "tools": [ "search_existing", "inspect_repo", "compare_candidates" ], "tool_schema_bytes": 4129, "tool_result": { "content": [ { "text": "{\n \"verdict\": {\n \"closest_match\": {\n \"name\": \"alisaifee/flask-limiter\",\n \"url\": \"https://github.com/alisaifee/flask-limiter\"\n },\n \"plausible_candidates\": 15,\n \"adjacent_candidates\": 40,\n \"recommendation\": \"use as is\",\n \"reasoning\": \"Strong metadata overlap and a recognized license. Inspect README and test the required behavior before deciding that it works unchanged.\",\n \"provisional\": true,\n \"search_complete\": true\n },\n \"warnings\": [],\n \"candidates\": [\n {\n \"name\": \"alisaifee/flask-limiter\",\n \"url\": \"https://github.com/alisaifee/flask-limiter\",\n \"stars\": 1207,\n \"last_push\": \"2026-05-10T23:07:49Z\",\n \"license\": \"MIT\",\n \"relevance_score\": 0.937,\n \"idea_term_coverage\": 1.0,\n \"missing_key_terms\": [],\n \"missing_constraints\": [],\n \"why_it_matched\": \"Strong core match; Core terms: 2/2 (limit, rate); 1207 stars, push 163 days ago; returned by 1 query variant(s), best GitHub rank 1.\"\n },\n {\n \"name\": \"laurentS/slowapi\",\n \"url\": \"https://github.com/laurentS/slowapi\",\n \"stars\": 2070,\n \"last_push\": \"2026-10-21T19:50:53Z\",\n \"license\": \"MIT\",\n \"relevance_score\": 0.912,\n \"idea_term_coverage\": 1.0,\n \"missing_key_terms\": [],\n \"missing_constraints\": [],\n \"why_it_matched\": \"Strong core match; Core terms: 2/2 (limit, rate); 2070 stars, push 0 days ago; returned by 1 query variant(s), best GitHub rank 3.\"\n }\n ],\n \"unique_candidates_evaluated\": 60,\n \"errors\": [],\n \"message\": null,\n \"retry_after_seconds\": null,\n \"coverage\": {\n \"bounded\": true,\n \"truncated\": true,\n \"repositories_per_query\": 30,\n \"queries_completed\": 2,\n \"queries_failed\": 0,\n \"queries_skipped\": 1,\n \"query_budget\": 2,\n \"bounded_plan_complete\": true,\n \"results_skipped\": 58,\n \"unretrieved_results\": null,\n \"note\": \"Counts refer to deduplicated retrieved projects, not all of GitHub. Metadata-only lexical ranking; no candidate has been run or tested.\"\n },\n \"notes\": [\n \"Unauthenticated mode: at most 2 search variants per call; set GITHUB_TOKEN for complete searches. Shared-IP quota still applies.\"\n ]\n}", "type": "text" } ], "isError": false, "structuredContent": { "verdict": { "closest_match": { "name": "alisaifee/flask-limiter", "url": "https://github.com/alisaifee/flask-limiter" }, "plausible_candidates": 15, "adjacent_candidates": 40, "recommendation": "use as is", "reasoning": "Strong metadata overlap and a recognized license. Inspect README and test the required behavior before deciding that it works unchanged.", "provisional": true, "search_complete": true }, "warnings": [], "candidates": [ { "name": "alisaifee/flask-limiter", "url": "https://github.com/alisaifee/flask-limiter", "stars": 1207, "last_push": "2026-05-10T23:07:49Z", "license": "MIT", "relevance_score": 0.937, "idea_term_coverage": 1.0, "missing_key_terms": [], "missing_constraints": [], "why_it_matched": "Strong core match; Core terms: 2/2 (limit, rate); 1207 stars, push 163 days ago; returned by 1 query variant(s), best GitHub rank 1." }, { "name": "laurentS/slowapi", "url": "https://github.com/laurentS/slowapi", "stars": 2070, "last_push": "2026-10-21T19:50:53Z", "license": "MIT", "relevance_score": 0.912, "idea_term_coverage": 1.0, "missing_key_terms": [], "missing_constraints": [], "why_it_matched": "Strong core match; Core terms: 2/2 (limit, rate); 2070 stars, push 0 days ago; returned by 1 query variant(s), best GitHub rank 3." } ], "unique_candidates_evaluated": 60, "errors": [], "message": null, "retry_after_seconds": null, "coverage": { "bounded": true, "truncated": true, "repositories_per_query": 30, "queries_completed": 2, "queries_failed": 0, "queries_skipped": 1, "query_budget": 2, "bounded_plan_complete": true, "results_skipped": 58, "unretrieved_results": null, "note": "Counts refer to deduplicated retrieved projects, not all of GitHub. Metadata-only lexical ranking; no candidate has been run or tested." }, "notes": [ "Unauthenticated mode: at most 2 search variants per call; set GITHUB_TOKEN for complete searches. Shared-IP quota still applies." ] } }, "tool_result_checked": true, "scope": "product operation" } } ], "safety": { "version": "z2x-1", "rules_version": "z2x-1", "summary": "Review signals: 2 known vulnerability.", "findings": [ { "kind": "known-vulnerability", "count": 2, "note": "OSV reports a known vulnerability in this dependency version.", "location_count": 1, "path": "certifi" } ], "context": [], "limitations": [ "A short run cannot establish that code is safe.", "Connections show IP addresses; proxies can hide the final hostname.", "Tracing covers dispatched commands and their children, not the whole OS; guest root can interfere with tracing.", "macos behavior observation needs an OS-native file/network/process collector; only the static scan ran.", "No Linux syscall trace was collected." ], "static": { "dependency_note": "OSV scanned supported lockfiles; unpinned dependencies and unsupported formats are outside this check." }, "evidence_url": "/safety/?run=TayHCode--search-first-gpt-6-luna-1792614702836516964", "evidence_file": "/data/safety/TayHCode--search-first-gpt-6-luna-1792614702836516964.json.gz" }, "agent_execution": "native_agent", "agent_runtime": "codex", "agent_runtime_version": "codex-cli 0.163.0", "tool_transport": "mcp-assigned-vm", "platform": "macos", "reasoning": "low", "service_tier": "standard", "resources": { "cpus": null, "memory_mib": 4096, "disk_mib": 32768 }, "native_tool_calls": 14, "native_bundle": null, "architecture": null, "backend": "tart", "platform_details": { "backend": "tart", "os": "macos" }, "platform_conflicts": null, "sandbox_source": null, "source_pin": { "commit": "2b7c5fd54b96bdd2a736eb2ae25db35d2cb71d7d", "url": "https://github.com/TayHCode/search-first" }, "purpose": "initial", "profile": { "model": "gpt-6-luna", "reasoning": "low" }, "fault": { "owner": null, "status": "resolved", "reason": null }, "fix_id": null, "verifies_run_id": null, "hosting": null, "operation_scope": "product operation", "tool_calls": 14, "guest_setup": null, "runtime": null, "runtime_version": null, "runtime_revision": null, "source_revision": null, "evidence_version": null, "failure_origin": null, "evidence_strength": "readme_checked", "works": "yes", "install_s": null, "first_use_s": null, "mcp": "local", "needs": [], "needs_which": null, "agent_interface": [ "mcp" ], "license": "MIT", "repo_url": "https://github.com/TayHCode/search-first", "cost_to_try": "free", "hardware": null, "what_failed": null, "local_or_hosted": "local", "cycle_checks": [ { "kind": "cycle", "claim": "Scheduled or looping main behavior", "source": "repository entry points", "input": "source inspection", "expected": "identify the product trigger", "verdict": "yes", "outcome": "Inspected README and CLI entry point: the product serves MCP requests on demand over stdio or Streamable HTTP and defines no scheduler, recurring task, webhook, or cycle trigger.", "evidence": "Inspected README and CLI entry point: the product serves MCP requests on demand over stdio or Streamable HTTP and defines no scheduler, recurring task, webhook, or cycle trigger.", "command": "" } ], "claims": [ { "kind": "claim", "claim": "Search existing public GitHub projects for an idea and return a ranked shortlist.", "source": "The default response returns the top **8** candidates", "input": "{\"tool\": \"search_existing\", \"arguments\": {\"idea\": \"a Python library for rate limiting outgoing HTTP requests\", \"queries\": [\"python rate limiter\", \"http client rate limit\"], \"limit\": 2}}", "expected": "Ranked candidates and coverage metadata", "command": " --python 3.12 --from search-first tools/call {\"tool\": \"search_existing\", \"arguments\": {\"idea\": \"a Python library for rate limiting outgoing HTTP requests\", \"queries\": [\"python rate limiter\", \"http client rate limit\"], \"limit\": 2}}", "verdict": "yes", "outcome": "Real input produced the expected result: alisaifee/flask-limiter", "evidence": "alisaifee/flask-limiter\\\",\\n \\\"url\\\": \\\"https://github.com/alisaifee/flask-limiter\\\"\\n },\\n \\\"plausible_candidates\\\": 15,\\n \\\"adjacent_candidates\\\": 40,\\n \\\"recommendation\\\": \\\"use as is\\\",\\n \\\"reasoning\\\": \\\"Strong metadata overlap and a recognized license. Inspect README and test the required behavior before deciding that it works unchanged.\\\",\\n \\\"provisional\\\": true,\\n \\\"search_complete\\\": true\\n },\\n \\\"warnings\\\": [],\\n \\\"candidates\\\": [\\n {\\n \\\"name\\\": \\\"alisaifee/flask-limiter\\\",\\n \\\"url\\\": \\\"https://github.com/alisaifee/flask-limiter\\\",\\n ", "source_checked": true, "source_file": "README.md" } ], "claims_note": null, "example_checks": [ { "kind": "example", "claim": "Install and run documented local CLI help", "source": "README.md \u2014 Install and run", "input": "uvx --python 3.12 --from . search-first --help", "command": " --python 3.12 --from search-first --help", "evidence": " SEARCH_FIRST_AUTH_TOKEN (only behind a trusted\n gateway)\n --rank-weights RANK_WEIGHTS\n Ranking weights as name=value,... over the defaults\n (or SEARCH_FIRST_RANK_WEIGHTS); names: idea_match,\n query_match, github_rank, variant_agreement,\n freshness, stars\n --overall-weights OVERALL_WEIGHTS\n Compare blend weights as name=value,... (or\n SEARCH_FIRST_OVERALL_WEIGHTS); names: fit,\n maintenance, stars", "deferred": false, "expected": "CLI usage and documented transport options", "returncode": 0, "verdict": "yes", "outcome": "command completed successfully" } ], "does_what_it_says": "yes", "maintenance": { "archived": false, "commits_90d": 9, "last_commit_at": "2026-10-15T16:25:15Z", "last_release_at": null, "median_reply_hours": null, "merged_prs_90d": 2, "note": "actively maintained: last commit 6 days ago; 9 commits and 2 merged PRs in 90 days; 0 open issues, 0 open PRs; no releases", "open_issues": 0, "open_prs": 0, "reply_share": null, "reply_threads": 0, "verdict": "maintained" }, "maintenance_verdict": "maintained", "maintenance_note": "actively maintained: last commit 6 days ago; 9 commits and 2 merged PRs in 90 days; 0 open issues, 0 open PRs; no releases", "maintained": "active", "maintained_note": "last commit 6 days ago", "last_commit_at": "2026-10-15T16:44:45Z", "last_release_at": null, "open_issues": null, "answered_issues": null, "platforms_claimed": [ "linux", "mac", "windows" ], "topic": "mcp-server", "agent_interfaces": [ { "interface": "mcp", "evidence": "independent checker", "observed": true, "operation_checked": true } ], "agent_interfaces_note": "README metadata is a discovery hint, not pinned documentation proof. Observed interfaces were exercised in this run. An empty list means unknown.", "mcp_context": { "tool_count": 3, "schema_bytes": 4129, "tokens_approx": 1033, "estimate_method": "ceil(compact UTF-8 tools/list JSON bytes / 4)", "scope": "returned tools/list page only; excludes client framing and tool results" }, "tested_at": "2026-10-21T20:33:34.646256+00:00", "tested_commit": "2b7c5fd54b96bdd2a736eb2ae25db35d2cb71d7d", "tested_with": { "agent": "codex", "agent_version": "codex-cli 0.163.0", "model": "gpt-6-luna", "model_route": "gpt-6-luna", "reasoning": "low", "os": "macos", "backend": "tart", "transport": "mcp-assigned-vm" }, "setup_identity": { "repo": "TayHCode/search-first", "source": "https://github.com/TayHCode/search-first", "commit": "2b7c5fd54b96bdd2a736eb2ae25db35d2cb71d7d", "source_revision": null, "os": "mac", "architecture": null, "backend": "tart", "model": "gpt-6-luna", "model_version": "gpt-6-luna", "agent_execution": "native_agent", "agent_runtime": "codex", "agent_runtime_version": "codex-cli 0.163.0", "tool_transport": "mcp-assigned-vm", "runtime": null, "runtime_version": null, "runtime_revision": null, "sandbox_source": null, "guest_setup": null, "source_pin": { "commit": "2b7c5fd54b96bdd2a736eb2ae25db35d2cb71d7d", "url": "https://github.com/TayHCode/search-first" }, "platform_details": { "os": "macos", "backend": "tart" }, "platform_conflicts": null, "reasoning": "low", "service_tier": "standard", "resources": { "cpus": null, "memory_mib": 4096, "disk_mib": 32768 }, "operation": "Installed the documented local uvx package entrypoint and called search_existing through an MCP stdio client.", "operation_scope": "product operation", "hosting": null, "kind": "mcp", "test_version": "readme-d03245928c6bf21a5c0246e4484c1a09a4c5155cda5d5b004f78645851ec6f83", "evidence_version": null }, "other_model": null, "other_run_id": null, "models_agree": null, "furthest_model": null, "transcript_url": "/transcripts/TayHCode--search-first-gpt-6-luna-1792614702836516964.txt", "report_url": "/reports/TayHCode--search-first-gpt-6-luna-1792614702836516964.json" }