{ "run_id": "carelessdept--ci-guard-mcp-claude-haiku-5-5-1792797963689250861", "date": "2026-10-23T23:32:13.742082+00:00", "repo": "carelessdept/ci-guard-mcp", "kind": "mcp", "source": "https://github.com/carelessdept/ci-guard-mcp", "commit": "11f7635d3c67081aeebcd0ffe133db043ea03b7b", "model": "claude-haiku-5-5", "model_version": "claude-haiku-5-5", "test_version": "readme-bdc1b9b3c0d2673aef87e076d14084a8123ddcf09d520c1ee2e364a37c3f4bb9", "pass": true, "outcome": "pass", "time_to_working_s": 340.82328678201884, "total_s": 370.05279755592346, "steps": [ "cd && pwd && ls -la && git status 2>&1 | head -3; id -u; python3 --version; cat README.md 2>/dev/null | head -200", "ls /inputs 2>&1; ls 2>&1; cat /etc/os-release | head -2", "cd && (git init -q 2>&1; git remote add origin https://github.com/carelessdept/ci-guard-mcp 2>&1; timeout 120 git fetch --depth 1 origin 11f7635d3c67081aeebcd0ffe133db043ea03b7b 2>&1 | tail -3; git reset --mixed FETCH_HEAD 2>&1 | tail -2; git log --oneline -1 2>&1); which node npm; node -v 2>&1; npm -v 2>&1; cat package.json | head -40", "(curl -fsSL https://deb.nodesource.com/setup_22.x -o /tmp/ns.sh && sudo -n bash /tmp/ns.sh && sudo -n apt-get install -y nodejs) 2>&1 | tail -5; node -v; npm -v", "cd && npm ci --ignore-scripts 2>&1 | tail -5 && npm run build 2>&1 | tail -5 && ls dist | head; node dist/cli.js scan . --format json --no-git > /tmp/scan-out.json 2>/tmp/scan-err.txt; echo rc=$?; head -c 600 /tmp/scan-out.json; cat /tmp/scan-err.txt | head -5", "mkdir -p /tmp/mcpclient && cat > /tmp/mcpclient/c.mjs <<'EOF'\nimport { spawn } from 'node:child_process';\nconst [,, tool, argsJson] = process.argv;\nconst p = spawn('node', ['','--mcp','--root',''], {stdio:['pipe','pipe','inherit']});\nlet buf=''; const waiters={};\np.stdout.on('data',d=>{buf+=d; let i; while((i=buf.indexOf('\\n'))>=0){const l=buf.slice(0,i); buf=buf.slice(i+1); if(l.trim()){const m=JSON.parse(l); if(m.id!==undefined && waiters[m.id]) waiters[m.id](m);}}});\nconst send=(o)=>p.stdin.write(JSON.stringify(o)+'\\n');\nconst req=(id,method,params)=>new Promise(r=>{waiters[id]=r; send({jsonrpc:'2.0',id,method,params});});\n(async()=>{\n const init=await req(1,'initialize',{protocolVersion:'2025-11-25',capabilities:{},clientInfo:{name:'probe',version:'0'}});\n console.log('INIT', JSON.stringify(init.result.serverInfo));\n send({jsonrpc:'2.0',method:'notifications/initialized'});\n if(!tool){const l=await req(2,'tools/list',{}); console.log(l.result.tools.map(t=>t.name).join(', '));}\n else {const r=await req(3,'tools/call',{name:tool,arguments:JSON.parse(argsJson||'{}')}); console.log(JSON.stringify(r.result??r.error).slice(0,1500));}\n p.stdin.end(); p.kill();\n})();\nEOF\ncd /tmp/mcpclient && timeout 60 node c.mjs && timeout 60 node c.mjs scan_repo '{\"path\":\".\"}' ; timeout 60 node c.mjs check_secret_leaks '{\"path\":\".\"}'", "mkdir -p /tmp/ciguard-fixture/.github/workflows && cat > /tmp/ciguard-fixture/.github/workflows/ci.yml <<'EOF'\nname: ci\non: [push]\npermissions:\n contents: read\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - run: echo \"${{ github.event.issue.title }}\"\nEOF\ncd && node dist/cli.js scan /tmp/ciguard-fixture --format json; echo rc=$?; grep -rn \"setInterval\\|cron\\|schedule\\|--once\\|--now\\|webhook\" src README.md | head; timeout 600 npm run check 2>&1 | tail -15; echo check_rc=$?", "cd && ls -la .coldpass; cat .gitignore; node dist/cli.js scan . --fail-on warn; echo rc=$?; node dist/cli.js scan /tmp/ciguard-fixture --format json | grep -c ruleId; grep -o \"Exit codes: \\*\\*0\\*\\* below threshold, \\*\\*1\\*\\* findings at or above threshold\" README.md; grep -o \"Responses contain text JSON plus \\`structuredContent\\` with the same report.\" README.md; grep -o \"^| \\`scan_repo\\` | All four auditors | \\`path?: string\\`, \\`includeIgnored?: boolean\\` |\" README.md", "cd && cat > probe.json <<'EOF'\n{\"kind\":\"mcp\",\"argv\":[\"node\",\"\",\"--mcp\",\"--root\",\"/tmp/ciguard-fixture\"],\"cwd\":\"\",\"tool\":\"scan_repo\",\"arguments\":{\"path\":\".\"},\"expected_text\":\"\\\"ruleId\\\":\\\"workflow/unpinned-action\\\"\"}\nEOF\ncat > claims.json <<'EOF'\n[\n {\"kind\":\"claim\",\"claim\":\"Exit code 1 when findings are at or above the threshold\",\"source\":\"Exit codes: **0** below threshold, **1** findings at or above threshold\",\"input\":\"/tmp/ciguard-fixture (workflow with script injection)\",\"expected\":\"exit 1 and the workflow/script-injection finding\",\"probe_kind\":\"cli\",\"argv\":[\"node\",\"\",\"scan\",\"/tmp/ciguard-fixture\",\"--format\",\"json\"],\"expected_returncode\":1,\"expected_text\":\"\\\"ruleId\\\": \\\"workflow/script-injection\\\"\",\"cwd\":\"\"},\n {\"kind\":\"claim\",\"claim\":\"MCP tool results include structuredContent with the same report\",\"source\":\"Responses contain text JSON plus `structuredContent` with the same report.\",\"input\":\"scan_repo on /tmp/ciguard-fixture\",\"expected\":\"report containing the workflow/unpinned-action finding\",\"probe_kind\":\"mcp\",\"argv\":[\"node\",\"\",\"--mcp\",\"--root\",\"/tmp/ciguard-fixture\"],\"tool\":\"scan_repo\",\"arguments\":{\"path\":\".\"},\"expected_text\":\"\\\"ruleId\\\":\\\"workflow/unpinned-action\\\"\",\"cwd\":\"\"}\n]\nEOF\ncat > examples.json <<'EOF'\n[\n {\"kind\":\"example\",\"claim\":\"Repository development check runs typecheck, tests and build\",\"source\":\"README.md Development and release\",\"input\":\"npm run check\",\"expected\":\"all 54 tests pass\",\"argv\":[\"npm\",\"run\",\"check\"],\"cwd\":\"\",\"env\":{},\"expected_text\":\"# fail 0\"},\n {\"kind\":\"example\",\"claim\":\"Scan the checkout with a warn threshold\",\"source\":\"README.md Development and release\",\"input\":\"node dist/cli.js scan . --fail-on warn\",\"expected\":\"0 errors, exit 0\",\"argv\":[\"node\",\"dist/cli.js\",\"scan\",\".\",\"--fail-on\",\"warn\"],\"cwd\":\"\",\"env\":{},\"expected_text\":\"0 errors\"}\n]\nEOF\ncat > cycles.json <<'EOF'\n{\"detected\":false,\"reason\":\"Inspected README.md and src/ for cron, schedule, setInterval, loop, webhook, --once and --now entry points; none exist. ci-guard-mcp is a one-shot CLI (scan), a one-shot GitHub Action, and a stdio MCP server that answers requests on demand. No product cycle to trigger.\",\"checks\":[]}\nEOF\nmkdir -p && cat > report.json <<'EOF'\n{\"outcome\":\"working\",\"fault_owner\":null,\"operation\":\"Install and run ci-guard-mcp from the pinned checkout: build, CLI scan, MCP scan_repo tool call, and the documented npm run check suite\",\"operation_scope\":\"Linux Ubuntu 24.04 guest; Node 22 (README requires Node 20+); local source checkout only, no npm publication, no browser or credentials needed\",\"steps\":[\"git init; git remote add origin https://github.com/carelessdept/ci-guard-mcp; git fetch --depth 1 origin 11f7635d3c67081aeebcd0ffe133db043ea03b7b; git reset --mixed FETCH_HEAD\",\"sudo NodeSource setup_22.x + apt-get install nodejs (Node v22.23.4, npm 10.9.9)\",\"npm ci --ignore-scripts\",\"npm run build\",\"node dist/cli.js scan . --format json\",\"node dist/cli.js scan . --fail-on warn\",\"node dist/cli.js scan /tmp/ciguard-fixture --format json (exit 1, real findings)\",\"node dist/cli.js --mcp --root /tmp/ciguard-fixture; tools/call scan_repo\",\"npm run check (54 tests pass)\"],\"deviations\":[\"Node.js was not installed in the guest; installed Node 22 from NodeSource via sudo apt-get (README requires Node 20+). No errors.\",\"Workspace was not a git repo; applied the supplied git init/fetch/reset pin steps.\",\"The supplied checker path /inputs/coldpass-preflight.py does not exist; used as the validator.\",\"Used a scratch fixture at /tmp/ciguard-fixture with a deliberately flawed workflow to exercise findings, since the repo itself has zero findings.\",\"No README 'npx' usage was run: the package is not published yet, so the local dist/cli.js path was used as the README documents for source checkouts.\"],\"stuck_step\":null,\"main_blocker\":null,\"plain_failure\":null,\"suggested_fix\":\"None required for this run. Optional: publish the package so the npx examples in the README can be run as written.\"}\nEOF\nfor f in probe claims examples cycles report; do python3 -m json.tool $f.json >/dev/null && echo \"$f ok\"; done; cd && = python3 -I --validate --replay 2>&1 | tail -40", "cd && sed -i 's/\"expected_text\":\"\\\\\"ruleId\\\\\":\\\\\"workflow\\/unpinned-action\\\\\"\"/\"expected_text\":\"workflow\\/unpinned-action\"/' probe.json && sed -i 's/\"expected_text\":\"\\\\\"ruleId\\\\\":\\\\\"workflow\\/unpinned-action\\\\\"\"/\"expected_text\":\"workflow\\/unpinned-action\"/' claims.json && grep -o '\"expected_text\":\"[^\"]*\"' probe.json claims.json && python3 -m json.tool probe.json >/dev/null && python3 -m json.tool claims.json >/dev/null && cd && = python3 -I --validate --replay 2>&1 | tail -c 1500", "cd && = python3 -I --validate --replay > /tmp/check-out.txt 2>&1; echo rc=$?; python3 -I -c \"\nimport json,sys\nd=json.loads(open('/tmp/check-out.txt').read().strip().splitlines()[-1])\nprint('valid:',d.get('valid')); print('problems:',d.get('problems'))\n\" ; grep -o '\"working\": [a-z]*' /tmp/check-out.txt | head; cat .coldpass/claims.json | python3 -m json.tool | grep expected_text", "pgrep -fa \"dist/cli.js\" || echo \"no servers running\"; ls " ], "stuck_step": null, "error_excerpt": null, "what_it_did": "Install and run ci-guard-mcp from the pinned checkout: build, CLI scan, MCP scan_repo tool call, and the documented npm run check suite", "suggested_fix": null, "tokens_in": 252492, "tokens_out": 11906, "cost_usd": 0.015150219999999999, "usage_complete": true, "checks": [ { "check": "operation", "status": "PASS", "details": { "working": true, "argv": [ "node", "", "--mcp", "--root", "/tmp/ciguard-fixture" ], "kind": "mcp", "serverInfo": { "name": "ci-guard-mcp", "version": "0.2.0" }, "tools": [ "scan_repo", "check_env_drift", "audit_github_workflows", "check_secret_leaks" ], "tool_schema_bytes": 2355, "tool_result": { "content": [ { "type": "text", "text": "{\"version\":\"0.2.0\",\"findings\":[{\"ruleId\":\"workflow/unpinned-action\",\"severity\":\"warn\",\"file\":\".github/workflows/ci.yml\",\"line\":9,\"message\":\"External action or reusable workflow is not pinned to an immutable digest.\",\"suggestion\":\"Pin actions to a full 40-character commit SHA and containers to a sha256 digest.\"},{\"ruleId\":\"workflow/script-injection\",\"severity\":\"error\",\"file\":\".github/workflows/ci.yml\",\"line\":10,\"message\":\"Untrusted GitHub context is interpolated directly into a shell script.\",\"suggestion\":\"Pass event fields through step env and use quoted shell variables.\"}],\"summary\":{\"error\":1,\"warn\":1,\"info\":0},\"filesScanned\":1}" } ], "structuredContent": { "version": "0.2.0", "findings": [ { "ruleId": "workflow/unpinned-action", "severity": "warn", "file": ".github/workflows/ci.yml", "line": 9, "message": "External action or reusable workflow is not pinned to an immutable digest.", "suggestion": "Pin actions to a full 40-character commit SHA and containers to a sha256 digest." }, { "ruleId": "workflow/script-injection", "severity": "error", "file": ".github/workflows/ci.yml", "line": 10, "message": "Untrusted GitHub context is interpolated directly into a shell script.", "suggestion": "Pass event fields through step env and use quoted shell variables." } ], "summary": { "error": 1, "warn": 1, "info": 0 }, "filesScanned": 1 }, "isError": false }, "tool_result_checked": true, "scope": "product operation" } } ], "safety": { "version": "z2x-1", "rules_version": "z2x-1", "summary": "No problems traced to this repository code in these checks.", "findings": [], "context": [], "limitations": [ "A short run cannot establish that code is safe.", "Connections show IP addresses; proxies can hide the final hostname.", "Tracing covers dispatched commands and their children, not the whole OS; guest root can interfere with tracing." ], "static": { "dependency_note": "OSV scanned supported lockfiles; unpinned dependencies and unsupported formats are outside this check." }, "evidence_url": "/safety/?run=carelessdept--ci-guard-mcp-claude-haiku-5-5-1792797963689250861", "evidence_file": "/data/safety/carelessdept--ci-guard-mcp-claude-haiku-5-5-1792797963689250861.json.gz" }, "agent_execution": "native_agent", "agent_runtime": "claude-code", "agent_runtime_version": "2.1.300 (Claude Code)", "tool_transport": "mcp-assigned-vm", "platform": "linux", "reasoning": "low", "service_tier": null, "resources": { "cpus": 2, "memory_mib": 4096, "disk_mib": 32768 }, "native_tool_calls": 12, "native_bundle": null, "architecture": null, "backend": "hyperv", "platform_details": { "backend": "hyperv", "os": "linux" }, "platform_conflicts": null, "sandbox_source": null, "source_pin": { "commit": "11f7635d3c67081aeebcd0ffe133db043ea03b7b", "url": "https://github.com/carelessdept/ci-guard-mcp" }, "purpose": "initial", "profile": { "model": "claude-haiku-5-5", "reasoning": "low" }, "fault": { "owner": null, "status": "resolved", "reason": null }, "fix_id": null, "verifies_run_id": null, "hosting": null, "operation_scope": "product operation", "tool_calls": 12, "guest_setup": null, "runtime": null, "runtime_version": null, "runtime_revision": null, "source_revision": null, "evidence_version": null, "failure_origin": null, "evidence_strength": "readme_checked", "works": "yes", "install_s": null, "first_use_s": null, "mcp": "local", "needs": [], "needs_which": null, "agent_interface": [ "mcp" ], "license": "MIT", "repo_url": "https://github.com/carelessdept/ci-guard-mcp", "cost_to_try": "free", "hardware": null, "what_failed": null, "local_or_hosted": "local", "cycle_checks": [ { "kind": "cycle", "claim": "Scheduled or looping main behavior", "source": "repository entry points", "input": "source inspection", "expected": "identify the product trigger", "verdict": "yes", "outcome": "Inspected README.md and src/ for cron, schedule, setInterval, loop, webhook, --once and --now entry points; none exist. ci-guard-mcp is a one-shot CLI (scan), a one-shot GitHub Action, and a stdio MCP server that answers requests on demand. No product cycle to trigger.", "evidence": "Inspected README.md and src/ for cron, schedule, setInterval, loop, webhook, --once and --now entry points; none exist. ci-guard-mcp is a one-shot CLI (scan), a one-shot GitHub Action, and a stdio MCP server that answers requests on demand. No product cycle to trigger.", "command": "" } ], "claims": [ { "kind": "claim", "claim": "Exit code 1 when findings are at or above the threshold", "source": "Exit codes: **0** below threshold, **1** findings at or above threshold", "input": "/tmp/ciguard-fixture (workflow with script injection)", "expected": "exit 1 and the workflow/script-injection finding", "command": "node scan /tmp/ciguard-fixture --format json", "verdict": "yes", "outcome": "Real input produced the expected result: \"ruleId\": \"workflow/script-injection\"", "evidence": "\"ruleId\": \"workflow/script-injection\",\n \"severity\": \"error\",\n \"file\": \".github/workflows/ci.yml\",\n \"line\": 10,\n \"message\": \"Untrusted GitHub context is interpolated directly into a shell script.\",\n \"suggestion\": \"Pass event fields through step env and use quoted shell variables.\"\n }\n ],\n \"summary\": {\n \"error\": 1,\n \"warn\": 1,\n \"info\": 0\n },\n \"filesScanned\": 1\n}\n", "returncode": 1, "source_checked": true, "source_file": "README.md" }, { "kind": "claim", "claim": "MCP tool results include structuredContent with the same report", "source": "Responses contain text JSON plus `structuredContent` with the same report.", "input": "{\"tool\": \"scan_repo\", \"arguments\": {\"path\": \".\"}}", "expected": "report containing the workflow/unpinned-action finding", "command": "node --mcp --root /tmp/ciguard-fixture tools/call {\"tool\": \"scan_repo\", \"arguments\": {\"path\": \".\"}}", "verdict": "yes", "outcome": "Real input produced the expected result: workflow/unpinned-action", "evidence": "workflow/unpinned-action\\\",\\\"severity\\\":\\\"warn\\\",\\\"file\\\":\\\".github/workflows/ci.yml\\\",\\\"line\\\":9,\\\"message\\\":\\\"External action or reusable workflow is not pinned to an immutable digest.\\\",\\\"suggestion\\\":\\\"Pin actions to a full 40-character commit SHA and containers to a sha256 digest.\\\"},{\\\"ruleId\\\":\\\"workflow/script-injection\\\",\\\"severity\\\":\\\"error\\\",\\\"file\\\":\\\".github/workflows/ci.yml\\\",\\\"line\\\":10,\\\"message\\\":\\\"Untrusted GitHub context is interpolated directly into a shell script.\\\",\\\"suggestion\\\":\\\"Pass event fields through step env and use quoted shell variables.\\\"}],\\\"summary\\\":{\\\"error", "source_checked": true, "source_file": "README.md" } ], "claims_note": null, "example_checks": [ { "kind": "example", "claim": "Repository development check runs typecheck, tests and build", "source": "README.md Development and release", "input": "npm run check", "command": "npm run check", "evidence": "ks\nok 53 - v0.2.0: detects direct secret interpolation and toJSON(secrets) log exfiltration in run blocks\n ---\n duration_ms: 0.770257\n type: 'test'\n ...\n# Subtest: v0.2.0: audits GitLab CI (.gitlab-ci.yml) for script injection, pwn requests, and unpinned includes/images\nok 54 - v0.2.0: audits GitLab CI (.gitlab-ci.yml) for script injection, pwn requests, and unpinned includes/images\n ---\n duration_ms: 2.086947\n type: 'test'\n ...\n1..54\n# tests 54\n# suites 0\n# pass 54\n# fail 0\n# cancelled 0\n# skipped 0\n# todo 0\n# duration_ms 17322.349656\n\n> ci-guard-mcp@0.2.0 build\n> tsc -p tsconfig.json", "deferred": false, "expected": "all 54 tests pass", "returncode": 0, "verdict": "yes", "outcome": "command completed successfully" }, { "kind": "example", "claim": "Scan the checkout with a warn threshold", "source": "README.md Development and release", "input": "node dist/cli.js scan . --fail-on warn", "command": "node dist/cli.js scan . --fail-on warn", "evidence": "50 files scanned \u00b7 0 errors \u00b7 0 warnings \u00b7 0 informational", "deferred": false, "expected": "0 errors, exit 0", "returncode": 0, "verdict": "yes", "outcome": "command completed successfully" } ], "does_what_it_says": "yes", "maintenance": { "last_commit_at": "2026-10-22T09:55:07Z", "commits_90d": 10, "merged_prs_90d": 0, "open_issues": 0, "open_prs": 0, "reply_share": null, "reply_threads": 0, "median_reply_hours": null, "last_release_at": "2026-10-22T09:55:14Z", "archived": false, "verdict": "maintained", "note": "actively maintained: last commit 1 days ago; 10 commits and 0 merged PRs in 90 days; 0 open issues, 0 open PRs; latest release 2026-10-22" }, "maintenance_verdict": "maintained", "maintenance_note": "actively maintained: last commit 1 days ago; 10 commits and 0 merged PRs in 90 days; 0 open issues, 0 open PRs; latest release 2026-10-22", "maintained": "active", "maintained_note": "last commit 1 days ago", "last_commit_at": "2026-10-22T09:55:09Z", "last_release_at": "2026-10-22T09:55:14Z", "open_issues": null, "answered_issues": null, "platforms_claimed": [ "linux", "mac", "windows" ], "topic": "mcp-server", "agent_interfaces": [ { "interface": "mcp", "evidence": "independent checker", "observed": true, "operation_checked": true } ], "agent_interfaces_note": "README metadata is a discovery hint, not pinned documentation proof. Observed interfaces were exercised in this run. An empty list means unknown.", "mcp_context": { "tool_count": 4, "schema_bytes": 2355, "tokens_approx": 589, "estimate_method": "ceil(compact UTF-8 tools/list JSON bytes / 4)", "scope": "returned tools/list page only; excludes client framing and tool results" }, "tested_at": "2026-10-23T23:32:13.742082+00:00", "tested_commit": "11f7635d3c67081aeebcd0ffe133db043ea03b7b", "tested_with": { "agent": "claude-code", "agent_version": "2.1.300 (Claude Code)", "model": "claude-haiku-5-5", "model_route": "claude-haiku-5-5", "reasoning": "low", "os": "linux", "backend": "hyperv", "transport": "mcp-assigned-vm" }, "setup_identity": { "repo": "carelessdept/ci-guard-mcp", "source": "https://github.com/carelessdept/ci-guard-mcp", "commit": "11f7635d3c67081aeebcd0ffe133db043ea03b7b", "source_revision": null, "os": "linux", "architecture": null, "backend": "hyperv", "model": "claude-haiku-5-5", "model_version": "claude-haiku-5-5", "agent_execution": "native_agent", "agent_runtime": "claude-code", "agent_runtime_version": "2.1.300 (Claude Code)", "tool_transport": "mcp-assigned-vm", "runtime": null, "runtime_version": null, "runtime_revision": null, "sandbox_source": null, "guest_setup": null, "source_pin": { "commit": "11f7635d3c67081aeebcd0ffe133db043ea03b7b", "url": "https://github.com/carelessdept/ci-guard-mcp" }, "platform_details": { "os": "linux", "backend": "hyperv" }, "platform_conflicts": null, "reasoning": "low", "service_tier": null, "resources": { "cpus": 2, "memory_mib": 4096, "disk_mib": 32768 }, "operation": "Install and run ci-guard-mcp from the pinned checkout: build, CLI scan, MCP scan_repo tool call, and the documented npm run check suite", "operation_scope": "product operation", "hosting": null, "kind": "mcp", "test_version": "readme-bdc1b9b3c0d2673aef87e076d14084a8123ddcf09d520c1ee2e364a37c3f4bb9", "evidence_version": null }, "other_model": null, "other_run_id": null, "models_agree": null, "furthest_model": null, "transcript_url": null, "report_url": "/reports/carelessdept--ci-guard-mcp-claude-haiku-5-5-1792797963689250861.json" }