{ "checker": { "claims": [ { "claim": "Unit tests run without an API key", "command": "", "evidence": "The only offline test command (npm test, 292 unit tests) is a test suite, not a product operation, so it cannot verify a product claim. It is run as a README example in examples.json instead.", "expected": "", "input": "", "kind": "claim", "outcome": "The only offline test command (npm test, 292 unit tests) is a test suite, not a product operation, so it cannot verify a product claim. It is run as a README example in examples.json instead.", "source": "# unit tests, no API key needed", "source_checked": true, "source_file": "README.md", "verdict": "blocked" }, { "claim": "Live judgments need a provider API key", "command": "", "evidence": "No TYPESAFE_API_KEY, OPENROUTER_API_KEY or Cloudflare/AI gateway credential is available in the guest, and credentials were not sought. The jev_noul call returns the missing-key error instead (see probe credentials_error).", "expected": "", "input": "", "kind": "claim", "outcome": "No TYPESAFE_API_KEY, OPENROUTER_API_KEY or Cloudflare/AI gateway credential is available in the guest, and credentials were not sought. The jev_noul call returns the missing-key error instead (see probe credentials_error).", "source": "Requires Node.js 22 or newer and an API key for a Jev provider", "source_checked": true, "source_file": "README.md", "verdict": "blocked" }, { "claim": "HTTP server serves MCP at /mcp with a health check at /health", "command": "", "evidence": "Only the /health start check was exercised (see probe). A tool call on /mcp needs a provider key, so the judgment result cannot be verified.", "expected": "", "input": "", "kind": "claim", "outcome": "Only the /health start check was exercised (see probe). A tool call on /mcp needs a provider key, so the judgment result cannot be verified.", "source": "It listens on `PORT` (default `8080`) and serves MCP at `/mcp`, with a health check at `/health`.", "source_checked": true, "source_file": "README.md", "verdict": "blocked" } ], "cycle_checks": [ { "claim": "Scheduled or looping main behavior", "command": "", "evidence": "Inspected README.md and package.json entry points (bin dist/index.js, src/index.ts, src/http.ts). The server is an on-demand stdio or stateless HTTP MCP server: it starts on a client request and handles tool calls; there is no cron, loop, --once/--now flag, scheduler or webhook trigger in the README or source. The only background behavior is the HTTP listener, which was verified by a start check instead.", "expected": "identify the product trigger", "input": "source inspection", "kind": "cycle", "outcome": "Inspected README.md and package.json entry points (bin dist/index.js, src/index.ts, src/http.ts). The server is an on-demand stdio or stateless HTTP MCP server: it starts on a client request and handles tool calls; there is no cron, loop, --once/--now flag, scheduler or webhook trigger in the README or source. The only background behavior is the HTTP listener, which was verified by a start check instead.", "source": "repository entry points", "verdict": "yes" } ], "details": { "checks": [ { "check": "list_tools", "details": { "argv": [ "node", "dist/index.js", "--http" ], "hosting": "local", "kind": "mcp_http", "protocolVersion": "2025-06-18", "scope": "remote hosted MCP initialize + tools/list", "serverInfo": { "name": "jev-mcp", "version": "0.14.5" }, "tool_result_checked": false, "tool_schema_bytes": 24135, "tools": [ "jev_verify", "jev_screen", "jev_noul", "jev_find", "jev_classify", "jev_decide", "jev_rerank", "jev_compare", "jev_extract", "jev_audit", "jev_review", "jev_gate" ], "transport": "streamable-http", "url": "http://127.0.0.1:8080/mcp", "working": false }, "optional": false, "status": "PASS" }, { "check": "list_tools", "details": { "argv": [ "node", "dist/index.js" ], "kind": "mcp", "scope": "partial: list_tools", "serverInfo": { "name": "jev-mcp", "version": "0.14.5" }, "tool_result_checked": false, "tool_schema_bytes": 24135, "tools": [ "jev_verify", "jev_screen", "jev_noul", "jev_find", "jev_classify", "jev_decide", "jev_rerank", "jev_compare", "jev_extract", "jev_audit", "jev_review", "jev_gate" ], "working": false }, "optional": false, "status": "PASS" }, { "check": "credentials_error", "details": { "argv": [ "node", "dist/index.js" ], "credentials_error": "MCP tool reported missing credentials: {\"content\": [{\"type\": \"text\", \"text\": \"No TYPESAFE_API_KEY, OPENROUTER_API_KEY (sk-or-), Cloudflare token (CLOUDFLARE_API_TOKEN or JEV_CLOUDFLARE_API_TOKEN) + CLOUDFLARE_ACCOUNT_ID, or AI_GATEWAY_API_KEY found. Set one, or JEV_PROVIDER to choose explicitly.\"}], \"isError\": true}", "kind": "mcp", "needs_credentials": true, "serverInfo": { "name": "jev-mcp", "version": "0.14.5" }, "tool_result": { "content": [ { "text": "No TYPESAFE_API_KEY, OPENROUTER_API_KEY (sk-or-), Cloudflare token (CLOUDFLARE_API_TOKEN or JEV_CLOUDFLARE_API_TOKEN) + CLOUDFLARE_ACCOUNT_ID, or AI_GATEWAY_API_KEY found. Set one, or JEV_PROVIDER to choose explicitly.", "type": "text" } ], "isError": true }, "tool_schema_bytes": 24135, "tools": [ "jev_verify", "jev_screen", "jev_noul", "jev_find", "jev_classify", "jev_decide", "jev_rerank", "jev_compare", "jev_extract", "jev_audit", "jev_review", "jev_gate" ], "working": false }, "optional": false, "status": "PASS" } ], "scope": "partial checks", "working": false }, "evidence_strength": "partly_checked", "example_checks": [ { "claim": "Build the TypeScript server as documented", "command": "npm run build", "deferred": false, "evidence": "> @jkudish/jev-mcp@0.14.5 build\n> tsc", "expected": "tsc compiles to dist/ without errors", "input": "npm run build", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "README.md Development section", "verdict": "yes" }, { "claim": "Run documented offline unit tests", "command": "npm test", "deferred": false, "evidence": "hing echoed\n ---\n duration_ms: 51.602503\n type: 'test'\n ...\n# Subtest: runRegex settles immediately with an error when the request is already aborted\nok 291 - runRegex settles immediately with an error when the request is already aborted\n ---\n duration_ms: 0.950265\n type: 'test'\n ...\n# Subtest: runRegex terminates its worker when the signal aborts mid-run\nok 292 - runRegex terminates its worker when the signal aborts mid-run\n ---\n duration_ms: 25.51719\n type: 'test'\n ...\n1..292\n# tests 292\n# suites 0\n# pass 292\n# fail 0\n# cancelled 0\n# skipped 0\n# todo 0\n# duration_ms 139134.002911", "expected": "292 unit tests pass with no API key", "input": "npm test", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "README.md Development section", "verdict": "yes" } ], "status": "PASS" }, "deviations": [ "Ubuntu 24.04 ships no Node.js; installed Node 22 from NodeSource (curl|bash of setup_22.x) as a prerequisite.", "Ran the built bin 'node dist/index.js' from the pinned checkout instead of 'npx -y @jkudish/jev-mcp', because npx would fetch the published package rather than the pinned source.", "The first jev_noul call used a wrong argument name ('proposition'); corrected to 'propositions' from the README. Not a product failure.", "The first --http check killed my own shell via pkill -f; re-run with a PID file. Not a product failure.", "Claim 'Unit tests run without an API key' marked testable:false: the checker does not count a unit suite as a product operation. Unit tests were run as an example instead.", "No live API judgment was obtained: no provider key was available and none was sought.", "Checker replay of an http /health start check followed the announced /mcp URL and got HTTP 405 on GET; the probe now uses kind mcp_http list_tools against http://127.0.0.1:8080/mcp (real MCP tools/list) instead.", "A first --http instance was left running from an earlier attempt and held port 8080; it was stopped by PID before the final replay." ], "fault_owner": "prerequisite", "main_blocker": "No provider API key in the guest, so every judgment tool returns a missing-key error.", "operation": "Start the Jev MCP server (stdio and stateless HTTP) and call a judgment tool (jev_noul) to get a calibrated probability", "operation_scope": "Pinned source jkudish/jev-mcp@6faa539 (v0.14.5) built locally with Node 22.23.4. Server boots, lists all 12 tools, and the HTTP /health endpoint responds. Live judgments need a TypeSafe, OpenRouter, Cloudflare or AI gateway key, which is not available in this guest and was not sought. Unit tests (292) pass offline.", "outcome": "needs_credentials", "plain_failure": null, "steps": [ "git init; git remote add origin https://github.com/jkudish/jev-mcp; git fetch --depth 1 origin 6faa53961ade29edd0a9a2381ce74f84d64540b0; git reset --mixed FETCH_HEAD", "NodeSource setup_22.x + apt-get install nodejs (Node v22.23.4, npm 10.9.9)", "npm install (README Development: npm install)", "npm run build (README Development: npm run build)", "npm test (README Development: unit tests, no API key needed): 292 pass, 0 fail", "node dist/index.js (bin from package.json; README stdio server) - tools/list returns 12 tools", "node dist/index.js (jev_noul with propositions) - returns 'No TYPESAFE_API_KEY, OPENROUTER_API_KEY ... found. Set one' (isError)", "node dist/index.js --http - serves /health and /mcp on 127.0.0.1:8080; stopped afterwards" ], "stuck_step": "Calling a judgment tool (e.g. jev_noul) requires a provider API key (TYPESAFE_API_KEY, OPENROUTER_API_KEY, Cloudflare token, or AI_GATEWAY_API_KEY).", "suggested_fix": "Run the probe again with a TypeSafe (or OpenRouter) key set in the server environment (TYPESAFE_API_KEY=ts_...) to observe a live judgment result and replay the live claims." }