{ "run_id": "spcaeo--toadstone-claude-haiku-5-5-1793020353883642353", "checker": { "claims": [ { "claim": "Crawl with --fail-on exits non-zero when open issues at that severity exist", "command": " main.py --crawl https://example.com/ --max-urls 5 --delay 0.5 --fail-on error --report ", "evidence": "FAILED: 4 issue(s) at severity 'error' or worse\n [error] Missing Meta Description - https://example.com/\n [error] Missing H1 Tag - https://example.com/\n [error] No Structured Data - https://example.com/\n [error] Sitemap File Error - https://example.com/sitemap.xml\n", "expected": "exit code 1 and FAILED summary", "input": "crawl https://example.com/ with --fail-on error", "kind": "claim", "outcome": "Real input produced the expected result: FAILED: 4 issue(s) at severity 'error' or worse", "returncode": 1, "source": "Exits non-zero when the crawl finds an open issue at that severity or worse, so", "source_checked": true, "source_file": "README.md", "verdict": "yes" }, { "claim": "Ten tools that read a crawl, read-only by default", "command": " -m src.mcp tools/call {\"tool\": \"list_issues\", \"arguments\": {\"crawl_id\": 1, \"severity\": \"error\"}}", "evidence": "{\"content\": [{\"type\": \"text\", \"text\": \"{\\n \\\"error\\\": \\\"Could not reach Toadstone at http://localhost:5057: ConnectionError: HTTPConnectionPool(host='localhost', port=5057): Max retries exceeded with url: /api/crawls/1/issues?severity=error&limit=50&offset=0 (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\\\",\\n \\\"status\\\": null,\\n \\\"path\\\": \\\"/api/crawls/1/issues\\\",\\n \\\"hint\\\": \\\"Toadstone is not reachable; check that it is running.\\\"\\n}\"}], \"isError\": true}", "expected": "the missing H1 finding on example.com", "input": "{\"tool\": \"list_issues\", \"arguments\": {\"crawl_id\": 1, \"severity\": \"error\"}}", "kind": "claim", "outcome": "Real input did not produce the expected result: Page has no H1 heading", "source": "Ten tools, read-only by default,", "source_checked": true, "source_file": "README.md", "verdict": "no" } ], "cycle_checks": [ { "claim": "Scheduled standing crawl runs one cycle when triggered; a run should start a crawl and return its id", "command": "curl -sS -m 20 -X POST http://localhost:5057/api/schedules/1/run", "evidence": "curl: (7) Failed to connect to localhost port 5057 after 6 ms: Couldn't connect to server; product invocation failed: {\"argv\": [\"curl\", \"-sS\", \"-m\", \"20\", \"-X\", \"POST\", \"http://localhost:5057/api/schedules/1/run\"], \"returncode\": 7, \"stdout\": \"\", \"stderr\": \"curl: (7) Failed to connect to localhost port 5057 after 6 ms: Couldn't connect to server\\n\", \"needs_credentials\": false, \"needs_browser\": false}", "expected": "success true with a crawl id (observed: success true, crawl_id null, no crawl row created; defect in src/core/crawl/lifecycle.py start_crawl which only creates a crawl record when session_id is passed)", "input": "schedule id 1, url https://example.com/", "kind": "cycle", "method": "run-once: POST /api/schedules/1/run on the local server (schedule 1 created with POST /api/schedules), instead of waiting for the 60-minute interval", "outcome": "Triggered one cycle by hand (run-once: POST /api/schedules/1/run on the local server (schedule 1 created with POST /api/schedules), instead of waiting for the 60-minute interval; normal trigger: every_minutes interval via Scheduler.tick every 30s (TICK_SECONDS); manual trigger is the run-now route): curl: (7) Failed to connect to localhost port 5057 after 6 ms: Couldn't connect to server; product invocation failed: {\"argv\": [\"curl\", \"-sS\", \"-m\", \"20\", \"-X\", \"POST\", \"http://localhost:5057/api/schedules/1/run\"], \"returncode\": 7, \"stdout\": \"\", \"stderr\": \"curl: (7) Failed to connect to localhost port 5057 after 6 ms: Couldn't connect to server\\n\", \"needs_credentials\": false, \"needs_browser\": false}", "returncode": 7, "schedule": "every_minutes interval via Scheduler.tick every 30s (TICK_SECONDS); manual trigger is the run-now route", "source": "src/api/schedules.py", "verdict": "no" } ], "details": { "argv": [ "", "-m", "src.mcp" ], "blocker": "MCP tool result did not confirm the expected product output", "kind": "mcp", "serverInfo": { "name": "toadstone", "version": "1.0.0" }, "tool_result": { "content": [ { "text": "{\n \"error\": \"Could not reach Toadstone at http://localhost:5057: ConnectionError: HTTPConnectionPool(host='localhost', port=5057): Max retries exceeded with url: /api/crawls/1/issues?severity=error&limit=50&offset=0 (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\",\n \"status\": null,\n \"path\": \"/api/crawls/1/issues\",\n \"hint\": \"Toadstone is not reachable; check that it is running.\"\n}", "type": "text" } ], "isError": true }, "tool_schema_bytes": 6572, "tools": [ "list_crawls", "crawl_status", "start_crawl", "list_issues", "issue_detail", "page_detail", "regression_diff", "export_dataset", "check_reference", "accept_issue" ], "working": false }, "evidence_strength": "partly_checked", "example_checks": [ { "claim": "Offline scheduler test suite (documented tests/*_tests.py, run directly)", "command": " tests/scheduler_tests.py", "deferred": false, "evidence": "ot_catch_up:\n PASS catch-up: a day of missed runs fires once, not twenty-four times - 1\n\ntest_the_thread_starts_and_stops:\n PASS thread: it is not running before start - ok\n PASS thread: it runs after start - ok\n PASS thread: starting twice is a no-op - ok\n PASS thread: it stops - ok\n\ntest_a_broken_tick_does_not_end_the_scheduler:\nScheduler tick failed: RuntimeError: boom\nScheduler tick failed: RuntimeError: boom\n PASS resilience: a tick that raises does not kill the loop - 2 ticks, running=True\n\n================================================================\n 34 passed, 0 failed", "expected": "all scheduler checks pass", "input": "no arguments", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "tests/scheduler_tests.py", "verdict": "yes" }, { "claim": "MCP test suite (CI runs tests/*_tests.py against a local server on port 5057)", "command": " tests/mcp_tests.py", "deferred": false, "evidence": "sts already crawled in this install', 'max_urls_per_crawl': 2000, 'mode': 'read-only', 'toadstone_url': 'http://localho\n PASS http: POST answers a JSON-RPC request - status=200\n PASS http: a tool call runs as the signed-in user - {'total': 2, 'count': 2, 'crawls': [{'id': 1, 'base_url': 'https://site.test', 'base_domain': 'site.test', 'status': 'completed', 'started_a\n PASS http: a notification gets 202 and no body - 202\n PASS http: unparseable JSON is a JSON-RPC parse error, not a 500 - status=200\n\n================================================================\n 128 passed, 0 failed", "expected": "all MCP checks pass", "input": "no arguments; server on http://localhost:5057", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "tests/mcp_tests.py", "verdict": "yes" }, { "claim": "Palette UI test (CI runs python tests/palette_tests.py)", "command": "", "deferred": false, "evidence": "needs a local Chromium: the test calls p.chromium.launch() with no CDP option, and the browser download step is not permitted in this guest. Observed error: Executable doesn't exist at ./ms-playwright/chromium_headless_shell. Ran instead: the CI crawl and MCP probes.", "expected": "documented command completes successfully", "input": "", "kind": "example", "outcome": "needs a local Chromium: the test calls p.chromium.launch() with no CDP option, and the browser download step is not permitted in this guest. Observed error: Executable doesn't exist at ./ms-playwright/chromium_headless_shell. Ran instead: the CI crawl and MCP probes.", "source": "tests/palette_tests.py", "verdict": "blocked" }, { "claim": "Router UI test (CI runs python tests/router_tests.py)", "command": "", "deferred": false, "evidence": "needs a local Chromium: the test calls p.chromium.launch() with no CDP option, and the browser download step is not permitted in this guest. Observed error: Executable doesn't exist at ./ms-playwright/chromium_headless_shell. Ran instead: the CI crawl and MCP probes.", "expected": "documented command completes successfully", "input": "", "kind": "example", "outcome": "needs a local Chromium: the test calls p.chromium.launch() with no CDP option, and the browser download step is not permitted in this guest. Observed error: Executable doesn't exist at ./ms-playwright/chromium_headless_shell. Ran instead: the CI crawl and MCP probes.", "source": "tests/router_tests.py", "verdict": "blocked" } ], "status": "FAIL" }, "diagnosis": null }