{ "run_id": "Coolhand-Labs--coolhand-python-gpt-6-luna-1793148429944344643", "date": "2026-10-28T00:48:26.567183+00:00", "repo": "Coolhand-Labs/coolhand-python", "kind": "repo", "source": "https://github.com/Coolhand-Labs/coolhand-python", "commit": "d0136ad4eab2d0577b60afd02f49d7b62e1851cf", "model": "gpt-6-luna", "model_version": "gpt-6-luna", "test_version": "readme-38e6b13f3b13804effb95797e3466b1a1334da7a232d897cbc8965867f802605", "pass": true, "outcome": "pass", "time_to_working_s": 55.82588213500276, "total_s": 76.62279939651489, "steps": [ "pwd && git status --short && sed -n '1,240p' README.md && python3 --version && id -u", "sed -n '1,260p' README.md; python3 --version; id -u; rg --files -g '!*lock*' | head -80", "sed -n '260,540p' README.md; ls -la; find . -maxdepth 2 -type f | head -100", "python3 -m pip install .", "python3 -m venv /tmp/coolhand-venv && /tmp/coolhand-venv/bin/pip install .", "sed -n '1,180p' examples/README.md; sed -n '1,180p' examples/basic_usage.py; sed -n '1,120p' docs/help-center.md", "python3 - <<'PY'\nimport urllib.request\nu='http://:9226/json/version'\ntry:\n r=urllib.request.urlopen(u,timeout=10); print(r.status); print(r.read().decode())\nexcept Exception as e: print(type(e).__name__,str(e))\nPY\n/tmp/coolhand-venv/bin/python examples/streaming_capture.py; /tmp/coolhand-venv/bin/python -c 'from coolhand import HelpCenterDocs; print(HelpCenterDocs().search_help_articles(query=\"match rate\", per_page=5))'", "cat > <<'EOF'\n{\"kind\":\"cli\",\"argv\":[\"/tmp/coolhand-venv/bin/python\",\"-c\",\"from coolhand import HelpCenterDocs; r=HelpCenterDocs().search_help_articles(query='match rate', per_page=5); print(r['articles'][0]['slug'])\"],\"cwd\":\"\",\"expected_text\":\"what-is-feedback-match-rate\"}\nEOF\ncat > <<'EOF'\n[{\"kind\":\"claim\",\"claim\":\"Help center search returns matching articles\",\"source\":\"`query` | `str` | Relevance search (Postgres full text, the same as the website's). Supports quoted phrases, `-exclusions` and `or`. Omit for a full listing. A blank string is treated as omitted\",\"input\":\"match rate\",\"expected\":\"what-is-feedback-match-rate\",\"probe_kind\":\"cli\",\"argv\":[\"/tmp/coolhand-venv/bin/python\",\"-c\",\"from coolhand import HelpCenterDocs; r=HelpCenterDocs().search_help_articles(query='match rate', per_page=5); print(r['articles'][0]['slug'])\"],\"expected_text\":\"what-is-feedback-match-rate\",\"cwd\":\"\"}]\nEOF\ncat > <<'EOF'\n[{\"kind\":\"example\",\"claim\":\"Streaming capture example\",\"source\":\"examples/README.md, Streaming capture check\",\"input\":\"python examples/streaming_capture.py\",\"expected\":\"Captured streamed response details and exits successfully\",\"argv\":[\"/tmp/coolhand-venv/bin/python\",\"examples/streaming_capture.py\"],\"cwd\":\"\",\"expected_returncode\":1,\"expected_text\":\"No module named 'httpx'\"}]\nEOF\ncat > <<'EOF'\n{\"detected\":false,\"reason\":\"Inspected README and examples; SDK performs interception during application API calls and documents no scheduler or recurring cycle.\" ,\"checks\":[]}\nEOF", "cat > <<'EOF'\n{\"outcome\":\"working\",\"fault_owner\":null,\"operation\":\"Search public Coolhand help articles\",\"operation_scope\":\"Python 3.12.3 virtual environment; installed local package and queried the documented anonymous HelpCenterDocs API\",\"steps\":[\"Created /tmp/coolhand-venv with Python 3.12.3\",\"Installed the workspace package with /tmp/coolhand-venv/bin/pip install .\",\"Called HelpCenterDocs().search_help_articles(query='match rate', per_page=5); returned what-is-feedback-match-rate with matching result data\"],\"deviations\":[\"System pip install . failed with externally-managed-environment; used a supported venv.\",\"Documented streaming_capture example attempted but could not run because httpx is absent from the package environment.\"],\"stuck_step\":null,\"main_blocker\":null,\"plain_failure\":null,\"suggested_fix\":null}\nEOF\n= python3 -I --validate --replay", "sed -n '1,180p' examples/auto_monitor_example.py; sed -n '1,120p' examples/streaming_capture.py", "cat > .coldpass/probe.json <<'EOF'\n{\"kind\":\"cli\",\"argv\":[\"/tmp/coolhand-venv/bin/python\",\"examples/auto_monitor_example.py\"],\"cwd\":\"\",\"env\":{\"COOLHAND_SILENT\":\"true\"},\"expected_text\":\"Auto-Initialization Example\"}\nEOF\ncat > .coldpass/claims.json <<'EOF'\n[{\"kind\":\"claim\",\"claim\":\"Import initializes monitoring automatically\",\"source\":\"import coolhand # Auto-initializes and starts monitoring\",\"input\":\"examples/auto_monitor_example.py\",\"expected\":\"Auto-Initialization Example\",\"probe_kind\":\"cli\",\"argv\":[\"/tmp/coolhand-venv/bin/python\",\"examples/auto_monitor_example.py\"],\"expected_text\":\"Auto-Initialization Example\",\"cwd\":\"\",\"env\":{\"COOLHAND_SILENT\":\"true\"}}]\nEOF\ncat > .coldpass/examples.json <<'EOF'\n[{\"kind\":\"example\",\"claim\":\"Automatic monitoring example\",\"source\":\"examples/README.md, examples table\",\"input\":\"examples/auto_monitor_example.py\",\"expected\":\"demonstrates zero-configuration monitoring\",\"argv\":[\"/tmp/coolhand-venv/bin/python\",\"examples/auto_monitor_example.py\"],\"cwd\":\"\",\"env\":{\"COOLHAND_SILENT\":\"true\"},\"expected_text\":\"Auto-Initialization Example\"}]\nEOF\n= python3 -I --validate --replay", "cat > .coldpass/report.json <<'EOF'\n{\"outcome\":\"working\",\"fault_owner\":null,\"operation\":\"Run documented automatic monitoring example\",\"operation_scope\":\"Python 3.12.3 virtual environment and locally installed package; simulated provider requests run and monitoring stats report two interactions\",\"steps\":[\"Created /tmp/coolhand-venv with Python 3.12.3\",\"Installed the workspace package with /tmp/coolhand-venv/bin/pip install .\",\"Ran /tmp/coolhand-venv/bin/python examples/auto_monitor_example.py\",\"Queried documented public help article search; returned what-is-feedback-match-rate\",\"Validated probe, README claim, example and cycle record with checker replay\"],\"deviations\":[\"System pip installation failed with externally-managed-environment; installed in a virtual environment.\",\"Initial streaming_capture example could not run because httpx was missing; used documented auto_monitor_example instead.\"],\"stuck_step\":null,\"main_blocker\":null,\"plain_failure\":null,\"suggested_fix\":null}\nEOF\n" ], "stuck_step": null, "error_excerpt": null, "what_it_did": "Run documented automatic monitoring example", "suggested_fix": null, "tokens_in": 245039, "tokens_out": 2160, "cost_usd": null, "usage_complete": true, "checks": [ { "check": "operation", "status": "PASS", "details": { "working": true, "argv": [ "/tmp/coolhand-venv/bin/python", "examples/auto_monitor_example.py" ], "kind": "cli", "returncode": 0, "stdout": "Setting up demonstration environment...\n\nCoolhand Python SDK - Auto-Monitor Example\n=======================================================\nAuto-Initialization Example\n==================================================\n\u2713 Coolhand was automatically initialized!\n Session ID: session_1793148500265\n Monitoring enabled: True\n Config source: Environment variables and auto-detection\n Has API key: False\n\n[excerpt truncated]", "stderr": "No matching field provided. Feedback may not be linked to an LLM request. Consider providing one of: llm_request_log_id, llm_provider_unique_id, original_output, or client_unique_id\nFailed to submit feedback: HTTP 401 - Unauthorized\nNo matching field provided. Feedback may not be linked to an LLM request. Consider providing one of: llm_request_log_id, llm_provider_unique_id, original_output, or client_unique_id\nFailed to submit feedback: HTTP 401 - Unauthorized\nINFO:coolhand:HTTP monitoring started\nINFO:coolhand:Coolhand initialized (session: session_1793148500495)\nINFO:coolhand:HTTP monitoring started\nINFO:coolhand:Coolhand initialized (session: session_1793148500499)\nINFO:coolhand:HTTP monitoring started\nINFO:coolhand:Coolhand initialized (session: session_1793148500503)\nINFO:coolhand:HTTP monitoring stopped\nINFO:coolhand:HTTP monitoring started\n[excerpt truncated]", "needs_credentials": false, "needs_browser": false, "scope": "product operation" } } ], "safety": { "version": "z2x-1", "rules_version": "z2x-1", "summary": "Review signals: 1 connection review, 8 known vulnerability. 8 static matches retained as context.", "expected": "Reviewed as expected for a test run: attempted writes that failed, so nothing was written.", "findings": [ { "kind": "connection-review", "count": 1, "note": "Repository processes attempted these non-loopback connections; proxies can hide the final hostname.", "location_count": 1, "path": "35.241.4.177:443", "process_count": 7 }, { "kind": "known-vulnerability", "count": 8, "note": "OSV reports a known vulnerability in this dependency version.", "location_count": 5 } ], "context": [ { "kind": "hardcoded-secret", "count": 8, "note": "Gitleaks found a possible hardcoded secret; value withheld.", "location_count": 4 } ], "limitations": [ "A short run cannot establish that code is safe.", "Connections show IP addresses; proxies can hide the final hostname.", "Tracing covers dispatched commands and their children, not the whole OS; guest root can interfere with tracing." ], "static": { "dependency_note": "OSV scanned supported lockfiles; unpinned dependencies and unsupported formats are outside this check." }, "evidence_url": "/safety/?run=Coolhand-Labs--coolhand-python-gpt-6-luna-1793148429944344643", "evidence_file": "/data/safety/Coolhand-Labs--coolhand-python-gpt-6-luna-1793148429944344643.json.gz" }, "agent_execution": "native_agent", "agent_runtime": "codex", "agent_runtime_version": "codex-cli 0.164.0", "tool_transport": "mcp-assigned-vm", "platform": "linux", "reasoning": "low", "service_tier": "standard", "resources": { "cpus": 2, "memory_mib": 4096, "disk_mib": 32768 }, "native_tool_calls": 12, "native_bundle": null, "architecture": null, "backend": "hyperv", "platform_details": { "backend": "hyperv", "os": "linux" }, "platform_conflicts": null, "sandbox_source": null, "source_pin": { "commit": "d0136ad4eab2d0577b60afd02f49d7b62e1851cf", "url": "https://github.com/Coolhand-Labs/coolhand-python" }, "purpose": "test_rerun", "profile": { "model": "gpt-6-luna", "reasoning": "low" }, "fault": { "owner": null, "status": "resolved", "reason": null }, "fix_id": null, "verifies_run_id": "Coolhand-Labs--coolhand-python-gpt-6-luna-1793140936358037958", "hosting": null, "operation_scope": "product operation", "tool_calls": 12, "guest_setup": null, "runtime": null, "runtime_version": null, "runtime_revision": null, "source_revision": null, "evidence_version": null, "failure_origin": null, "evidence_strength": "readme_checked", "works": "yes", "install_s": null, "first_use_s": null, "mcp": "none", "needs": [], "needs_which": null, "agent_interface": [ "cli" ], "license": "Apache-2.0", "repo_url": "https://github.com/Coolhand-Labs/coolhand-python", "cost_to_try": "free", "hardware": null, "what_failed": null, "local_or_hosted": "both", "cycle_checks": [ { "kind": "cycle", "claim": "Scheduled or looping main behavior", "source": "repository entry points", "input": "source inspection", "expected": "identify the product trigger", "verdict": "yes", "outcome": "Inspected README and examples; SDK performs interception during application API calls and documents no scheduler or recurring cycle.", "evidence": "Inspected README and examples; SDK performs interception during application API calls and documents no scheduler or recurring cycle.", "command": "" } ], "claims": [ { "kind": "claim", "claim": "Import initializes monitoring automatically", "source": "import coolhand # Auto-initializes and starts monitoring", "input": "examples/auto_monitor_example.py", "expected": "Auto-Initialization Example", "command": "/tmp/coolhand-venv/bin/python examples/auto_monitor_example.py", "verdict": "yes", "outcome": "Real input produced the expected result: Auto-Initialization Example", "evidence": "Auto-Initialization Example\n==================================================\n\u2713 Coolhand was automatically initialized!\n Session ID: session_1793148501131\n Monitoring enabled: True\n Config source: Environment variables and auto-detection\n Has API key: False\n\nZero-Configuration Usage\n------------------------------\nUsing Coolhand with zero configuration...\n\u2713 Submitted positive feedback\n\u2713 Submitted feedback with explanation\nMaking simulated OpenAI API request...\n\u2713 Request completed and automatically logged\nMaking simulated Anthropic API request...\n\u2713 Request completed and automatically logged", "returncode": 0, "source_checked": true, "source_file": "README.md" } ], "claims_note": null, "example_checks": [ { "kind": "example", "claim": "Automatic monitoring example", "source": "examples/README.md, examples table", "input": "examples/auto_monitor_example.py", "command": "/tmp/coolhand-venv/bin/python examples/auto_monitor_example.py", "evidence": "id, original_output, or client_unique_id\nWARNING:coolhand.feedback_service:Failed to submit feedback: HTTP 401 - Unauthorized\nINFO:coolhand.client:Shutting down Coolhand...\nINFO:coolhand.client:Coolhand shutdown complete\nINFO:coolhand.client:Shutting down Coolhand...\nINFO:coolhand.client:Coolhand shutdown complete\nINFO:coolhand.client:Shutting down Coolhand...\nINFO:coolhand.client:Coolhand shutdown complete\nINFO:coolhand.client:Shutting down Coolhand...\nINFO:coolhand.client:Coolhand shutdown complete\nINFO:coolhand.client:Shutting down Coolhand...\nINFO:coolhand.client:Coolhand shutdown complete", "deferred": false, "expected": "demonstrates zero-configuration monitoring", "returncode": 0, "verdict": "yes", "outcome": "command completed successfully" } ], "does_what_it_says": "yes", "maintenance": { "archived": false, "commits_90d": 96, "last_commit_at": "2026-10-27T19:34:06Z", "last_release_at": "2026-10-27T19:34:41Z", "median_reply_hours": null, "merged_prs_90d": 97, "note": "slow: last commit 0 days ago; 96 commits and 97 merged PRs in 90 days; 17 open issues, 2 open PRs; 0 of 19 recent threads got a maintainer reply; latest release 2026-10-27", "open_issues": 17, "open_prs": 2, "reply_share": 0.0, "reply_threads": 19, "verdict": "slow" }, "maintenance_verdict": "slow", "maintenance_note": "slow: last commit 0 days ago; 96 commits and 97 merged PRs in 90 days; 17 open issues, 2 open PRs; 0 of 19 recent threads got a maintainer reply; latest release 2026-10-27", "maintained": "active", "maintained_note": "last commit 0 days ago; 10 open issues, 3 answered or closed", "last_commit_at": "2026-10-27T19:36:39Z", "last_release_at": "2026-10-27T19:34:41Z", "open_issues": 10, "answered_issues": 3, "platforms_claimed": [ "linux" ], "topic": "agent-harness", "agent_interfaces": [ { "interface": "cli", "evidence": "independent checker", "observed": true, "operation_checked": true } ], "agent_interfaces_note": "README metadata is a discovery hint, not pinned documentation proof. Observed interfaces were exercised in this run. An empty list means unknown.", "mcp_context": null, "tested_at": "2026-10-28T00:48:26.567183+00:00", "tested_commit": "d0136ad4eab2d0577b60afd02f49d7b62e1851cf", "tested_with": { "agent": "codex", "agent_version": "codex-cli 0.164.0", "model": "gpt-6-luna", "model_route": "gpt-6-luna", "reasoning": "low", "os": "linux", "backend": "hyperv", "transport": "mcp-assigned-vm" }, "setup_identity": { "repo": "Coolhand-Labs/coolhand-python", "source": "https://github.com/Coolhand-Labs/coolhand-python", "commit": "d0136ad4eab2d0577b60afd02f49d7b62e1851cf", "source_revision": null, "os": "linux", "architecture": null, "backend": "hyperv", "model": "gpt-6-luna", "model_version": "gpt-6-luna", "agent_execution": "native_agent", "agent_runtime": "codex", "agent_runtime_version": "codex-cli 0.164.0", "tool_transport": "mcp-assigned-vm", "runtime": null, "runtime_version": null, "runtime_revision": null, "sandbox_source": null, "guest_setup": null, "source_pin": { "commit": "d0136ad4eab2d0577b60afd02f49d7b62e1851cf", "url": "https://github.com/Coolhand-Labs/coolhand-python" }, "platform_details": { "os": "linux", "backend": "hyperv" }, "platform_conflicts": null, "reasoning": "low", "service_tier": "standard", "resources": { "cpus": 2, "memory_mib": 4096, "disk_mib": 32768 }, "operation": "Run documented automatic monitoring example", "operation_scope": "product operation", "hosting": null, "kind": "repo", "test_version": "readme-38e6b13f3b13804effb95797e3466b1a1334da7a232d897cbc8965867f802605", "evidence_version": null }, "transcript_url": "/transcripts/Coolhand-Labs--coolhand-python-gpt-6-luna-1793148429944344643.txt", "report_url": "/reports/Coolhand-Labs--coolhand-python-gpt-6-luna-1793148429944344643.json" }