{ "run_id": "ashishsinha1602--langchain-schemagate-claude-haiku-5-5-1793152078034794133", "date": "2026-10-28T01:52:20.397382+00:00", "repo": "ashishsinha1602/langchain-schemagate", "kind": "repo", "source": "https://github.com/ashishsinha1602/langchain-schemagate", "commit": "44330041b91fb9e3825040a3763a857e9bfe169b", "model": "claude-haiku-5-5", "model_version": "claude-haiku-5-5", "test_version": "readme-c03644542c69939a9374958136a8b3431f7456269d3fb4b3c433785f59cfd637", "pass": true, "outcome": "pass", "time_to_working_s": 186.0543543120002, "total_s": 262.3624472618103, "steps": [ "cd && pwd && ls -la && id -u && python3 --version && cat README.md | head -c 6000", "cd && cat pyproject.toml && ls -R langchain_schemagate tests .github && cat .github/workflows/* 2>/dev/null | head -c 3000 && ls -la .coldpass && echo $CDP_URL", "cd && python3 -m venv && install -q --upgrade pip 2>&1 | tail -3; cd && install -e \".[test]\" 2>&1 | tail -5", "cd && cat langchain_schemagate/__init__.py | head -c 4000; echo ----; cat tests/unit_tests/test_standard.py | head -c 1500; echo ----; -q 2>&1 | tail -15", "cd && ls | grep -i -E \"schema|gate\" ; cat 2>/dev/null; cat 2>/dev/null; cd /tmp && -I -c \"import schemagate.demo_schema as d; print(d.__file__)\"; ls ", "cd /tmp && S=; $S --help 2>&1 | head -60; echo ----; grep -n -i \"demo\\|langchain\" | head -30", "cd /tmp && S=; $S demo \"salary by employee\" 2>&1 | head -40; echo \"EXIT=$?\"; echo ----; $S demo \"salary by employee\" --principal okta:hr --role payroll 2>&1 | head -40", "cd && cat > probe.json <<'EOF'\n{\"kind\":\"cli\",\"argv\":[\"\",\"demo\",\"salary by employee\",\"--principal\",\"okta:hr\",\"--role\",\"payroll\"],\"cwd\":\"\",\"expected_text\":\"main.hr_compensation\"}\nEOF\ncat > claims.json <<'EOF'\n[{\"kind\":\"claim\",\"claim\":\"A table the caller may not read is never selected as a candidate, so it cannot reach the prompt\",\"source\":\"README.md\",\"quote\":\"A table the caller can't read is never a candidate, so it can't\",\"input\":\"salary by employee\",\"expected\":\"10 of 42 objects selected for a default caller without the payroll role\",\"probe_kind\":\"cli\",\"argv\":[\"\",\"demo\",\"salary by employee\"],\"expected_text\":\"10 of 42 objects selected\",\"cwd\":\"\"}]\nEOF\ncat > examples.json <<'EOF'\n[{\"kind\":\"example\",\"claim\":\"repo test suite (CI command pytest -q) runs LangChain standard retriever tests and the access-control test\",\"source\":\".github/workflows/ci.yml\",\"input\":\"pytest -q\",\"expected\":\"all tests pass\",\"argv\":[\"\",\"-q\"],\"cwd\":\"\",\"expected_text\":\"7 passed\"}]\nEOF\ncat > cycles.json <<'EOF'\n{\"detected\":false,\"reason\":\"Entry points inspected: langchain_schemagate/__init__.py exports only SchemagateRetriever and prompt_fragment (a BaseRetriever library, no CLI, server, cron, scheduler, loop, webhook or --once/--now option). The schemagate dependency's CLI has no scheduler either. Nothing in this repo runs on a schedule, so no product cycle exists.\",\"checks\":[]}\nEOF\nfor f in probe claims examples cycles; do -I -c \"import json,sys;json.load(open('$f.json'));print('$f ok')\"; done", "cd && cat > claims.json <<'EOF'\n[{\"kind\":\"claim\",\"claim\":\"A table the caller may not read is never selected as a candidate, so it cannot reach the prompt\",\"source\":\"A table the caller can't read is never a candidate, so it can't\",\"input\":\"salary by employee\",\"expected\":\"10 of 42 objects selected for a default caller without the payroll role\",\"probe_kind\":\"cli\",\"argv\":[\"\",\"demo\",\"salary by employee\"],\"expected_text\":\"10 of 42 objects selected\",\"cwd\":\"\"}]\nEOF\ncd && = python3 -I --validate --replay 2>&1 | tail -60", "cd && cat > report.json <<'EOF'\n{\"outcome\":\"working\",\"fault_owner\":null,\"operation\":\"Select the tables a question needs, filtered by the caller's roles, using the schemagate engine that langchain-schemagate wraps\",\"operation_scope\":\"Offline selection against schemagate's bundled 42-object demo schema, run via the documented schemagate demo CLI. The README's Postgres bootstrap (Catalog().bootstrap('postgresql://localhost/app')) and the LangChain retriever Python API were not run directly: no database is available in the guest, and the README gives Python code, not a CLI.\",\"steps\":[\"pip install -e \\\".[test]\\\" (in a Python 3.12 venv at ; CI command from .github/workflows/ci.yml)\",\"pytest -q (7 passed)\",\"schemagate demo \\\"salary by employee\\\" (analyst caller: 10 of 42 objects selected, hr_compensation absent)\",\"schemagate demo \\\"salary by employee\\\" --principal okta:hr --role payroll (9 of 42 selected, main.hr_compensation present)\"],\"deviations\":[\"The README's Python quickstart (Catalog().bootstrap on localhost Postgres) was not run: no Postgres database in the guest.\",\"The README has no CLI; the probe uses the schemagate CLI (documented in the schemagate dependency's packaged README, which the langchain README links to) as the real product entry point.\",\"Python 3.12.3 was used (requires-python >=3.10; CI matrix 3.10 and 3.13 not tested).\"],\"stuck_step\":null,\"main_blocker\":null,\"plain_failure\":null,\"suggested_fix\":\"None for the tested path. To cover the README quickstart, run it against a Postgres instance and check that the retriever returns only tables the caller may read.\"}\nEOF\n -I -c \"import json;json.load(open('report.json'));print('report ok')\"; cd && = python3 -I --validate --replay 2>&1 | head -c 300" ], "stuck_step": null, "error_excerpt": null, "what_it_did": "Select the tables a question needs, filtered by the caller's roles, using the schemagate engine that langchain-schemagate wraps", "suggested_fix": null, "tokens_in": 168922, "tokens_out": 7514, "cost_usd": 0.00968083, "usage_complete": true, "checks": [ { "check": "operation", "status": "PASS", "details": { "working": true, "argv": [ "", "demo", "salary by employee", "--principal", "okta:hr", "--role", "payroll" ], "kind": "cli", "returncode": 0, "stdout": "schemagate 1.14.0 -- demo schema, 42 objects, caller okta:hr roles=['payroll']\nhr_compensation is restricted to the 'payroll' role.\n\n> salary by employee\n9 of 42 objects selected\n\n main.hr_compensation\n main.hr_employee\n main.v_employee_headcount\n main.sales_order_status\n main.sales_invoice_draft\n main.core_country\n[excerpt truncated]", "stderr": "", "needs_credentials": false, "needs_browser": false, "scope": "product operation" } } ], "safety": { "version": "z2x-1", "rules_version": "z2x-1", "summary": "No problems traced to this repository code in these checks.", "expected": "Reviewed as expected for a test run: attempted writes that failed, so nothing was written.", "findings": [], "context": [], "limitations": [ "A short run cannot establish that code is safe.", "Connections show IP addresses; proxies can hide the final hostname.", "Tracing covers dispatched commands and their children, not the whole OS; guest root can interfere with tracing." ], "static": { "dependency_note": "OSV scanned supported lockfiles; unpinned dependencies and unsupported formats are outside this check." }, "evidence_url": "/safety/?run=ashishsinha1602--langchain-schemagate-claude-haiku-5-5-1793152078034794133", "evidence_file": "/data/safety/ashishsinha1602--langchain-schemagate-claude-haiku-5-5-1793152078034794133.json.gz" }, "agent_execution": "native_agent", "agent_runtime": "claude-code", "agent_runtime_version": "2.1.310 (Claude Code)", "tool_transport": "mcp-assigned-vm", "platform": "linux", "reasoning": "low", "service_tier": null, "resources": { "cpus": 2, "memory_mib": 4096, "disk_mib": 32768 }, "native_tool_calls": 10, "native_bundle": null, "architecture": null, "backend": "hyperv", "platform_details": { "backend": "hyperv", "os": "linux" }, "platform_conflicts": null, "sandbox_source": null, "source_pin": { "commit": "44330041b91fb9e3825040a3763a857e9bfe169b", "url": "https://github.com/ashishsinha1602/langchain-schemagate" }, "purpose": "initial", "profile": { "model": "claude-haiku-5-5", "reasoning": "low" }, "fault": { "owner": null, "status": "resolved", "reason": null }, "fix_id": null, "verifies_run_id": null, "hosting": null, "operation_scope": "product operation", "tool_calls": 10, "guest_setup": null, "runtime": null, "runtime_version": null, "runtime_revision": null, "source_revision": null, "evidence_version": null, "failure_origin": null, "evidence_strength": "readme_checked", "works": "yes", "install_s": null, "first_use_s": null, "mcp": "none", "needs": [], "needs_which": null, "agent_interface": [ "cli" ], "license": "Apache-2.0", "repo_url": "https://github.com/ashishsinha1602/langchain-schemagate", "cost_to_try": "free", "hardware": null, "what_failed": null, "local_or_hosted": "local", "cycle_checks": [ { "kind": "cycle", "claim": "Scheduled or looping main behavior", "source": "repository entry points", "input": "source inspection", "expected": "identify the product trigger", "verdict": "yes", "outcome": "Entry points inspected: langchain_schemagate/__init__.py exports only SchemagateRetriever and prompt_fragment (a BaseRetriever library, no CLI, server, cron, scheduler, loop, webhook or --once/--now option). The schemagate dependency's CLI has no scheduler either. Nothing in this repo runs on a schedule, so no product cycle exists.", "evidence": "Entry points inspected: langchain_schemagate/__init__.py exports only SchemagateRetriever and prompt_fragment (a BaseRetriever library, no CLI, server, cron, scheduler, loop, webhook or --once/--now option). The schemagate dependency's CLI has no scheduler either. Nothing in this repo runs on a schedule, so no product cycle exists.", "command": "" } ], "claims": [ { "kind": "claim", "claim": "A table the caller may not read is never selected as a candidate, so it cannot reach the prompt", "source": "A table the caller can't read is never a candidate, so it can't", "input": "salary by employee", "expected": "10 of 42 objects selected for a default caller without the payroll role", "command": " demo 'salary by employee'", "verdict": "yes", "outcome": "Real input produced the expected result: 10 of 42 objects selected", "evidence": "10 of 42 objects selected\n\n main.hr_employee\n main.v_employee_headcount\n main.sales_order_status\n main.sales_invoice_draft\n main.core_country\n main.billing_credit_note\n main.core_party\n main.hr_department\n main.sales_order\n main.billing_invoice\n\n", "returncode": 0, "source_checked": true, "source_file": "README.md" } ], "claims_note": null, "example_checks": [ { "kind": "example", "claim": "repo test suite (CI command pytest -q) runs LangChain standard retriever tests and the access-control test", "source": ".github/workflows/ci.yml", "input": "pytest -q", "command": " -q", "evidence": "te-packages/pytest_asyncio/plugin.py:300: PytestDeprecationWarning: The configuration option \"asyncio_default_fixture_loop_scope\" is unset.\nThe event loop scope for asynchronous fixtures will default to the \"fixture\" caching scope. Future versions of pytest-asyncio will default the loop scope for asynchronous fixtures to \"function\" scope. Set the default fixture loop scope explicitly in order to avoid unexpected behavior in the future. Valid fixture loop scopes are: \"function\", \"class\", \"module\", \"package\", \"session\"\n\n warnings.warn(PytestDeprecationWarning(_DEFAULT_FIXTURE_LOOP_SCOPE_UNSET))", "deferred": false, "expected": "all tests pass", "returncode": 0, "verdict": "yes", "outcome": "command completed successfully" } ], "does_what_it_says": "yes", "maintenance": { "last_commit_at": "2026-10-27T23:01:36Z", "commits_90d": 1, "merged_prs_90d": 0, "open_issues": 0, "open_prs": 0, "reply_share": null, "reply_threads": 0, "median_reply_hours": null, "last_release_at": null, "archived": false, "verdict": "maintained", "note": "actively maintained: last commit 0 days ago; 1 commits and 0 merged PRs in 90 days; 0 open issues, 0 open PRs; no releases" }, "maintenance_verdict": "maintained", "maintenance_note": "actively maintained: last commit 0 days ago; 1 commits and 0 merged PRs in 90 days; 0 open issues, 0 open PRs; no releases", "maintained": "active", "maintained_note": "last commit 0 days ago", "last_commit_at": "2026-10-27T23:37:33Z", "last_release_at": null, "open_issues": null, "answered_issues": null, "platforms_claimed": [ "linux" ], "topic": "agent-harness", "agent_interfaces": [ { "interface": "cli", "evidence": "independent checker", "observed": true, "operation_checked": true } ], "agent_interfaces_note": "README metadata is a discovery hint, not pinned documentation proof. Observed interfaces were exercised in this run. An empty list means unknown.", "mcp_context": null, "tested_at": "2026-10-28T01:52:20.397382+00:00", "tested_commit": "44330041b91fb9e3825040a3763a857e9bfe169b", "tested_with": { "agent": "claude-code", "agent_version": "2.1.310 (Claude Code)", "model": "claude-haiku-5-5", "model_route": "claude-haiku-5-5", "reasoning": "low", "os": "linux", "backend": "hyperv", "transport": "mcp-assigned-vm" }, "setup_identity": { "repo": "ashishsinha1602/langchain-schemagate", "source": "https://github.com/ashishsinha1602/langchain-schemagate", "commit": "44330041b91fb9e3825040a3763a857e9bfe169b", "source_revision": null, "os": "linux", "architecture": null, "backend": "hyperv", "model": "claude-haiku-5-5", "model_version": "claude-haiku-5-5", "agent_execution": "native_agent", "agent_runtime": "claude-code", "agent_runtime_version": "2.1.310 (Claude Code)", "tool_transport": "mcp-assigned-vm", "runtime": null, "runtime_version": null, "runtime_revision": null, "sandbox_source": null, "guest_setup": null, "source_pin": { "commit": "44330041b91fb9e3825040a3763a857e9bfe169b", "url": "https://github.com/ashishsinha1602/langchain-schemagate" }, "platform_details": { "os": "linux", "backend": "hyperv" }, "platform_conflicts": null, "reasoning": "low", "service_tier": null, "resources": { "cpus": 2, "memory_mib": 4096, "disk_mib": 32768 }, "operation": "Select the tables a question needs, filtered by the caller's roles, using the schemagate engine that langchain-schemagate wraps", "operation_scope": "product operation", "hosting": null, "kind": "repo", "test_version": "readme-c03644542c69939a9374958136a8b3431f7456269d3fb4b3c433785f59cfd637", "evidence_version": null }, "other_model": null, "other_run_id": null, "models_agree": null, "furthest_model": null, "transcript_url": "/transcripts/ashishsinha1602--langchain-schemagate-claude-haiku-5-5-1793152078034794133.txt", "report_url": "/reports/ashishsinha1602--langchain-schemagate-claude-haiku-5-5-1793152078034794133.json" }