{ "checker": { "claims": [ { "claim": "Restricted tables are scoped to the caller's roles: with the payroll role, hr_compensation is selected", "command": " --url demo tools/call {\"tool\": \"select_schema\", \"arguments\": {\"question\": \"salary by employee\", \"principal\": \"okta:hr\", \"roles\": [\"payroll\"]}}", "evidence": "\u2026hr_compensation\\\",\\n \\\"main.hr_employee\\\",\\n \\\"main.v_employee_headcount\\\",\\n \\\"main.sales_order_status\\\",\\n \\\"main.sales_invoice_draft\\\",\\n \\\"main.core_country\\\",\\n \\\"main.core_party\\\",\\n \\\"main.hr_department\\\",\\n \\\"main.sales_order\\\"\\n ],\\n \\\"ddl\\\": \\\"-- salary history; restricted\\\\nTABLE main.hr_compensation (\\\\n id INTEGER PK,\\\\n id_employee INTEGER,\\\\n effective_from TEXT,\\\\n annual_amount REAL,\\\\n currency TEXT,\\\\n pay_grade TEXT\\\\n)\\\\n-- FK hr_compensation(id_employee) -> hr_employee\\\\n\\\\nTABLE main.hr_employee (\\\\n id INTEGER PK,\\\\n id_party INTEGER,\\\\", "expected": "hr_compensation is in the selection", "input": "{\"tool\": \"select_schema\", \"arguments\": {\"question\": \"salary by employee\", \"principal\": \"okta:hr\", \"roles\": [\"payroll\"]}}", "kind": "claim", "outcome": "Real input produced the expected result: hr_compensation", "source": "Absent, not ranked low. A table the caller may not read never enters the", "source_checked": true, "source_file": "README.md", "verdict": "yes" }, { "claim": "The demo CLI with --principal okta:hr --role payroll ranks hr_compensation as selected", "command": " demo 'salary by employee' --principal okta:hr --role payroll", "evidence": "\u2026main.hr_compensation\n main.hr_employee\n main.v_employee_headcount\n main.sales_order_status\n main.sales_invoice_draft\n main.core_country\n main.core_party\n main.hr_department\n main.sales_order\n\n", "expected": "main.hr_compensation listed", "input": "salary by employee, principal okta:hr, role payroll", "kind": "claim", "outcome": "Real input produced the expected result: main.hr_compensation", "returncode": 0, "source": "schemagate demo \"salary by employee\" --principal okta:hr --role payroll # now it is first", "source_checked": true, "source_file": "README.md", "verdict": "yes" } ], "cycle_checks": [ { "claim": "Scheduled or looping main behavior", "command": "", "evidence": "Entry points inspected: schemagate CLI (demo, select, studio, describe, certify) and schemagate-mcp are one-shot commands or servers that answer requests; grep of src/schemagate for cron, schedule, --once, webhook and while-True loops found no scheduler or product cycle. Each command runs once per invocation.", "expected": "identify the product trigger", "input": "source inspection", "kind": "cycle", "outcome": "Entry points inspected: schemagate CLI (demo, select, studio, describe, certify) and schemagate-mcp are one-shot commands or servers that answer requests; grep of src/schemagate for cron, schedule, --once, webhook and while-True loops found no scheduler or product cycle. Each command runs once per invocation.", "source": "repository entry points", "verdict": "yes" } ], "details": { "argv": [ "", "--url", "demo" ], "kind": "mcp", "scope": "product operation", "serverInfo": { "name": "schemagate", "version": "1.14.0" }, "tool_result": { "content": [ { "text": "{\n \"question\": \"salary by employee\",\n \"selected\": 9,\n \"total_objects\": 42,\n \"objects\": [\n \"main.hr_compensation\",\n \"main.hr_employee\",\n \"main.v_employee_headcount\",\n \"main.sales_order_status\",\n \"main.sales_invoice_draft\",\n \"main.core_country\",\n \"main.core_party\",\n \"main.hr_department\",\n \"main.sales_order\"\n ],\n \"ddl\": \"-- salary history; restricted\\nTABLE main.hr_compensation (\\n id INTEGER PK,\\n id_employee INTEGER,\\n effective_from TEXT,\\n annual_amount REAL,\\n currency TEXT,\\n pay_grade TEXT\\n)\\n-- FK hr_compensation(id_employee) -> hr_employee\\n\\nTABLE main.hr_employee (\\n id INTEGER PK,\\n id_party INTEGER,\\n employee_number TEXT,\\n hired_on TEXT,\\n id_manager INTEGER,\\n id_department INTEGER,\\n status TEXT\\n)\\n-- FK hr_employee(id_manager) -> hr_employee\\n-- FK hr_employee(id_party) -> core_party\\n-- FK hr_employee(id_department) -> hr_department (inferred from the column name, not declared)\\n\\nVIEW main.v_employee_headcount (\\n department TEXT,\\n headcount NULL\\n)\\n\\nTABLE main.sales_order_status (\\n id INTEGER PK,\\n code TEXT,\\n label TEXT,\\n is_terminal INTEGER\\n)\\n\\n-- pre-issue drafts only; NOT real revenue, exclude from financials\\nTABLE main.sales_invoice_draft (\\n id INTEGER PK,\\n id_order INTEGER,\\n drafted_on TEXT,\\n state TEXT,\\n total_gross REAL\\n)\\n-- FK sales_invoice_draft(id_order) -> sales_order\\n\\nTABLE main.core_country (\\n id INTEGER PK,\\n iso2 TEXT,\\n name TEXT,\\n region TEXT\\n)\\n\\nTABLE main.core_party (\\n id INTEGER PK,\\n display_name TEXT,\\n party_type TEXT,\\n id_primary_address INTEGER,\\n status TEXT,\\n created_at TEXT\\n)\\n-- FK core_party(id_primary_address) -> core_address\\n\\nTABLE main.hr_department (\\n id INTEGER PK,\\n code TEXT,\\n name TEXT,\\n cost_centre TEXT\\n)\\n\\nTABLE main.sales_order (\\n id INTEGER PK,\\n order_number TEXT,\\n id_customer INTEGER,\\n id_ship_address INTEGER,\\n id_quote INTEGER,\\n ordered_at TEXT,\\n id_status INTEGER,\\n currency TEXT,\\n total_net REAL\\n)\\n-- FK sales_order(id_quote) -> sales_quote\\n-- FK sales_order(id_ship_address) -> core_address\\n-- FK sales_order(id_customer) -> crm_customer\\n-- FK sales_order(id_status) -> sales_order_status (inferred from the column name, not declared)\",\n \"explain\": [\n {\n \"object\": \"main.hr_compensation\",\n \"score\": 0.0489,\n \"reason\": \"hybrid\"\n },\n {\n \"object\": \"main.hr_employee\",\n \"score\": 0.0487,\n \"reason\": \"hybrid\"\n },\n {\n \"object\": \"main.v_employee_headcount\",\n \"score\": 0.0481,\n \"reason\": \"hybrid\"\n },\n {\n \"object\": \"main.sales_order_status\",\n \"score\": 0.0156,\n \"reason\": \"vector\"\n },\n {\n \"object\": \"main.sales_invoice_draft\",\n \"score\": 0.0154,\n \"reason\": \"vector\"\n },\n {\n \"object\": \"main.core_country\",\n \"score\": 0.0152,\n \"reason\": \"vector\"\n },\n {\n \"object\": \"main.core_party\",\n \"score\": 0.0,\n \"reason\": \"fk\"\n },\n {\n \"object\": \"main.hr_department\",\n \"score\": 0.0,\n \"reason\": \"fk\"\n },\n {\n \"object\": \"main.sales_order\",\n \"score\": 0.0,\n \"reason\": \"fk\"\n }\n ]\n}", "type": "text" } ], "isError": false, "structuredContent": { "result": { "ddl": "-- salary history; restricted\nTABLE main.hr_compensation (\n id INTEGER PK,\n id_employee INTEGER,\n effective_from TEXT,\n annual_amount REAL,\n currency TEXT,\n pay_grade TEXT\n)\n-- FK hr_compensation(id_employee) -> hr_employee\n\nTABLE main.hr_employee (\n id INTEGER PK,\n id_party INTEGER,\n employee_number TEXT,\n hired_on TEXT,\n id_manager INTEGER,\n id_department INTEGER,\n status TEXT\n)\n-- FK hr_employee(id_manager) -> hr_employee\n-- FK hr_employee(id_party) -> core_party\n-- FK hr_employee(id_department) -> hr_department (inferred from the column name, not declared)\n\nVIEW main.v_employee_headcount (\n department TEXT,\n headcount NULL\n)\n\nTABLE main.sales_order_status (\n id INTEGER PK,\n code TEXT,\n label TEXT,\n is_terminal INTEGER\n)\n\n-- pre-issue drafts only; NOT real revenue, exclude from financials\nTABLE main.sales_invoice_draft (\n id INTEGER PK,\n id_order INTEGER,\n drafted_on TEXT,\n state TEXT,\n total_gross REAL\n)\n-- FK sales_invoice_draft(id_order) -> sales_order\n\nTABLE main.core_country (\n id INTEGER PK,\n iso2 TEXT,\n name TEXT,\n region TEXT\n)\n\nTABLE main.core_party (\n id INTEGER PK,\n display_name TEXT,\n party_type TEXT,\n id_primary_address INTEGER,\n status TEXT,\n created_at TEXT\n)\n-- FK core_party(id_primary_address) -> core_address\n\nTABLE main.hr_department (\n id INTEGER PK,\n code TEXT,\n name TEXT,\n cost_centre TEXT\n)\n\nTABLE main.sales_order (\n id INTEGER PK,\n order_number TEXT,\n id_customer INTEGER,\n id_ship_address INTEGER,\n id_quote INTEGER,\n ordered_at TEXT,\n id_status INTEGER,\n currency TEXT,\n total_net REAL\n)\n-- FK sales_order(id_quote) -> sales_quote\n-- FK sales_order(id_ship_address) -> core_address\n-- FK sales_order(id_customer) -> crm_customer\n-- FK sales_order(id_status) -> sales_order_status (inferred from the column name, not declared)", "explain": [ { "object": "main.hr_compensation", "reason": "hybrid", "score": 0.0489 }, { "object": "main.hr_employee", "reason": "hybrid", "score": 0.0487 }, { "object": "main.v_employee_headcount", "reason": "hybrid", "score": 0.0481 }, { "object": "main.sales_order_status", "reason": "vector", "score": 0.0156 }, { "object": "main.sales_invoice_draft", "reason": "vector", "score": 0.0154 }, { "object": "main.core_country", "reason": "vector", "score": 0.0152 }, { "object": "main.core_party", "reason": "fk", "score": 0.0 }, { "object": "main.hr_department", "reason": "fk", "score": 0.0 }, { "object": "main.sales_order", "reason": "fk", "score": 0.0 } ], "objects": [ "main.hr_compensation", "main.hr_employee", "main.v_employee_headcount", "main.sales_order_status", "main.sales_invoice_draft", "main.core_country", "main.core_party", "main.hr_department", "main.sales_order" ], "question": "salary by employee", "selected": 9, "total_objects": 42 } } }, "tool_result_checked": true, "tool_schema_bytes": 16092, "tools": [ "select_schema", "list_objects", "describe_object", "run_query", "semantic_query", "answer", "deep_answer", "refresh_catalog", "health" ], "working": true }, "evidence_strength": "readme_checked", "example_checks": [ { "claim": "demo --prompt prints the DDL the model gets", "command": " demo 'late shipments by carrier' --prompt", "deferred": false, "evidence": "count_pct REAL,\n line_net REAL\n)\n-- FK sales_order_line(id_product) -> cat_product\n-- FK sales_order_line(id_order) -> sales_order\n\nTABLE main.core_party (\n id INTEGER PK,\n display_name TEXT,\n party_type TEXT,\n id_primary_address INTEGER,\n status TEXT,\n created_at TEXT\n)\n-- FK core_party(id_primary_address) -> core_address\n\nTABLE main.cat_product (\n id INTEGER PK,\n sku TEXT,\n name TEXT,\n id_category INTEGER,\n unit_of_measure TEXT,\n list_price REAL,\n status TEXT,\n is_serialised INTEGER\n)\n-- FK cat_product(id_category) -> cat_category (inferred from the column name, not declared)", "expected": "DDL with TABLE main.ship_carrier", "input": "late shipments by carrier --prompt", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "README.md: schemagate demo \"late shipments by carrier\" --prompt # the DDL the model gets", "verdict": "yes" }, { "claim": "offline CLI test suite runs under pytest", "command": " tests/test_cli.py -q -p no:cacheprovider", "deferred": false, "evidence": "............. [100%]\n13 passed in 34.73s", "expected": "13 passed", "input": "tests/test_cli.py", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "TESTING.md: everything on this page runs from `pytest`", "verdict": "yes" } ], "status": "PASS" }, "deviations": [ "Installed from the pinned local checkout (pip install ) instead of PyPI 'pip install schemagate'; the installed version is 1.14.0.", "Installed MCP extra with pip into the venv instead of the README's uvx route (uv is present but was not used).", "The first test run used 'python -m pytest'; the recorded example uses the pytest executable from the venv.", "The checker path /inputs/coldpass-preflight.py does not exist in this guest; used as instructed by the environment.", "Not exercised because they need a model key or a real database: schemagate demo --answer, select against a real engine, describe, studio, certify.", "No product scheduler or cycle exists, so cycles.json has detected:false." ], "fault_owner": null, "main_blocker": null, "operation": "Install schemagate from the pinned source and run its text-to-SQL schema selection: the demo CLI and the schemagate-mcp server (select_schema tool), which filter the schema by the caller's roles so restricted tables are absent or present.", "operation_scope": "Local, offline, bundled 42-object demo schema (no database, no API key, no model call). Python 3.12.3 in a venv at Not covered: --answer (needs a model provider key), describe, studio, certify, and real database engines.", "outcome": "working", "plain_failure": null, "steps": [ "python3 -m venv ", " install (README: pip install schemagate; installed from the pinned checkout instead)", " demo \"salary by employee\" -> hr_compensation absent for caller demo:analyst", " demo \"salary by employee\" --principal okta:hr --role payroll -> main.hr_compensation listed", " install '' (README MCP section needs the mcp extra; uvx was not used)", " --url demo (MCP tool select_schema with principal okta:hr, roles [payroll] -> 9 objects including main.hr_compensation)", " demo \"late shipments by carrier\" --prompt -> DDL with TABLE main.ship_carrier", " tests/test_cli.py -q -p no:cacheprovider -> 13 passed" ], "stuck_step": null, "suggested_fix": null }