{
  "run_id": "ashishsinha1602--schemagate-claude-haiku-5-5-1793152981068694457",
  "checker": {
    "claims": [
      {
        "claim": "Restricted tables are scoped to the caller's roles: with the payroll role, hr_compensation is selected",
        "command": "<guest-path> --url demo tools/call {\"tool\": \"select_schema\", \"arguments\": {\"question\": \"salary by employee\", \"principal\": \"okta:hr\", \"roles\": [\"payroll\"]}}",
        "evidence": "\u2026hr_compensation\\\",\\n    \\\"main.hr_employee\\\",\\n    \\\"main.v_employee_headcount\\\",\\n    \\\"main.sales_order_status\\\",\\n    \\\"main.sales_invoice_draft\\\",\\n    \\\"main.core_country\\\",\\n    \\\"main.core_party\\\",\\n    \\\"main.hr_department\\\",\\n    \\\"main.sales_order\\\"\\n  ],\\n  \\\"ddl\\\": \\\"-- salary history; restricted\\\\nTABLE main.hr_compensation (\\\\n  id INTEGER PK,\\\\n  id_employee INTEGER,\\\\n  effective_from TEXT,\\\\n  annual_amount REAL,\\\\n  currency TEXT,\\\\n  pay_grade TEXT\\\\n)\\\\n-- FK hr_compensation(id_employee) -> hr_employee\\\\n\\\\nTABLE main.hr_employee (\\\\n  id INTEGER PK,\\\\n  id_party INTEGER,\\\\",
        "expected": "hr_compensation is in the selection",
        "input": "{\"tool\": \"select_schema\", \"arguments\": {\"question\": \"salary by employee\", \"principal\": \"okta:hr\", \"roles\": [\"payroll\"]}}",
        "kind": "claim",
        "outcome": "Real input produced the expected result: hr_compensation",
        "source": "Absent, not ranked low. A table the caller may not read never enters the",
        "source_checked": true,
        "source_file": "README.md",
        "verdict": "yes"
      },
      {
        "claim": "The demo CLI with --principal okta:hr --role payroll ranks hr_compensation as selected",
        "command": "<guest-path> demo 'salary by employee' --principal okta:hr --role payroll",
        "evidence": "\u2026main.hr_compensation\n  main.hr_employee\n  main.v_employee_headcount\n  main.sales_order_status\n  main.sales_invoice_draft\n  main.core_country\n  main.core_party\n  main.hr_department\n  main.sales_order\n\n",
        "expected": "main.hr_compensation listed",
        "input": "salary by employee, principal okta:hr, role payroll",
        "kind": "claim",
        "outcome": "Real input produced the expected result: main.hr_compensation",
        "returncode": 0,
        "source": "schemagate demo \"salary by employee\" --principal okta:hr --role payroll  # now it is first",
        "source_checked": true,
        "source_file": "README.md",
        "verdict": "yes"
      }
    ],
    "cycle_checks": [
      {
        "claim": "Scheduled or looping main behavior",
        "command": "",
        "evidence": "Entry points inspected: schemagate CLI (demo, select, studio, describe, certify) and schemagate-mcp are one-shot commands or servers that answer requests; grep of src/schemagate for cron, schedule, --once, webhook and while-True loops found no scheduler or product cycle. Each command runs once per invocation.",
        "expected": "identify the product trigger",
        "input": "source inspection",
        "kind": "cycle",
        "outcome": "Entry points inspected: schemagate CLI (demo, select, studio, describe, certify) and schemagate-mcp are one-shot commands or servers that answer requests; grep of src/schemagate for cron, schedule, --once, webhook and while-True loops found no scheduler or product cycle. Each command runs once per invocation.",
        "source": "repository entry points",
        "verdict": "yes"
      }
    ],
    "details": {
      "argv": [
        "<guest-path>",
        "--url",
        "demo"
      ],
      "kind": "mcp",
      "scope": "product operation",
      "serverInfo": {
        "name": "schemagate",
        "version": "1.14.0"
      },
      "tool_result": {
        "content": [
          {
            "text": "{\n  \"question\": \"salary by employee\",\n  \"selected\": 9,\n  \"total_objects\": 42,\n  \"objects\": [\n    \"main.hr_compensation\",\n    \"main.hr_employee\",\n    \"main.v_employee_headcount\",\n    \"main.sales_order_status\",\n    \"main.sales_invoice_draft\",\n    \"main.core_country\",\n    \"main.core_party\",\n    \"main.hr_department\",\n    \"main.sales_order\"\n  ],\n  \"ddl\": \"-- salary history; restricted\\nTABLE main.hr_compensation (\\n  id INTEGER PK,\\n  id_employee INTEGER,\\n  effective_from TEXT,\\n  annual_amount REAL,\\n  currency TEXT,\\n  pay_grade TEXT\\n)\\n-- FK hr_compensation(id_employee) -> hr_employee\\n\\nTABLE main.hr_employee (\\n  id INTEGER PK,\\n  id_party INTEGER,\\n  employee_number TEXT,\\n  hired_on TEXT,\\n  id_manager INTEGER,\\n  id_department INTEGER,\\n  status TEXT\\n)\\n-- FK hr_employee(id_manager) -> hr_employee\\n-- FK hr_employee(id_party) -> core_party\\n-- FK hr_employee(id_department) -> hr_department  (inferred from the column name, not declared)\\n\\nVIEW main.v_employee_headcount (\\n  department TEXT,\\n  headcount NULL\\n)\\n\\nTABLE main.sales_order_status (\\n  id INTEGER PK,\\n  code TEXT,\\n  label TEXT,\\n  is_terminal INTEGER\\n)\\n\\n-- pre-issue drafts only; NOT real revenue, exclude from financials\\nTABLE main.sales_invoice_draft (\\n  id INTEGER PK,\\n  id_order INTEGER,\\n  drafted_on TEXT,\\n  state TEXT,\\n  total_gross REAL\\n)\\n-- FK sales_invoice_draft(id_order) -> sales_order\\n\\nTABLE main.core_country (\\n  id INTEGER PK,\\n  iso2 TEXT,\\n  name TEXT,\\n  region TEXT\\n)\\n\\nTABLE main.core_party (\\n  id INTEGER PK,\\n  display_name TEXT,\\n  party_type TEXT,\\n  id_primary_address INTEGER,\\n  status TEXT,\\n  created_at TEXT\\n)\\n-- FK core_party(id_primary_address) -> core_address\\n\\nTABLE main.hr_department (\\n  id INTEGER PK,\\n  code TEXT,\\n  name TEXT,\\n  cost_centre TEXT\\n)\\n\\nTABLE main.sales_order (\\n  id INTEGER PK,\\n  order_number TEXT,\\n  id_customer INTEGER,\\n  id_ship_address INTEGER,\\n  id_quote INTEGER,\\n  ordered_at TEXT,\\n  id_status INTEGER,\\n  currency TEXT,\\n  total_net REAL\\n)\\n-- FK sales_order(id_quote) -> sales_quote\\n-- FK sales_order(id_ship_address) -> core_address\\n-- FK sales_order(id_customer) -> crm_customer\\n-- FK sales_order(id_status) -> sales_order_status  (inferred from the column name, not declared)\",\n  \"explain\": [\n    {\n      \"object\": \"main.hr_compensation\",\n      \"score\": 0.0489,\n      \"reason\": \"hybrid\"\n    },\n    {\n      \"object\": \"main.hr_employee\",\n      \"score\": 0.0487,\n      \"reason\": \"hybrid\"\n    },\n    {\n      \"object\": \"main.v_employee_headcount\",\n      \"score\": 0.0481,\n      \"reason\": \"hybrid\"\n    },\n    {\n      \"object\": \"main.sales_order_status\",\n      \"score\": 0.0156,\n      \"reason\": \"vector\"\n    },\n    {\n      \"object\": \"main.sales_invoice_draft\",\n      \"score\": 0.0154,\n      \"reason\": \"vector\"\n    },\n    {\n      \"object\": \"main.core_country\",\n      \"score\": 0.0152,\n      \"reason\": \"vector\"\n    },\n    {\n      \"object\": \"main.core_party\",\n      \"score\": 0.0,\n      \"reason\": \"fk\"\n    },\n    {\n      \"object\": \"main.hr_department\",\n      \"score\": 0.0,\n      \"reason\": \"fk\"\n    },\n    {\n      \"object\": \"main.sales_order\",\n      \"score\": 0.0,\n      \"reason\": \"fk\"\n    }\n  ]\n}",
            "type": "text"
          }
        ],
        "isError": false,
        "structuredContent": {
          "result": {
            "ddl": "-- salary history; restricted\nTABLE main.hr_compensation (\n  id INTEGER PK,\n  id_employee INTEGER,\n  effective_from TEXT,\n  annual_amount REAL,\n  currency TEXT,\n  pay_grade TEXT\n)\n-- FK hr_compensation(id_employee) -> hr_employee\n\nTABLE main.hr_employee (\n  id INTEGER PK,\n  id_party INTEGER,\n  employee_number TEXT,\n  hired_on TEXT,\n  id_manager INTEGER,\n  id_department INTEGER,\n  status TEXT\n)\n-- FK hr_employee(id_manager) -> hr_employee\n-- FK hr_employee(id_party) -> core_party\n-- FK hr_employee(id_department) -> hr_department  (inferred from the column name, not declared)\n\nVIEW main.v_employee_headcount (\n  department TEXT,\n  headcount NULL\n)\n\nTABLE main.sales_order_status (\n  id INTEGER PK,\n  code TEXT,\n  label TEXT,\n  is_terminal INTEGER\n)\n\n-- pre-issue drafts only; NOT real revenue, exclude from financials\nTABLE main.sales_invoice_draft (\n  id INTEGER PK,\n  id_order INTEGER,\n  drafted_on TEXT,\n  state TEXT,\n  total_gross REAL\n)\n-- FK sales_invoice_draft(id_order) -> sales_order\n\nTABLE main.core_country (\n  id INTEGER PK,\n  iso2 TEXT,\n  name TEXT,\n  region TEXT\n)\n\nTABLE main.core_party (\n  id INTEGER PK,\n  display_name TEXT,\n  party_type TEXT,\n  id_primary_address INTEGER,\n  status TEXT,\n  created_at TEXT\n)\n-- FK core_party(id_primary_address) -> core_address\n\nTABLE main.hr_department (\n  id INTEGER PK,\n  code TEXT,\n  name TEXT,\n  cost_centre TEXT\n)\n\nTABLE main.sales_order (\n  id INTEGER PK,\n  order_number TEXT,\n  id_customer INTEGER,\n  id_ship_address INTEGER,\n  id_quote INTEGER,\n  ordered_at TEXT,\n  id_status INTEGER,\n  currency TEXT,\n  total_net REAL\n)\n-- FK sales_order(id_quote) -> sales_quote\n-- FK sales_order(id_ship_address) -> core_address\n-- FK sales_order(id_customer) -> crm_customer\n-- FK sales_order(id_status) -> sales_order_status  (inferred from the column name, not declared)",
            "explain": [
              {
                "object": "main.hr_compensation",
                "reason": "hybrid",
                "score": 0.0489
              },
              {
                "object": "main.hr_employee",
                "reason": "hybrid",
                "score": 0.0487
              },
              {
                "object": "main.v_employee_headcount",
                "reason": "hybrid",
                "score": 0.0481
              },
              {
                "object": "main.sales_order_status",
                "reason": "vector",
                "score": 0.0156
              },
              {
                "object": "main.sales_invoice_draft",
                "reason": "vector",
                "score": 0.0154
              },
              {
                "object": "main.core_country",
                "reason": "vector",
                "score": 0.0152
              },
              {
                "object": "main.core_party",
                "reason": "fk",
                "score": 0.0
              },
              {
                "object": "main.hr_department",
                "reason": "fk",
                "score": 0.0
              },
              {
                "object": "main.sales_order",
                "reason": "fk",
                "score": 0.0
              }
            ],
            "objects": [
              "main.hr_compensation",
              "main.hr_employee",
              "main.v_employee_headcount",
              "main.sales_order_status",
              "main.sales_invoice_draft",
              "main.core_country",
              "main.core_party",
              "main.hr_department",
              "main.sales_order"
            ],
            "question": "salary by employee",
            "selected": 9,
            "total_objects": 42
          }
        }
      },
      "tool_result_checked": true,
      "tool_schema_bytes": 16092,
      "tools": [
        "select_schema",
        "list_objects",
        "describe_object",
        "run_query",
        "semantic_query",
        "answer",
        "deep_answer",
        "refresh_catalog",
        "health"
      ],
      "working": true
    },
    "evidence_strength": "readme_checked",
    "example_checks": [
      {
        "claim": "demo --prompt prints the DDL the model gets",
        "command": "<guest-path> demo 'late shipments by carrier' --prompt",
        "deferred": false,
        "evidence": "count_pct REAL,\n  line_net REAL\n)\n-- FK sales_order_line(id_product) -> cat_product\n-- FK sales_order_line(id_order) -> sales_order\n\nTABLE main.core_party (\n  id INTEGER PK,\n  display_name TEXT,\n  party_type TEXT,\n  id_primary_address INTEGER,\n  status TEXT,\n  created_at TEXT\n)\n-- FK core_party(id_primary_address) -> core_address\n\nTABLE main.cat_product (\n  id INTEGER PK,\n  sku TEXT,\n  name TEXT,\n  id_category INTEGER,\n  unit_of_measure TEXT,\n  list_price REAL,\n  status TEXT,\n  is_serialised INTEGER\n)\n-- FK cat_product(id_category) -> cat_category  (inferred from the column name, not declared)",
        "expected": "DDL with TABLE main.ship_carrier",
        "input": "late shipments by carrier --prompt",
        "kind": "example",
        "outcome": "command completed successfully",
        "returncode": 0,
        "source": "README.md: schemagate demo \"late shipments by carrier\" --prompt   # the DDL the model gets",
        "verdict": "yes"
      },
      {
        "claim": "offline CLI test suite runs under pytest",
        "command": "<guest-path> tests/test_cli.py -q -p no:cacheprovider",
        "deferred": false,
        "evidence": ".............                                                            [100%]\n13 passed in 34.73s",
        "expected": "13 passed",
        "input": "tests/test_cli.py",
        "kind": "example",
        "outcome": "command completed successfully",
        "returncode": 0,
        "source": "TESTING.md: everything on this page runs from `pytest`",
        "verdict": "yes"
      }
    ],
    "status": "PASS"
  },
  "diagnosis": null
}