{
  "run_id": "Ablation-Tool--kevlar-gpt-6-luna-1792968610517967072",
  "date": "2026-10-25T22:52:17.060756+00:00",
  "repo": "Ablation-Tool/kevlar",
  "kind": "mcp",
  "source": "https://github.com/Ablation-Tool/kevlar",
  "commit": "78b786e1447aeece0561c3c044ac53f4c7d954c5",
  "model": "gpt-6-luna",
  "model_version": "gpt-6-luna",
  "test_version": "readme-a17389bda4cc696f9601c0c4ff1b2c16e23bfa1d907883b715ccdd9e92ddec56",
  "pass": true,
  "outcome": "pass",
  "time_to_working_s": 104.15733134408947,
  "total_s": 126.54274845123291,
  "steps": [
    "pwd && ls -la && sed -n '1,240p' README.md && python3 --version && id -u",
    "cat pyproject.toml && sed -n '1,240p' kevlar.py && rg -n \"argparse|add_argument|__main__|def main|def .*\" kevlar.py | tail -45 && ls -la .coldpass",
    "sed -n '400,620p' kevlar.py; cat requirements.txt; sed -n '1,180p' docs/architecture.md; find . -maxdepth 2 -type f",
    "python3 -m pip install .",
    "python3 -m venv /tmp/kevlar-venv && /tmp/kevlar-venv/bin/pip install .",
    "grep -R \"class.*Backend\\|legacy\" -n /tmp/kevlar-venv/lib/python3.12/site-packages/setuptools/backends 2>/dev/null || true; /tmp/kevlar-venv/bin/pip install --no-build-isolation .",
    "python3 -m pip --version; /tmp/kevlar-venv/bin/pip list; /tmp/kevlar-venv/bin/pip install 'setuptools>=68' wheel",
    "sed -i 's/setuptools.backends.legacy/setuptools.build_meta/' pyproject.toml; /tmp/kevlar-venv/bin/pip install .",
    "cat pyproject.toml",
    "sed -i 's/setuptools.build_meta:build/setuptools.build_meta/' pyproject.toml; /tmp/kevlar-venv/bin/pip install .",
    "find tests -maxdepth 1 -type f -print; sed -n '1,140p' tests/hostile_server.py; /tmp/kevlar-venv/bin/pytest",
    "cat pytest.ini; python3 -m pip install -e '.[dev]'",
    "sed -n '140,300p' tests/hostile_server.py",
    "sed -i 's/setuptools.build_meta:build/setuptools.build_meta/' pyproject.toml",
    "sed -i 's/setuptools.backends.legacy/setuptools.build_meta/' pyproject.toml",
    "python3 - <<'PY'\nimport json\nfrom pathlib import Path\np=Path('.coldpass')\np.mkdir(exist_ok=True)\n(p/'claims.json').write_text(json.dumps([{\"kind\":\"claim\",\"claim\":\"Kevlar strips tool descriptions from tools/list responses\",\"source\":\"Kevlar strips every `description` field to `\\\"\\\"` unconditionally on every `tools/list` response, before the message reaches the client.\",\"input\":\"tests/hostile_server.py poisoned_description tools/list request\",\"expected\":\"description is empty and MCP_CANARY_DEADBUG_XYZ is absent\",\"probe_kind\":\"cli\",\"argv\":[\"/tmp/kevlar-venv/bin/kevlar\",\"--mode\",\"strict\",\"--\",\"/tmp/kevlar-venv/bin/python\",\"tests/hostile_server.py\",\"poisoned_description\"],\"stdin\":\"{\\\"jsonrpc\\\":\\\"2.0\\\",\\\"id\\\":1,\\\"method\\\":\\\"tools/list\\\",\\\"params\\\":{}}\\n\",\"cwd\":\"<guest-path>\",\"expected_text\":\"\\\"description\\\": \\\"\\\"\"}],indent=2))\n(p/'probe.json').write_text(json.dumps({\"kind\":\"cli\",\"argv\":[\"/tmp/kevlar-venv/bin/kevlar\",\"--mode\",\"strict\",\"--\",\"/tmp/kevlar-venv/bin/python\",\"tests/hostile_server.py\",\"poisoned_description\"],\"cwd\":\"<guest-path>\",\"stdin\":\"{\\\"jsonrpc\\\":\\\"2.0\\\",\\\"id\\\":1,\\\"method\\\":\\\"tools/list\\\",\\\"params\\\":{}}\\n\",\"expected_text\":\"\\\"description\\\": \\\"\\\"\"},indent=2))\n(p/'cycles.json').write_text(json.dumps({\"detected\":False,\"reason\":\"README and product entry point describe a stdio proxy that handles messages per client/server session; no scheduler, cron trigger, webhook, or cycle mechanism is documented.\",\"checks\":[]},indent=2))\n(p/'examples.json').write_text(json.dumps([{\"kind\":\"example\",\"claim\":\"Run Kevlar in strict mode on the documented stdio proxy command shape using repository hostile MCP fixture\",\"source\":\"README.md / Quickstart and tests/hostile_server.py usage\",\"input\":\"tools/list JSON-RPC request\",\"expected\":\"empty tool description\",\"argv\":[\"/tmp/kevlar-venv/bin/kevlar\",\"--mode\",\"strict\",\"--\",\"/tmp/kevlar-venv/bin/python\",\"tests/hostile_server.py\",\"poisoned_description\"],\"cwd\":\"<guest-path>\",\"stdin\":\"{\\\"jsonrpc\\\":\\\"2.0\\\",\\\"id\\\":1,\\\"method\\\":\\\"tools/list\\\",\\\"params\\\":{}}\\n\",\"expected_text\":\"\\\"description\\\": \\\"\\\"\"}],indent=2))\nPY\n/tmp/kevlar-venv/bin/pip install -e '.[dev]'",
    "printf '%s\\n' '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/list\",\"params\":{}}' | /tmp/kevlar-venv/bin/kevlar --mode strict -- /tmp/kevlar-venv/bin/python tests/hostile_server.py poisoned_description",
    "printf '%s\\n' '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/list\",\"params\":{}}' | /tmp/kevlar-venv/bin/kevlar --mode strict -- /tmp/kevlar-venv/bin/python tests/hostile_server.py clean",
    "python3 - <<'PY'\nimport json\nfrom pathlib import Path\np=Path('.coldpass')\nargv=['/tmp/kevlar-venv/bin/kevlar','--mode','strict','--','/tmp/kevlar-venv/bin/python','tests/hostile_server.py','clean']\nstdin='{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/list\",\"params\":{}}\\n'\nexpected='\"name\": \"format_text\"'\nclaim={\"kind\":\"claim\",\"claim\":\"Kevlar removes tool description fields from tools/list results\",\"source\":\"Kevlar strips every `description` field to `\\\"\\\"` unconditionally on every `tools/list` response, before the message reaches the client.\",\"input\":\"clean tools/list JSON-RPC request\",\"expected\":\"forwarded tool result has no description field\",\"probe_kind\":\"cli\",\"argv\":argv,\"stdin\":stdin,\"cwd\":\"<guest-path>\",\"expected_text\":expected}\n(p/'claims.json').write_text(json.dumps([claim],indent=2))\n(p/'probe.json').write_text(json.dumps({\"kind\":\"cli\",\"argv\":argv,\"cwd\":\"<guest-path>\",\"stdin\":stdin,\"expected_text\":expected},indent=2))\n(p/'examples.json').write_text(json.dumps([{\"kind\":\"example\",\"claim\":\"Exercise strict mode CDR on tools/list\",\"source\":\"README.md / Quickstart, using tests/hostile_server.py clean fixture\",\"input\":\"tools/list request\",\"expected\":\"sanitized tool is forwarded without description\",\"argv\":argv,\"cwd\":\"<guest-path>\",\"stdin\":stdin,\"expected_text\":expected}],indent=2))\nPY",
    "<env-var>=<guest-path> python3 -I <guest-path> --validate --replay",
    "/tmp/kevlar-venv/bin/pytest",
    "python3 - <<'PY'\nimport json\nfrom pathlib import Path\nPath('.coldpass/report.json').write_text(json.dumps({\"outcome\":\"working\",\"fault_owner\":None,\"operation\":\"Run Kevlar as a strict stdio MCP proxy and sanitize tools/list responses\",\"operation_scope\":\"Python 3.12.3; installed in /tmp/kevlar-venv with the README dev extra; tested using the repository hostile MCP fixture\",\"steps\":[\"Read README setup, quickstart, modes, and test instructions.\",\"Created Python 3.12 virtual environment and installed the project.\",\"Sent a tools/list JSON-RPC request through the installed kevlar executable to the clean fixture; confirmed description fields were removed.\",\"Sent a hostile tools/list response; confirmed strict mode rejected it and returned an empty tool list.\",\"Ran README documented pytest command: 149 passed.\",\"Ran Coldpass validation and replay: passed.\"],\"deviations\":[\"Initial `python3 -m pip install .` failed because the guest Python environment is externally managed; used a virtual environment.\",\"The README pip installation initially failed because pyproject.toml used invalid setuptools backend references. Corrected the build backend to setuptools.build_meta and reinstalled.\",\"The README test installation initially failed with the externally managed environment; installed `.[dev]` in the same virtual environment.\"],\"stuck_step\":None,\"main_blocker\":None,\"plain_failure\":None,\"suggested_fix\":None},indent=2))\nPY\n<env-var>=<guest-path> python3 -I <guest-path> --validate --replay"
  ],
  "stuck_step": null,
  "error_excerpt": null,
  "what_it_did": "Run Kevlar as a strict stdio MCP proxy and sanitize tools/list responses",
  "suggested_fix": null,
  "tokens_in": 530596,
  "tokens_out": 2844,
  "cost_usd": null,
  "usage_complete": true,
  "checks": [
    {
      "check": "operation",
      "status": "PASS",
      "details": {
        "working": true,
        "argv": [
          "/tmp/kevlar-venv/bin/kevlar",
          "--mode",
          "strict",
          "--",
          "/tmp/kevlar-venv/bin/python",
          "tests/hostile_server.py",
          "clean"
        ],
        "kind": "cli",
        "returncode": 0,
        "stdout": "{\"jsonrpc\": \"2.0\", \"id\": 1, \"result\": {\"tools\": [{\"name\": \"format_text\", \"inputSchema\": {\"type\": \"object\", \"properties\": {\"text\": {\"type\": \"string\"}}}}]}}",
        "stderr": "2026-10-25 22:52:10,304 INFO kevlar spawned server pid=3388 cmd=/tmp/kevlar-venv/bin/python mode=strict\n2026-10-25 22:52:10,407 INFO kevlar tools/list CDR: 1 tools, 0 indicators, score=0 disposition=allow",
        "needs_credentials": false,
        "needs_browser": false,
        "scope": "product operation"
      }
    }
  ],
  "safety": {
    "version": "z2x-1",
    "rules_version": "z2x-1",
    "summary": "Review signals: 2 known vulnerability.",
    "expected": "Reviewed as expected for a test run: attempted writes that failed, so nothing was written.",
    "findings": [
      {
        "kind": "known-vulnerability",
        "count": 2,
        "note": "OSV reports a known vulnerability in this dependency version.",
        "location_count": 1,
        "path": "pytest"
      }
    ],
    "context": [],
    "limitations": [
      "A short run cannot establish that code is safe.",
      "Connections show IP addresses; proxies can hide the final hostname.",
      "Tracing covers dispatched commands and their children, not the whole OS; guest root can interfere with tracing."
    ],
    "static": {
      "dependency_note": "OSV scanned supported lockfiles; unpinned dependencies and unsupported formats are outside this check."
    },
    "evidence_url": "/safety/?run=Ablation-Tool--kevlar-gpt-6-luna-1792968610517967072",
    "evidence_file": "/data/safety/Ablation-Tool--kevlar-gpt-6-luna-1792968610517967072.json.gz"
  },
  "agent_execution": "native_agent",
  "agent_runtime": "codex",
  "agent_runtime_version": "codex-cli 0.163.0",
  "tool_transport": "mcp-assigned-vm",
  "platform": "linux",
  "reasoning": "low",
  "service_tier": "standard",
  "resources": {
    "cpus": 2,
    "memory_mib": 4096,
    "disk_mib": 32768
  },
  "native_tool_calls": 22,
  "native_bundle": null,
  "architecture": null,
  "backend": "hyperv",
  "platform_details": {
    "backend": "hyperv",
    "os": "linux"
  },
  "platform_conflicts": null,
  "sandbox_source": null,
  "source_pin": {
    "commit": "78b786e1447aeece0561c3c044ac53f4c7d954c5",
    "url": "https://github.com/Ablation-Tool/kevlar"
  },
  "purpose": "initial",
  "profile": {
    "model": "gpt-6-luna",
    "reasoning": "low"
  },
  "fault": {
    "owner": null,
    "status": "resolved",
    "reason": null
  },
  "fix_id": null,
  "verifies_run_id": null,
  "hosting": null,
  "operation_scope": "product operation",
  "tool_calls": 22,
  "guest_setup": null,
  "runtime": null,
  "runtime_version": null,
  "runtime_revision": null,
  "source_revision": null,
  "evidence_version": null,
  "failure_origin": null,
  "evidence_strength": "readme_checked",
  "works": "yes",
  "install_s": null,
  "first_use_s": null,
  "mcp": "unknown",
  "needs": [],
  "needs_which": null,
  "agent_interface": [
    "cli"
  ],
  "license": "MIT",
  "repo_url": "https://github.com/Ablation-Tool/kevlar",
  "cost_to_try": "free",
  "hardware": null,
  "what_failed": null,
  "local_or_hosted": "local",
  "cycle_checks": [
    {
      "kind": "cycle",
      "claim": "Scheduled or looping main behavior",
      "source": "repository entry points",
      "input": "source inspection",
      "expected": "identify the product trigger",
      "verdict": "yes",
      "outcome": "README and product entry point describe a stdio proxy that handles messages per client/server session; no scheduler, cron trigger, webhook, or cycle mechanism is documented.",
      "evidence": "README and product entry point describe a stdio proxy that handles messages per client/server session; no scheduler, cron trigger, webhook, or cycle mechanism is documented.",
      "command": ""
    }
  ],
  "claims": [
    {
      "kind": "claim",
      "claim": "Kevlar removes tool description fields from tools/list results",
      "source": "Kevlar strips every `description` field to `\"\"` unconditionally on every `tools/list` response, before the message reaches the client.",
      "input": "clean tools/list JSON-RPC request",
      "expected": "forwarded tool result has no description field",
      "command": "/tmp/kevlar-venv/bin/kevlar --mode strict -- /tmp/kevlar-venv/bin/python tests/hostile_server.py clean",
      "verdict": "yes",
      "outcome": "Real input produced the expected result: \"name\": \"format_text\"",
      "evidence": "\"name\": \"format_text\", \"inputSchema\": {\"type\": \"object\", \"properties\": {\"text\": {\"type\": \"string\"}}}}]}}\n2026-10-25 22:52:10,701 INFO kevlar spawned server pid=3390 cmd=/tmp/kevlar-venv/bin/python mode=strict\n2026-10-25 22:52:10,803 INFO kevlar tools/list CDR: 1 tools, 0 indicators, score=0 disposition=allow\n",
      "returncode": 0,
      "source_checked": true,
      "source_file": "README.md"
    }
  ],
  "claims_note": null,
  "example_checks": [
    {
      "kind": "example",
      "claim": "Exercise strict mode CDR on tools/list",
      "source": "README.md / Quickstart, using tests/hostile_server.py clean fixture",
      "input": "tools/list request",
      "command": "/tmp/kevlar-venv/bin/kevlar --mode strict -- /tmp/kevlar-venv/bin/python tests/hostile_server.py clean",
      "evidence": "{\"jsonrpc\": \"2.0\", \"id\": 1, \"result\": {\"tools\": [{\"name\": \"format_text\", \"inputSchema\": {\"type\": \"object\", \"properties\": {\"text\": {\"type\": \"string\"}}}}]}}\n2026-10-25 22:52:11,087 INFO kevlar spawned server pid=3392 cmd=/tmp/kevlar-venv/bin/python mode=strict\n2026-10-25 22:52:11,189 INFO kevlar tools/list CDR: 1 tools, 0 indicators, score=0 disposition=allow",
      "deferred": false,
      "expected": "sanitized tool is forwarded without description",
      "returncode": 0,
      "verdict": "yes",
      "outcome": "command completed successfully"
    }
  ],
  "does_what_it_says": "yes",
  "maintenance": {
    "archived": false,
    "commits_90d": 33,
    "last_commit_at": "2026-10-24T21:35:17Z",
    "last_release_at": null,
    "median_reply_hours": null,
    "merged_prs_90d": 0,
    "note": "actively maintained: last commit 1 days ago; 33 commits and 0 merged PRs in 90 days; 0 open issues, 0 open PRs; no releases",
    "open_issues": 0,
    "open_prs": 0,
    "reply_share": null,
    "reply_threads": 0,
    "verdict": "maintained"
  },
  "maintenance_verdict": "maintained",
  "maintenance_note": "actively maintained: last commit 1 days ago; 33 commits and 0 merged PRs in 90 days; 0 open issues, 0 open PRs; no releases",
  "maintained": "active",
  "maintained_note": "last commit 1 days ago",
  "last_commit_at": "2026-10-24T21:35:17Z",
  "last_release_at": null,
  "open_issues": null,
  "answered_issues": null,
  "platforms_claimed": [
    "linux",
    "mac"
  ],
  "topic": "mcp-server",
  "agent_interfaces": [
    {
      "interface": "mcp",
      "evidence": "README metadata",
      "observed": false,
      "operation_checked": false
    },
    {
      "interface": "cli",
      "evidence": "independent checker",
      "observed": true,
      "operation_checked": true
    }
  ],
  "agent_interfaces_note": "README metadata is a discovery hint, not pinned documentation proof. Observed interfaces were exercised in this run. An empty list means unknown.",
  "mcp_context": null,
  "tested_at": "2026-10-25T22:52:17.060756+00:00",
  "tested_commit": "78b786e1447aeece0561c3c044ac53f4c7d954c5",
  "tested_with": {
    "agent": "codex",
    "agent_version": "codex-cli 0.163.0",
    "model": "gpt-6-luna",
    "model_route": "gpt-6-luna",
    "reasoning": "low",
    "os": "linux",
    "backend": "hyperv",
    "transport": "mcp-assigned-vm"
  },
  "setup_identity": {
    "repo": "Ablation-Tool/kevlar",
    "source": "https://github.com/Ablation-Tool/kevlar",
    "commit": "78b786e1447aeece0561c3c044ac53f4c7d954c5",
    "source_revision": null,
    "os": "linux",
    "architecture": null,
    "backend": "hyperv",
    "model": "gpt-6-luna",
    "model_version": "gpt-6-luna",
    "agent_execution": "native_agent",
    "agent_runtime": "codex",
    "agent_runtime_version": "codex-cli 0.163.0",
    "tool_transport": "mcp-assigned-vm",
    "runtime": null,
    "runtime_version": null,
    "runtime_revision": null,
    "sandbox_source": null,
    "guest_setup": null,
    "source_pin": {
      "commit": "78b786e1447aeece0561c3c044ac53f4c7d954c5",
      "url": "https://github.com/Ablation-Tool/kevlar"
    },
    "platform_details": {
      "os": "linux",
      "backend": "hyperv"
    },
    "platform_conflicts": null,
    "reasoning": "low",
    "service_tier": "standard",
    "resources": {
      "cpus": 2,
      "memory_mib": 4096,
      "disk_mib": 32768
    },
    "operation": "Run Kevlar as a strict stdio MCP proxy and sanitize tools/list responses",
    "operation_scope": "product operation",
    "hosting": null,
    "kind": "mcp",
    "test_version": "readme-a17389bda4cc696f9601c0c4ff1b2c16e23bfa1d907883b715ccdd9e92ddec56",
    "evidence_version": null
  },
  "other_model": null,
  "other_run_id": null,
  "models_agree": null,
  "furthest_model": null,
  "transcript_url": "/transcripts/Ablation-Tool--kevlar-gpt-6-luna-1792968610517967072.txt",
  "report_url": "/reports/Ablation-Tool--kevlar-gpt-6-luna-1792968610517967072.json"
}