{ "checker": { "claims": [ { "claim": "The offline demo runs a real FAIL then PASS loop and writes reports", "command": "python3 -m readtrial demo --out trial-results/claim-demo-checker-ea4c505eb0584602a7028b7a23f462aa", "evidence": "After: PASS (README step exit 0; declared check passed)\ndiff.json: 1 resolved finding; evidence in before/ and after/.\n", "expected": "After: PASS line printed and diff written", "input": "python3 -m readtrial demo --out trial-results/claim-demo", "kind": "claim", "outcome": "Real input produced the expected result: After: PASS (README step exit 0; declared check passed)", "returncode": 0, "source": "The demo finishes in under 20 seconds and writes real sample reports and a diff;", "source_checked": true, "source_file": "README.md", "verdict": "yes" }, { "claim": "Isolated LXC trial needs Claude CLI credentials on the controller", "command": "", "evidence": "Isolated trial requires LXD (doctor reports LXD missing, exit 1) and an authenticated claude CLI on the controller; neither present and no credentials available in this guest", "expected": "", "input": "", "kind": "claim", "outcome": "Isolated trial requires LXD (doctor reports LXD missing, exit 1) and an authenticated claude CLI on the controller; neither present and no credentials available in this guest", "source": "Install/authenticate the Claude CLI on the controller before the trial; credentials", "source_checked": true, "source_file": "README.md", "verdict": "blocked" } ], "cycle_checks": [ { "claim": "Scheduled or looping main behavior", "command": "", "evidence": "Entry points inspected (readtrial/cli.py subcommands, readtrial/*.py). grep for cron, schedule, interval, webhook, --once, --now found no product scheduler or loop. The only repetition is the manual 'rerun' subcommand, run on demand.", "expected": "identify the product trigger", "input": "source inspection", "kind": "cycle", "outcome": "Entry points inspected (readtrial/cli.py subcommands, readtrial/*.py). grep for cron, schedule, interval, webhook, --once, --now found no product scheduler or loop. The only repetition is the manual 'rerun' subcommand, run on demand.", "source": "repository entry points", "verdict": "yes" } ], "details": { "argv": [ "python3", "-m", "readtrial", "demo", "--out", "trial-results/probe-demo-checker-24f3944e8ea84b17966a3f2f8001cecb" ], "kind": "cli", "needs_browser": false, "needs_credentials": false, "returncode": 0, "scope": "product operation", "stderr": "", "stdout": "OFFLINE DEMO: fake agent and fake target; no model, SSH, LXC or repository execution.\nBefore: FAIL (README step exit 3; declared check failed)\nAfter: PASS (README step exit 0; declared check passed)\ndiff.json: 1 resolved finding; evidence in before/ and after/.", "working": false }, "evidence_strength": "partly_checked", "example_checks": [ { "claim": "Repository unit suite runs offline", "command": "python3 -m unittest discover -s tests -q", "deferred": false, "evidence": " written (valid for 24 hours).\nssh:example: AGENT deviation: ignored added cd; selected the exact README command from the clone root: py -3 hello.py --name Check\n\nRecorded 0 requirements; Factory decides gate passage.\nreadtrial: previous run is incompatible or did not fetch source successfully\nreadtrial: --out must be a new folder beneath job.evidence_dir\nreadtrial: model and source hash overrides must match the supplied job\n----------------------------------------------------------------------\nRan 399 tests in 93.608s\n\nOK (skipped=1)", "expected": "tests pass (Ran 399 tests, OK, 1 skipped)", "input": "python3 -m unittest discover -s tests -q", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "README.md Linux quickstart", "verdict": "yes" }, { "claim": "Isolated LXC trial with Claude agent", "command": "", "deferred": false, "evidence": "Missing prerequisites: LXD not installed (readtrial doctor exit 1, 'Missing: LXD installed'), and no authenticated claude CLI on the controller (not on PATH). Not run; offline demo and unit suite ran instead.", "expected": "documented command completes successfully", "input": "", "kind": "example", "outcome": "Missing prerequisites: LXD not installed (readtrial doctor exit 1, 'Missing: LXD installed'), and no authenticated claude CLI on the controller (not on PATH). Not run; offline demo and unit suite ran instead.", "source": "README.md Isolated trials", "verdict": "blocked" } ], "status": "PASS" }, "deviations": [ "Isolated trial not run: LXD not installed and claude CLI not available/authenticated; installing LXD and credentials was not done. Recorded as a prerequisite, not a product failure.", "Help output was used only as setup and does not count as a product claim.", "Unit suite (399 tests) also writes .scratch/ output; gitignored." ], "fault_owner": null, "main_blocker": "The saved checks do not fully confirm documented product behavior.", "operation": "Run readtrial's documented offline demo (FAIL then PASS loop) and its unit test suite from the pinned checkout", "operation_scope": "Linux, Python 3.12.3 (README needs 3.11+), no network, no model, no LXD, no agent CLI. The isolated LXC trial (README 'Isolated trials') was NOT exercised: it needs LXD (doctor exit 1, LXD missing) and an authenticated claude CLI, neither present.", "outcome": "unverified", "plain_failure": null, "steps": [ "git init; git remote add origin https://github.com/TayHCode/readtrial; git fetch --depth 1 origin 1662f23f8b0d88c1fe3d028265786f3f3b141171; git reset --mixed FETCH_HEAD", "python3 --version", "python3 -m readtrial --help", "python3 -m readtrial demo --out trial-results/demo", "python3 -m unittest discover -s tests -q", "python3 -m readtrial doctor (setup check, exit 1: LXD missing)" ], "stuck_step": "python3 -m readtrial demo --out trial-results/probe-demo-checker-24f3944e8ea84b17966a3f2f8001cecb", "suggested_fix": "None for the tested scope. To test the full agent trial, install LXD with a size-capped pool and an authenticated claude CLI on the controller, then run the README 'Isolated trials' command." }