{ "checker": { "claims": [ { "claim": "Evaluation prints top-1 accuracy on the held-out test file", "command": " runs/synthetic.pt data/synthetic/test.jsonl", "evidence": "\"top1\": 0.9975,\n \"top3\": 1.0,\n \"ece\": 0.004109287678147666,\n \"examples\": 400\n },\n \"shuffled_context\": {\n \"top1\": 0.2275,\n \"top3\": 0.6825,\n \"ece\": 0.6250180058996193,\n \"examples\": 400\n },\n \"device\": \"cpu\"\n}\n", "expected": "model top1 printed as 0.9975", "input": "runs/synthetic.pt data/synthetic/test.jsonl", "kind": "claim", "outcome": "Real input produced the expected result: \"top1\": 0.9975", "returncode": 0, "source": "The evaluation prints top-1 accuracy, which is the fraction of correct first choices.", "source_checked": true, "source_file": "README.md", "verdict": "yes" }, { "claim": "Score one new menu with the trained model", "command": " runs/synthetic.pt --context 'Choose the exact badge amber badger. Badge: amber badger.' --option 'azure crane' --option 'amber badger' --option 'gold heron'", "evidence": "amber badger\",\n \"probability\": 1.0\n },\n {\n \"option\": \"gold heron\",\n \"probability\": 4.9812562963325036e-08\n }\n]\n", "expected": "amber badger receives probability near 1.0", "input": "context about amber badger with three options", "kind": "claim", "outcome": "Real input produced the expected result: amber badger", "returncode": 0, "source": "They create local synthetic data, train on it, evaluate the saved model and score one new menu.", "source_checked": true, "source_file": "README.md", "verdict": "yes" }, { "claim": "Shuffled-context control is reported alongside the model", "command": " runs/synthetic.pt data/synthetic/test.jsonl", "evidence": "\"top1\": 0.2275,\n \"top3\": 0.6825,\n \"ece\": 0.6250180058996193,\n \"examples\": 400\n },\n \"device\": \"cpu\"\n}\n", "expected": "shuffled_context top1 printed as 0.2275", "input": "runs/synthetic.pt data/synthetic/test.jsonl", "kind": "claim", "outcome": "Real input produced the expected result: \"top1\": 0.2275", "returncode": 0, "source": "The command also prints a shuffled-context control, which pairs each menu with the wrong context.", "source_checked": true, "source_file": "README.md", "verdict": "yes" } ], "cycle_checks": [ { "claim": "Scheduled or looping main behavior", "command": "", "evidence": "Entry points inspected (jevlike-data, jevlike-train, jevlike-eval, jevlike-predict in pyproject.toml; jevlike/*.py; examples). No cron, scheduler, --once/--now, loop or webhook exists in the product. The only sleep calls are game-pacing loops in examples/doom/play.py and examples/chess/play.py, which are not product cycles. Each CLI runs once when invoked.", "expected": "identify the product trigger", "input": "source inspection", "kind": "cycle", "outcome": "Entry points inspected (jevlike-data, jevlike-train, jevlike-eval, jevlike-predict in pyproject.toml; jevlike/*.py; examples). No cron, scheduler, --once/--now, loop or webhook exists in the product. The only sleep calls are game-pacing loops in examples/doom/play.py and examples/chess/play.py, which are not product cycles. Each CLI runs once when invoked.", "source": "repository entry points", "verdict": "yes" } ], "details": { "argv": [ "", "runs/synthetic.pt", "--context", "Choose the exact badge amber badger. Badge: amber badger.", "--option", "azure crane", "--option", "amber badger", "--option", "gold heron" ], "kind": "cli", "needs_browser": false, "needs_credentials": false, "returncode": 0, "scope": "product operation", "stderr": "", "stdout": "[\n {\n \"option\": \"azure crane\",\n \"probability\": 1.0349932466624523e-08\n },\n {\n \"option\": \"amber badger\",\n \"probability\": 1.0\n },\n {\n \"option\": \"gold heron\",\n \"probability\": 4.9812562963325036e-08\n[excerpt truncated]", "working": true }, "evidence_strength": "readme_checked", "example_checks": [ { "claim": "Quickstart training on synthetic data writes a checkpoint", "command": " data/synthetic/train.jsonl --validation data/synthetic/validation.jsonl --output runs/replay-train.pt", "deferred": false, "evidence": "h\": 4, \"train_nll\": 0.042619120180606845, \"validation_nll\": 0.0345140016078949, \"device\": \"cpu\"}\n{\"epoch\": 5, \"train_nll\": 0.019992097705602646, \"validation_nll\": 0.028758792504668236, \"device\": \"cpu\"}\n{\"epoch\": 6, \"train_nll\": 0.021846030103042723, \"validation_nll\": 0.019645809847861528, \"device\": \"cpu\"}\n{\"epoch\": 7, \"train_nll\": 0.009171441148966552, \"validation_nll\": 0.007989753340370952, \"device\": \"cpu\"}\n{\"epoch\": 8, \"train_nll\": 0.003976261492818594, \"validation_nll\": 0.008053538376116194, \"device\": \"cpu\"}\n{\"checkpoint\": \"runs/replay-train.pt\", \"best_validation_nll\": 0.007989753340370952}", "expected": "checkpoint written with best_validation_nll", "input": "data/synthetic/train.jsonl with validation; fresh output runs/replay-train.pt", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "README.md Quickstart", "verdict": "yes" }, { "claim": "Doom example plays the released joint checkpoint and writes a trace", "command": " examples/doom/play.py examples/checkpoints/joint-imitation.pt --episodes 10 --game-seconds 35.3 --device cpu --capture-resolution 640x480 --output runs/doom.mp4 --trace runs/doom-trace.json", "deferred": false, "evidence": "{\"episodes\": 10, \"mean_reward\": -94.78393249511718, \"mean_kills\": 0.4, \"mean_wall_game_ratio\": 1.0215097106955706, \"online_inference_ms\": 2.748335302327975, \"cpu_ms\": 1.0682277600000134, \"video\": \"runs/doom.mp4\", \"trace\": \"runs/doom-trace.json\"}", "expected": "JSON summary with episodes 10 and video/trace paths", "input": "examples/checkpoints/joint-imitation.pt, 10 episodes, 640x480 capture", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "README.md Demo", "verdict": "yes" }, { "claim": "Doom example smoke tests", "command": " -q examples/doom/test_smoke.py", "deferred": false, "evidence": ".. [100%]\n2 passed in 5.91s", "expected": "2 passed", "input": "examples/doom/test_smoke.py", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "AGENTS.md Setup test commands", "verdict": "yes" }, { "claim": "Chess example smoke test", "command": " test_smoke.py", "deferred": false, "evidence": "ok", "expected": "ok", "input": "examples/chess/test_smoke.py", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "AGENTS.md Setup test commands", "verdict": "yes" }, { "claim": "Repository offline test suite", "command": " -q", "deferred": false, "evidence": ". [100%]\n1 passed in 11.19s", "expected": "1 passed", "input": "tests/test_smoke.py", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "AGENTS.md Setup test commands", "verdict": "yes" }, { "claim": "Film rendering from the Doom trace", "command": "", "deferred": true, "evidence": "not run: needs Node/npm (absent in guest) and a separate npm/npx setup step that fetches an extra tool; that step was not performed", "expected": "documented command completes successfully", "input": "", "kind": "example", "outcome": "not run: needs Node/npm (absent in guest) and a separate npm/npx setup step that fetches an extra tool; that step was not performed", "source": "README.md Demo", "verdict": "deferred" }, { "claim": "Wikispeedia next-click data example", "command": "", "deferred": true, "evidence": "not run: scripts/get_wikispeedia.sh downloads external SNAP archives; not needed for the core quickstart", "expected": "documented command completes successfully", "input": "", "kind": "example", "outcome": "not run: scripts/get_wikispeedia.sh downloads external SNAP archives; not needed for the core quickstart", "source": "README.md Wikispeedia example", "verdict": "deferred" }, { "claim": "Frozen Hugging Face encoder training", "command": "", "deferred": true, "evidence": "not run: requires transformers install and downloading Qwen/Qwen2.5-0.5B from Hugging Face (large model download)", "expected": "documented command completes successfully", "input": "", "kind": "example", "outcome": "not run: requires transformers install and downloading Qwen/Qwen2.5-0.5B from Hugging Face (large model download)", "source": "README.md Use a frozen pretrained encoder", "verdict": "deferred" } ], "status": "PASS" }, "deviations": [ "uv was not present; installed via pip (apt python3-pip first, since pip was missing). Setup only.", "Quickstart eval: top1 0.9975 for the model vs 0.2275 for the shuffled-context control. Predict gave 'amber badger' probability 1.0.", "Doom play run: 10 episodes gave mean_kills 0.4 and mean_reward -94.78; the README cites 0.60 kills and -97.50 reward for the same checkpoint. Results vary between runs, so this is not treated as a mismatch.", "README Demo film step not run: it needs Node/npm, which is absent, and a separate tool download that this guest does not permit. This is a prerequisite gap for that step only; the core operation is unaffected.", "Wikispeedia and frozen Hugging Face encoder examples deferred: they download external data or a large model; not needed for the core operation.", "The README's frozen-encoder path was not exercised, so its Hugging Face download behaviour is unverified." ], "fault_owner": null, "main_blocker": null, "operation": "Train, evaluate and score a one-pass option scorer with the jevlike CLIs from the README Quickstart (jevlike-data, jevlike-train, jevlike-eval, jevlike-predict), plus the documented Doom and chess example checks.", "operation_scope": "Linux Ubuntu 24.04, Python 3.12.3, CPU only, pinned commit 94f5fd1b0b11d52bbdfdf4e0ee6aa96b568f8452. Installed uv 0.13.5 via pip (user install) and python3-pip/python3-venv via apt. Game extras installed via uv. No Node/npm, no external datasets or Hugging Face models.", "outcome": "working", "plain_failure": null, "steps": [ "git init; git remote add origin https://github.com/vinnylarouge/jevlike; git fetch --depth 1 origin 94f5fd1b0b11d52bbdfdf4e0ee6aa96b568f8452; git reset --mixed FETCH_HEAD", "sudo apt-get install -y python3-pip python3-venv", "python3 -m pip install --user --break-system-packages uv", "uv venv", "source .venv/bin/activate && uv pip install -e '.[dev]'", "jevlike-data synthetic --output data/synthetic", "jevlike-train data/synthetic/train.jsonl --validation data/synthetic/validation.jsonl --output runs/synthetic.pt", "jevlike-eval runs/synthetic.pt data/synthetic/test.jsonl", "jevlike-predict runs/synthetic.pt --context \"Choose the exact badge amber badger. Badge: amber badger.\" --option \"azure crane\" --option \"amber badger\" --option \"gold heron\"", "uv pip install -e '.[dev,games]' (AGENTS.md setup)", "pytest -q; PYTHONPATH=examples/doom pytest -q examples/doom/test_smoke.py; cd examples/chess && python test_smoke.py", "python examples/doom/play.py examples/checkpoints/joint-imitation.pt --episodes 10 --game-seconds 35.3 --device cpu --capture-resolution 640x480 --output runs/doom.mp4 --trace runs/doom-trace.json" ], "stuck_step": null, "suggested_fix": "None needed for the core operation. To run the README film step, provide Node/npm and allow its separate tool download." }