{ "run_id": "TheoLeeCJ--SemIf-OpenJev-claude-haiku-5-5-1792791092174823671", "checker": { "claims": [ { "claim": "Each result contains typed option scores, timing, the exact model revision, and a prompt hash.", "command": " --mode direct --backend llamacpp --gguf --model Qwen/Qwen3.5-4B --revision 851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a --input examples/decisions.jsonl --output ", "evidence": "\"prompt_sha256\": \"7ac35785358f0c656eeb741ca0a51f03e0133753a71e2879a18ad0ab56d0c024\", \"prompt_version\": \"direct-options-v1\", \"model\": {\"source\": \"Qwen/Qwen3.5-4B\", \"revision\": \"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\", \"backend\": \"llamacpp\", \"dtype\": \"gguf-quantized\", \"gguf\": {\"file\": \"Qwen_Qwen3.5-4B-Q4_K_M.gguf\", \"bytes\": 3013027808, \"sha256\": \"13c16f426047e2de38cd075bdade4a7bcbc8c774384876f677740cda65f8a983\"}, \"vocab_size\": 248320, \"threads\": 2, \"n_gpu_layers\": 0, \"max_prompt_tokens\": 4096, \"context_tokens\": 4352, \"decode_chunk\": 512, \"llama_cpp_python_version\": \"0.3.35\", \"transformers_vers", "expected": "each row has a prompt hash and the pinned model revision", "input": "examples/decisions.jsonl scored with the llama.cpp CPU backend", "kind": "claim", "outcome": "Real input produced the expected result: \"prompt_sha256\": \"7ac35785358f0c656eeb741ca0a51f03e0133753a71e2879a18ad0ab56d0c024\"", "returncode": 0, "source": "Each result contains typed option scores, timing, the exact model revision, and a prompt hash.", "source_checked": true, "source_file": "README.md", "verdict": "yes" }, { "claim": "CPU only: the llama.cpp backend scores the same prompts from a local GGUF checkpoint with no CUDA device.", "command": " --mode direct --backend llamacpp --gguf --model Qwen/Qwen3.5-4B --revision 851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a --input examples/decisions.jsonl --output ", "evidence": "\"backend\": \"llamacpp\", \"dtype\": \"gguf-quantized\", \"gguf\": {\"file\": \"Qwen_Qwen3.5-4B-Q4_K_M.gguf\", \"bytes\": 3013027808, \"sha256\": \"13c16f426047e2de38cd075bdade4a7bcbc8c774384876f677740cda65f8a983\"}, \"vocab_size\": 248320, \"threads\": 2, \"n_gpu_layers\": 0, \"max_prompt_tokens\": 4096, \"context_tokens\": 4352, \"decode_chunk\": 512, \"llama_cpp_python_version\": \"0.3.35\", \"transformers_version\": \"5.17.0\", \"serving_config\": \"llamacpp-direct-v1\"}, \"readout\": \"quantized last-position logits restricted to declared answer slots; no generated tokens\", \"probability_status\": \"conditional option score over quantiz", "expected": "output rows record backend llamacpp", "input": "examples/decisions.jsonl on a guest with no CUDA device, using the local Q4_K_M GGUF", "kind": "claim", "outcome": "Real input produced the expected result: \"backend\": \"llamacpp\"", "returncode": 0, "source": "**CPU only:** the llama.cpp backend scores the same prompts from a local GGUF", "source_checked": true, "source_file": "README.md", "verdict": "yes" } ], "cycle_checks": [ { "claim": "Scheduled or looping main behavior", "command": "", "evidence": "Entry points inspected: pyproject.toml defines only the one-shot console script semif-score (semif_phase1.cli:main). It scores the input file once and exits. A grep of src/, README.md and docs/ found no cron, scheduler, webhook, --once/--now, or polling loop. The only loop-like matches are GPU looped-mode code in shared.py and the README phrase 'decoding loop', which is a negation. Nothing runs repeatedly, so no product cycle exists to trigger.", "expected": "identify the product trigger", "input": "source inspection", "kind": "cycle", "outcome": "Entry points inspected: pyproject.toml defines only the one-shot console script semif-score (semif_phase1.cli:main). It scores the input file once and exits. A grep of src/, README.md and docs/ found no cron, scheduler, webhook, --once/--now, or polling loop. The only loop-like matches are GPU looped-mode code in shared.py and the README phrase 'decoding loop', which is a negation. Nothing runs repeatedly, so no product cycle exists to trigger.", "source": "repository entry points", "verdict": "yes" } ], "details": { "argv": [ "", "--mode", "direct", "--backend", "llamacpp", "--gguf", "", "--model", "Qwen/Qwen3.5-4B", "--revision", "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "--input", "examples/decisions.jsonl", "--output", "" ], "expected_file": "", "file_checked": true, "file_excerpt": "\"prompt_sha256\": \"7ac35785358f0c656eeb741ca0a51f03e0133753a71e2879a18ad0ab56d0c024\", \"prompt_version\": \"direct-options-v1\", \"model\": {\"source\": \"Qwen/Qwen3.5-4B\", \"revision\": \"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\", \"backend\": \"llamacpp\", \"dtype\": \"gguf-quantized\", \"gguf\": {\"file\": \"Qwen_Qwen3.5-4B-Q4_K_M.gguf\", \"bytes\": 3013027808, \"sha256\": \"13c16f426047e2de38cd075bdade4a7bcbc8c774384876f677740cda65f8a983\"}, \"vocab_size\": 248320, \"threads\": 2, \"n_gpu_layers\": 0, \"max_prompt_tokens\": 4096, \"context_tokens\": 4352, \"decode_chunk\": 512, \"llama_cpp_python_version\": \"0.3.35\", \"transformers_version\": \"5.17.0\", \"serving_config\": \"llamacpp-direct-v1\"}, \"readout\": \"quantized last-position logits restricted to declared answer slots; no generated tokens\", \"probability_status\": \"conditional option score over quantized weights; uncalibrated as decision confidence\", \"forward_seconds\": 4.701214177999873, \"total_seconds\": 4.703709764999985}\n{\"id\": \"route-1\", \"option_ids\": [\"account_access\", \"billing\", \"sales\"], \"probabilities\": [0.9999629642207696, 2.62066227817402e-05, 1.0829156448702313e-05], \"option_logits\": [28.472476959228516, 17.923015594482422, 17.03924560546875], \"answer_token_ids\": [32, 33, 34], \"input_tokens\": 130, \"allowed_token_mass\": 0.9998695699189398, \"full_vocab_argmax_id\": 32, \"prompt_sha256\": \"016267542103ab91422c425d2234ec7e1b9aef77f9d3e481b96db2de7cca470a\", \"prompt_version\": \"direct-options-v1\", \"model\": {\"source\": \"Qwen/Qwen3.5-4B\", \"revision\": \"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\", \"backend\": \"llamacpp\", \"dtype\": \"gguf-quantized\", \"gguf\": {\"file\": \"Qwen_Qwen3.5-4B-Q4_K_M.gguf\", \"bytes\": 3013027808, \"sha256\": \"13c16f426047e2de38cd075bdade4a7bcbc8c774384876f677740cda65f8a983\"}, \"vocab_size\": 248320, \"threads\": 2, \"n_gpu_layers\": 0, \"max_prompt_tokens\": 4096, \"context_tokens\": 4352, \"decode_chunk\": 512, \"llama_cpp_python_version\": \"0.3.35\", \"transformers_version\": \"5.17.0\", \"serving_config\": \"llamacpp-direct-v1\"}, \"readout\": \"quantized last-position logits restricted to declared answer slots; no generated tokens\", \"probability_status\": \"conditional option score over quantized weights; uncalibrated as decision confidence\", \"forward_seconds\": 3.589164976999882, \"total_seconds\": 3.591024411000035}\n{\"id\": \"policy-1\", \"option_ids\": [\"required\", \"not_required\", \"insufficient\"], \"probabilities\": [0.08280425302875945, 0.9112643235874939, 0.005931423383746777], \"option_logits\": [23.673660278320312, 26.07201385498047, 21.037445068359375], \"answer_token_ids\": [32, 33, 34], \"input_tokens\": 154, \"allowed_token_mass\": 0.9996493322869658, \"full_vocab_argmax_id\": 33, \"prompt_sha256\": \"6884fc4a9117b63369dd79e3042b92f92678f1e08dba2600e93d1da6754b1787\", \"prompt_version\": \"direct-options-v1\", \"model\": {\"source\": \"Qwen/Qwen3.5-4B\", \"revision\": \"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\", \"backend\": \"llamacpp\", \"dtype\": \"gguf-quantized\", \"gguf\": {\"file\": \"Qwen_Qwen3.5-4B-Q4_K_M.gguf\", \"bytes\": 3013027808, \"sha256\": \"13c16f426047e2de38cd075bdade4a7bcbc8c774384876f677740cda65f8a983\"}, \"vocab_size\": 248320, \"threads\": 2, \"n_gpu_layers\": 0, \"max_prompt_tokens\": 4096, \"context_tokens\": 4352, \"decode_chunk\": 512, \"llama_cpp_python_version\": \"0.3.35\", \"transformers_version\": \"5.17.0\", \"serving_config\": \"llamacpp-direct-v1\"}, \"readout\": \"quantized last-position logits restricted to declared answer slots; no generated tokens\", \"probability_status\": \"conditional option score over quantized weights; uncalibrated as decision confidence\", \"forward_seconds\": 4.108065505000013, \"total_seconds\": 4.109861006999836}\n", "kind": "cli", "needs_browser": false, "needs_credentials": false, "returncode": 0, "scope": "product operation", "stderr": " dev = CPU\nllama_kv_cache: layer 32: filtered\nllama_kv_cache: CPU KV buffer size = 136.00 MiB\nllama_kv_cache: size = 136.00 MiB ( 4352 cells, 8 layers, 1/1 seqs), K (f16): 68.00 MiB, V (f16): 68.00 MiB\nllama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256\nllama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256\nllama_memory_recurrent, layer 0: dev = CPU\nllama_memory_recurrent, layer 1: dev = CPU\nllama_memory_recurrent, layer 2: dev = CPU\nllama_memory_recurrent: layer 3: skipped\nllama_memory_recurrent, layer 4: dev = CPU\nllama_memory_recurrent, layer 5: dev = CPU\n[excerpt truncated]", "stdout": "", "working": true }, "evidence_strength": "readme_checked", "example_checks": [ { "claim": "Score the owned decision examples on CPU with the llama.cpp GGUF backend", "command": " --mode direct --backend llamacpp --gguf --model Qwen/Qwen3.5-4B --revision 851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a --input examples/decisions.jsonl --output ", "deferred": false, "evidence": "56\": \"13c16f426047e2de38cd075bdade4a7bcbc8c774384876f677740cda65f8a983\"}, \"vocab_size\": 248320, \"threads\": 2, \"n_gpu_layers\": 0, \"max_prompt_tokens\": 4096, \"context_tokens\": 4352, \"decode_chunk\": 512, \"llama_cpp_python_version\": \"0.3.35\", \"transformers_version\": \"5.17.0\", \"serving_config\": \"llamacpp-direct-v1\"}, \"readout\": \"quantized last-position logits restricted to declared answer slots; no generated tokens\", \"probability_status\": \"conditional option score over quantized weights; uncalibrated as decision confidence\", \"forward_seconds\": 4.295587840999815, \"total_seconds\": 4.298743025000022}\n", "expected": "three scored rows with a prompt hash", "input": "examples/decisions.jsonl and the local Q4_K_M GGUF", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "README.md Quick start, CPU only llama.cpp path", "verdict": "yes" }, { "claim": "Offline test suite", "command": " -q", "deferred": false, "evidence": ".....................................................s..........s.... [100%]\n67 passed, 3 skipped in 5.84s", "expected": "all offline tests pass", "input": "repository tests", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "docs/REPRODUCE.md, pytest -q", "verdict": "yes" } ], "status": "PASS" }, "diagnosis": null }