{ "run_id": "TheoLeeCJ--SemIf-OpenJev-gpt-6-luna-1792853761015836982", "checker": { "claims": [ { "claim": "Score owned examples and produce typed option scores.", "command": "/tmp/semif-venv/bin/semif-score --mode direct --backend llamacpp --gguf /tmp/semif-model/Qwen_Qwen3.5-4B-Q4_K_M.gguf --model Qwen/Qwen3.5-4B --revision 851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a --input examples/decisions.jsonl --output /tmp/semif-coldpass-claim.jsonl-checker-4076f1046d5c44899785f2aa5a138461", "evidence": "\"id\": \"support-1\", \"option_ids\": [\"yes\", \"no\", \"insufficient\"], \"probabilities\": [0.9994890719602687, 0.00013567599535170604, 0.00037525204437965866], \"option_logits\": [28.294504165649414, 19.389774322509766, 20.407102584838867], \"answer_token_ids\": [32, 33, 34], \"input_tokens\": 142, \"allowed_token_mass\": 0.9998756442730353, \"full_vocab_argmax_id\": 32, \"prompt_sha256\": \"7ac35785358f0c656eeb741ca0a51f03e0133753a71e2879a18ad0ab56d0c024\", \"prompt_version\": \"direct-options-v1\", \"model\": {\"source\": \"Qwen/Qwen3.5-4B\", \"revision\": \"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\", \"backend\": \"llamacpp\", \"dt", "expected": "support-1 result includes typed scores", "input": "examples/decisions.jsonl", "kind": "claim", "outcome": "Real input produced the expected result: \"id\": \"support-1\"", "returncode": 0, "source": "Each result contains typed option scores, timing, the exact model revision, and a prompt hash.", "source_checked": true, "source_file": "README.md", "verdict": "yes" } ], "cycle_checks": [ { "claim": "Scheduled or looping main behavior", "command": "", "evidence": "README and src entry points describe command-driven scoring; no scheduler or recurring cycle was found.", "expected": "identify the product trigger", "input": "source inspection", "kind": "cycle", "outcome": "README and src entry points describe command-driven scoring; no scheduler or recurring cycle was found.", "source": "repository entry points", "verdict": "yes" } ], "details": { "argv": [ "/tmp/semif-venv/bin/semif-score", "--mode", "direct", "--backend", "llamacpp", "--gguf", "/tmp/semif-model/Qwen_Qwen3.5-4B-Q4_K_M.gguf", "--model", "Qwen/Qwen3.5-4B", "--revision", "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "--input", "examples/decisions.jsonl", "--output", "/tmp/semif-coldpass-probe.jsonl-checker-3da80f9bff8a4a78b285b7df31f1e776" ], "expected_file": "/tmp/semif-coldpass-probe.jsonl-checker-3da80f9bff8a4a78b285b7df31f1e776", "file_checked": true, "file_excerpt": "\"id\": \"support-1\", \"option_ids\": [\"yes\", \"no\", \"insufficient\"], \"probabilities\": [0.9994890719602687, 0.00013567599535170604, 0.00037525204437965866], \"option_logits\": [28.294504165649414, 19.389774322509766, 20.407102584838867], \"answer_token_ids\": [32, 33, 34], \"input_tokens\": 142, \"allowed_token_mass\": 0.9998756442730353, \"full_vocab_argmax_id\": 32, \"prompt_sha256\": \"7ac35785358f0c656eeb741ca0a51f03e0133753a71e2879a18ad0ab56d0c024\", \"prompt_version\": \"direct-options-v1\", \"model\": {\"source\": \"Qwen/Qwen3.5-4B\", \"revision\": \"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\", \"backend\": \"llamacpp\", \"dtype\": \"gguf-quantized\", \"gguf\": {\"file\": \"Qwen_Qwen3.5-4B-Q4_K_M.gguf\", \"bytes\": 3013027808, \"sha256\": \"13c16f426047e2de38cd075bdade4a7bcbc8c774384876f677740cda65f8a983\"}, \"vocab_size\": 248320, \"threads\": 2, \"n_gpu_layers\": 0, \"max_prompt_tokens\": 4096, \"context_tokens\": 4352, \"decode_chunk\": 512, \"llama_cpp_python_version\": \"0.3.35\", \"transformers_version\": \"5.17.0\", \"serving_config\": \"llamacpp-direct-v1\"}, \"readout\": \"quantized last-position logits restricted to declared answer slots; no generated tokens\", \"probability_status\": \"conditional option score over quantized weights; uncalibrated as decision confidence\", \"forward_seconds\": 5.932923989000074, \"total_seconds\": 5.935320555999851}\n{\"id\": \"route-1\", \"option_ids\": [\"account_access\", \"billing\", \"sales\"], \"probabilities\": [0.9999628974085876, 2.6670441244641933e-05, 1.0432150167767386e-05], \"option_logits\": [28.321033477783203, 17.78911590576172, 16.850452423095703], \"answer_token_ids\": [32, 33, 34], \"input_tokens\": 130, \"allowed_token_mass\": 0.999870232621057, \"full_vocab_argmax_id\": 32, \"prompt_sha256\": \"016267542103ab91422c425d2234ec7e1b9aef77f9d3e481b96db2de7cca470a\", \"prompt_version\": \"direct-options-v1\", \"model\": {\"source\": \"Qwen/Qwen3.5-4B\", \"revision\": \"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\", \"backend\": \"llamacpp\", \"dtype\": \"gguf-quantized\", \"gguf\": {\"file\": \"Qwen_Qwen3.5-4B-Q4_K_M.gguf\", \"bytes\": 3013027808, \"sha256\": \"13c16f426047e2de38cd075bdade4a7bcbc8c774384876f677740cda65f8a983\"}, \"vocab_size\": 248320, \"threads\": 2, \"n_gpu_layers\": 0, \"max_prompt_tokens\": 4096, \"context_tokens\": 4352, \"decode_chunk\": 512, \"llama_cpp_python_version\": \"0.3.35\", \"transformers_version\": \"5.17.0\", \"serving_config\": \"llamacpp-direct-v1\"}, \"readout\": \"quantized last-position logits restricted to declared answer slots; no generated tokens\", \"probability_status\": \"conditional option score over quantized weights; uncalibrated as decision confidence\", \"forward_seconds\": 4.423341889000085, \"total_seconds\": 4.426572518000057}\n{\"id\": \"policy-1\", \"option_ids\": [\"required\", \"not_required\", \"insufficient\"], \"probabilities\": [0.06092024663437009, 0.935643051777468, 0.0034367015881619045], \"option_logits\": [23.35482406616211, 26.08649253845215, 20.47977066040039], \"answer_token_ids\": [32, 33, 34], \"input_tokens\": 154, \"allowed_token_mass\": 0.9996553685559919, \"full_vocab_argmax_id\": 33, \"prompt_sha256\": \"6884fc4a9117b63369dd79e3042b92f92678f1e08dba2600e93d1da6754b1787\", \"prompt_version\": \"direct-options-v1\", \"model\": {\"source\": \"Qwen/Qwen3.5-4B\", \"revision\": \"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\", \"backend\": \"llamacpp\", \"dtype\": \"gguf-quantized\", \"gguf\": {\"file\": \"Qwen_Qwen3.5-4B-Q4_K_M.gguf\", \"bytes\": 3013027808, \"sha256\": \"13c16f426047e2de38cd075bdade4a7bcbc8c774384876f677740cda65f8a983\"}, \"vocab_size\": 248320, \"threads\": 2, \"n_gpu_layers\": 0, \"max_prompt_tokens\": 4096, \"context_tokens\": 4352, \"decode_chunk\": 512, \"llama_cpp_python_version\": \"0.3.35\", \"transformers_version\": \"5.17.0\", \"serving_config\": \"llamacpp-direct-v1\"}, \"readout\": \"quantized last-position logits restricted to declared answer slots; no generated tokens\", \"probability_status\": \"conditional option score over quantized weights; uncalibrated as decision confidence\", \"forward_seconds\": 4.969243106000022, \"total_seconds\": 4.970980280999811}\n", "kind": "cli", "needs_browser": false, "needs_credentials": false, "returncode": 0, "scope": "product operation", "stderr": " dev = CPU\nllama_kv_cache: layer 32: filtered\nllama_kv_cache: CPU KV buffer size = 136.00 MiB\nllama_kv_cache: size = 136.00 MiB ( 4352 cells, 8 layers, 1/1 seqs), K (f16): 68.00 MiB, V (f16): 68.00 MiB\nllama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256\nllama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256\nllama_memory_recurrent, layer 0: dev = CPU\nllama_memory_recurrent, layer 1: dev = CPU\nllama_memory_recurrent, layer 2: dev = CPU\nllama_memory_recurrent: layer 3: skipped\nllama_memory_recurrent, layer 4: dev = CPU\nllama_memory_recurrent, layer 5: dev = CPU\n[excerpt truncated]", "stdout": "", "working": false }, "evidence_strength": "partly_checked", "example_checks": [ { "claim": "Score owned decisions with CPU GGUF backend", "command": "/tmp/semif-venv/bin/semif-score --mode direct --backend llamacpp --gguf /tmp/semif-model/Qwen_Qwen3.5-4B-Q4_K_M.gguf --model Qwen/Qwen3.5-4B --revision 851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a --input examples/decisions.jsonl --output /tmp/semif-coldpass-example.jsonl", "deferred": false, "evidence": "usage: semif-score [-h] --mode {direct,serial,shared,reranker}\n [--backend {torch,mlx,llamacpp}] [--mlx-bits {4,8}]\n [--mlx-cache-limit-mib MLX_CACHE_LIMIT_MIB] [--gguf GGUF]\n [--llama-threads LLAMA_THREADS] --model MODEL --revision\n REVISION --input INPUT --output OUTPUT\n [--max-tokens MAX_TOKENS] [--device {auto,cuda,mps,cpu}]\n [--dtype {bfloat16,float16,float32}]\nsemif-score: error: Output must be new and max-tokens must be positive", "expected": "one scored JSONL record per decision", "input": "examples/decisions.jsonl", "kind": "example", "outcome": "command exited with code 2", "returncode": 2, "source": "README.md / Quick start / CPU only", "verdict": "no" } ], "status": "PASS" }, "diagnosis": null }