{ "run_id": "TheoLeeCJ--SemIf-OpenJev-claude-haiku-5-5-1792791092174823671", "date": "2026-10-23T21:53:03.593019+00:00", "repo": "TheoLeeCJ/SemIf-OpenJev", "kind": "repo", "source": "https://github.com/TheoLeeCJ/SemIf-OpenJev", "commit": "23cf1f39fc9534fe81437200959b6dfc7106e45a", "model": "claude-haiku-5-5", "model_version": "claude-haiku-5-5", "test_version": "readme-bdc1b9b3c0d2673aef87e076d14084a8123ddcf09d520c1ee2e364a37c3f4bb9", "pass": true, "outcome": "pass", "time_to_working_s": 1202.8321583860088, "total_s": 1291.4181656837463, "steps": [ "cd && pwd && ls -la && id -u && cat /etc/os-release | head -3 && python3 --version && (git status 2>&1 | head -3)", "cd && cat README.md 2>/dev/null | head -300", "cd && cat AGENTS.md; cat pyproject.toml; cat requirements.txt; ls .coldpass examples docs tests src; nvidia-smi 2>&1 | head -3; nproc; free -g | head -2; df -h /tmp | tail -1; env | grep -i -E 'proxy|hf_|cuda' ; sudo -n true && echo SUDO_OK", "cd && cat docs/REPRODUCE.md | head -150; echo ----; cat examples/* | head -40; echo ----; ls -la /inputs 2>&1; ls 2>&1", "cd /tmp; for u in https://pypi.org/simple/torch/ https://huggingface.co/api/models/Qwen/Qwen3.5-4B https://huggingface.co/bartowski/Qwen_Qwen3.5-4B-GGUF https://github.com; do echo -n \"$u -> \"; curl -s -o /dev/null -m 20 -w \"%{http_code}\\n\" \"$u\" || echo fail; done; python3 -m venv --help >/dev/null 2>&1 && echo VENV_OK || echo NO_VENV; pip --version; ls /usr/bin/python3*", "sed -n 1,200p | head -120", "cd && cat .gitignore && python3 -m venv .venv && . .venv/bin/activate && python --version && pip install -q --upgrade pip 2>&1 | tail -3; pip install -e '.[test]' > 2>&1; echo EXIT=$?; tail -15 ", "cd && rm -rf .venv && (sudo apt-get update -qq && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3.12-venv python3-pip) > 2>&1; echo APT=$?; tail -3 ; python3 -m venv .venv && . .venv/bin/activate && pip --version && pip install -q --upgrade pip 2>&1 | tail -2; echo PIPUP=$?", "cd && . .venv/bin/activate && timeout 3000 pip install -e '.[test]' > 2>&1; echo EXIT=$?; tail -8 ", "cd && . .venv/bin/activate && python -c \"import torch;print(torch.__version__, torch.cuda.is_available())\"; ls src/semif_phase1; grep -n -E \"add_argument|device|cuda\" src/semif_phase1/cli.py | head -60; cat manifests/models.json 2>/dev/null | head -30", "cd && . .venv/bin/activate && timeout 900 pytest -q 2>&1 | tail -15; echo EXIT=${PIPESTATUS[0]}", "cd && sed -n 34,110p src/semif_phase1/cli.py; grep -n -E \"def load_causal_model|cuda|device\" src/semif_phase1/core.py | head -30", "cd && . .venv/bin/activate && mkdir -p && CUDA_VISIBLE_DEVICES=0 timeout 600 semif-score --mode direct --model Qwen/Qwen3.5-4B --revision 851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a --input examples/decisions.jsonl --output 2>&1 | tail -8; echo EXIT=${PIPESTATUS[0]}; ls ; sed -n 99,110p src/semif_phase1/core.py", "cd && sed -n 1,60p src/semif_phase1/llamacpp_backend.py; grep -n -i \"gguf\" manifests/models.json docs/*.md | head -20; which cmake g++ gcc make; cat /proc/cpuinfo | grep -m1 flags | tr ' ' '\\n' | grep -E \"avx|fma\" | tr '\\n' ' '", "cd && (sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq cmake build-essential > 2>&1; echo APT=$? >> ) ; tail -1 ; cmake --version | head -1; . .venv/bin/activate && nohup sh -c \"pip install -e '.[test,llamacpp]' > 2>&1; echo PIPEXIT=\\$? >> \" >/dev/null 2>&1 &\nmkdir -p && cd && nohup sh -c \". ; export HF_HOME=; hf download bartowski/Qwen_Qwen3.5-4B-GGUF Qwen_Qwen3.5-4B-Q4_K_M.gguf --local-dir > 2>&1; echo DLEXIT=\\$? >> \" >/dev/null 2>&1 &\nsleep 5; echo started; cat 2>/dev/null | tail -2", "sleep 240; tail -2 ; echo ---; tail -c 400 ; ls -la ; du -sh 2>/dev/null", "for i in $(seq 1 28); do if grep -q PIPEXIT ; then break; fi; sleep 30; done; tail -4 | cut -c1-600", "cd && . .venv/bin/activate && ls ; time (timeout 1500 semif-score --mode direct --backend llamacpp --gguf --model Qwen/Qwen3.5-4B --revision 851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a --input examples/decisions.jsonl --output > 2>&1); echo EXIT=$?; tail -15 | cut -c1-500; ls -la ", "cd && wc -l results-direct.jsonl && python3 -c \"\nimport json\nfor l in open('results-direct.jsonl'):\n r=json.loads(l)\n print({k:(v if not isinstance(v,(dict,list)) or len(json.dumps(v))<200 else '...') for k,v in r.items()})\n\" | cut -c1-900", "cd && head -1 | python3 -c \"import json,sys; r=json.load(sys.stdin); print(r['model'])\"; grep -n -i -E \"revision\" | head -c 300; echo; grep -rn -i -E \"schedul|cron|webhook|\\-\\-once|\\-\\-now|while True|loop\" src/ README.md docs/ 2>/dev/null | head; echo \"grep-exit=$?\"; grep -n -o '\"prompt_sha256\": \"[0-9a-f]*\"' | head -1; echo; (cd results/raw && sha256sum -c SHA256SUMS 2>&1 | tail -2)", "cd && grep -F -c \"Each result contains typed option scores, timing, the exact model revision, and a prompt hash.\" README.md; grep -F -c \"**CPU only:** the llama.cpp backend scores the same prompts from a local GGUF\" README.md; rm -f ; mkdir -p .coldpass && cd .coldpass && cat > probe.json <<'EOF'\n{\"kind\":\"cli\",\"argv\":[\"\",\"--mode\",\"direct\",\"--backend\",\"llamacpp\",\"--gguf\",\"\",\"--model\",\"Qwen/Qwen3.5-4B\",\"--revision\",\"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\",\"--input\",\"examples/decisions.jsonl\",\"--output\",\"\"],\"cwd\":\"\",\"expected_file\":\"\",\"expected_text\":\"\\\"prompt_sha256\\\": \\\"7ac35785358f0c656eeb741ca0a51f03e0133753a71e2879a18ad0ab56d0c024\\\"\"}\nEOF\ncat > claims.json <<'EOF'\n[{\"kind\":\"claim\",\"claim\":\"Each result contains typed option scores, timing, the exact model revision, and a prompt hash.\",\"source\":\"Each result contains typed option scores, timing, the exact model revision, and a prompt hash.\",\"input\":\"examples/decisions.jsonl scored with the llama.cpp CPU backend\",\"expected\":\"each row has a prompt hash and the pinned model revision\",\"probe_kind\":\"cli\",\"argv\":[\"\",\"--mode\",\"direct\",\"--backend\",\"llamacpp\",\"--gguf\",\"\",\"--model\",\"Qwen/Qwen3.5-4B\",\"--revision\",\"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\",\"--input\",\"examples/decisions.jsonl\",\"--output\",\"\"],\"cwd\":\"\",\"expected_file\":\"\",\"expected_text\":\"\\\"prompt_sha256\\\": \\\"7ac35785358f0c656eeb741ca0a51f03e0133753a71e2879a18ad0ab56d0c024\\\"\"},\n{\"kind\":\"claim\",\"claim\":\"CPU only: the llama.cpp backend scores the same prompts from a local GGUF checkpoint with no CUDA device.\",\"source\":\"**CPU only:** the llama.cpp backend scores the same prompts from a local GGUF\",\"input\":\"examples/decisions.jsonl on a guest with no CUDA device, using the local Q4_K_M GGUF\",\"expected\":\"output rows record backend llamacpp\",\"probe_kind\":\"cli\",\"argv\":[\"\",\"--mode\",\"direct\",\"--backend\",\"llamacpp\",\"--gguf\",\"\",\"--model\",\"Qwen/Qwen3.5-4B\",\"--revision\",\"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\",\"--input\",\"examples/decisions.jsonl\",\"--output\",\"\"],\"cwd\":\"\",\"expected_file\":\"\",\"expected_text\":\"\\\"backend\\\": \\\"llamacpp\\\"\"}]\nEOF\ncat > examples.json <<'EOF'\n[{\"kind\":\"example\",\"claim\":\"Score the owned decision examples on CPU with the llama.cpp GGUF backend\",\"source\":\"README.md Quick start, CPU only llama.cpp path\",\"input\":\"examples/decisions.jsonl and the local Q4_K_M GGUF\",\"expected\":\"three scored rows with a prompt hash\",\"argv\":[\"\",\"--mode\",\"direct\",\"--backend\",\"llamacpp\",\"--gguf\",\"\",\"--model\",\"Qwen/Qwen3.5-4B\",\"--revision\",\"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\",\"--input\",\"examples/decisions.jsonl\",\"--output\",\"\"],\"cwd\":\"\",\"env\":{},\"expected_file\":\"\",\"expected_text\":\"\\\"id\\\": \\\"policy-1\\\"\"},\n{\"kind\":\"example\",\"claim\":\"Offline test suite\",\"source\":\"docs/REPRODUCE.md, pytest -q\",\"input\":\"repository tests\",\"expected\":\"all offline tests pass\",\"argv\":[\"\",\"-q\"],\"cwd\":\"\",\"env\":{},\"expected_text\":\"67 passed\"}]\nEOF\ncat > cycles.json <<'EOF'\n{\"detected\":false,\"reason\":\"Entry points inspected: pyproject.toml defines only the one-shot console script semif-score (semif_phase1.cli:main). It scores the input file once and exits. A grep of src/, README.md and docs/ found no cron, scheduler, webhook, --once/--now, or polling loop. The only loop-like matches are GPU looped-mode code in shared.py and the README phrase 'decoding loop', which is a negation. Nothing runs repeatedly, so no product cycle exists to trigger.\",\"checks\":[]}\nEOF\nfor f in probe claims examples cycles; do python3 -m json.tool $f.json >/dev/null && echo \"$f OK\"; done; ls -la ", "cd && cat > report.json <<'EOF'\n{\"outcome\":\"working\",\"fault_owner\":null,\"operation\":\"Score README decision examples (examples/decisions.jsonl) with the semif-score CLI, using the documented CPU-only llama.cpp backend and the pinned Qwen3.5-4B revision\",\"operation_scope\":\"CPU-only path on Ubuntu 24.04, 2 vCPU, 3 GB RAM, no NVIDIA GPU. Uses the Q4_K_M GGUF (3.01 GB) from bartowski/Qwen_Qwen3.5-4B-GGUF. The primary CUDA BF16 path in the README was not exercised because this guest has no GPU.\",\"steps\":[\"sudo apt-get install python3.12-venv python3-pip (python3 -m venv .venv failed with ensurepip missing)\",\"python3 -m venv .venv && pip install --upgrade pip\",\"pip install -e '.[test]'\",\"pytest -q (67 passed, 3 skipped)\",\"CUDA_VISIBLE_DEVICES=0 semif-score --mode direct --model Qwen/Qwen3.5-4B --revision 851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a --input examples/decisions.jsonl --output (failed: MPS is unavailable; no CUDA device)\",\"sudo apt-get install cmake build-essential; pip install -e '.[test,llamacpp]' (llama-cpp-python 0.3.35 built from source)\",\"hf download bartowski/Qwen_Qwen3.5-4B-GGUF Qwen_Qwen3.5-4B-Q4_K_M.gguf (3.01 GB)\",\"semif-score --mode direct --backend llamacpp --gguf --model Qwen/Qwen3.5-4B --revision 851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a --input examples/decisions.jsonl --output (exit 0, 3 rows scored in about 45 s)\"],\"deviations\":[\"Ubuntu venv lacked ensurepip; installed python3.12-venv via apt (setup, not a product fault).\",\"GPU path from README Quick start failed with 'MPS is unavailable' because the guest has no CUDA device. Substituted the README's documented CPU-only llama.cpp path.\",\"Downloaded the GGUF with the hf CLI from the README-cited repo; the GGUF was not in the workspace.\"],\"stuck_step\":null,\"main_blocker\":null,\"plain_failure\":null,\"suggested_fix\":null}\nEOF\npython3 -m json.tool report.json >/dev/null && echo REPORT_OK; cd && = python3 -I --validate --replay 2>&1 | tail -40; echo CHECK_EXIT=${PIPESTATUS[0]}" ], "stuck_step": null, "error_excerpt": null, "what_it_did": "Score README decision examples (examples/decisions.jsonl) with the semif-score CLI, using the documented CPU-only llama.cpp backend and the pinned Qwen3.5-4B revision", "suggested_fix": null, "tokens_in": 590253, "tokens_out": 17704, "cost_usd": 0.02556268, "usage_complete": true, "checks": [ { "check": "operation", "status": "PASS", "details": { "working": true, "argv": [ "", "--mode", "direct", "--backend", "llamacpp", "--gguf", "", "--model", "Qwen/Qwen3.5-4B", "--revision", "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "--input", "examples/decisions.jsonl", "--output", "" ], "kind": "cli", "returncode": 0, "stdout": "", "stderr": " dev = CPU\nllama_kv_cache: layer 32: filtered\nllama_kv_cache: CPU KV buffer size = 136.00 MiB\nllama_kv_cache: size = 136.00 MiB ( 4352 cells, 8 layers, 1/1 seqs), K (f16): 68.00 MiB, V (f16): 68.00 MiB\nllama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256\nllama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256\nllama_memory_recurrent, layer 0: dev = CPU\nllama_memory_recurrent, layer 1: dev = CPU\nllama_memory_recurrent, layer 2: dev = CPU\nllama_memory_recurrent: layer 3: skipped\nllama_memory_recurrent, layer 4: dev = CPU\nllama_memory_recurrent, layer 5: dev = CPU\n[excerpt truncated]", "expected_file": "", "file_checked": true, "file_excerpt": "\"prompt_sha256\": \"7ac35785358f0c656eeb741ca0a51f03e0133753a71e2879a18ad0ab56d0c024\", \"prompt_version\": \"direct-options-v1\", \"model\": {\"source\": \"Qwen/Qwen3.5-4B\", \"revision\": \"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\", \"backend\": \"llamacpp\", \"dtype\": \"gguf-quantized\", \"gguf\": {\"file\": \"Qwen_Qwen3.5-4B-Q4_K_M.gguf\", \"bytes\": 3013027808, \"sha256\": \"13c16f426047e2de38cd075bdade4a7bcbc8c774384876f677740cda65f8a983\"}, \"vocab_size\": 248320, \"threads\": 2, \"n_gpu_layers\": 0, \"max_prompt_tokens\": 4096, \"context_tokens\": 4352, \"decode_chunk\": 512, \"llama_cpp_python_version\": \"0.3.35\", \"transformers_version\": \"5.17.0\", \"serving_config\": \"llamacpp-direct-v1\"}, \"readout\": \"quantized last-position logits restricted to declared answer slots; no generated tokens\", \"probability_status\": \"conditional option score over quantized weights; uncalibrated as decision confidence\", \"forward_seconds\": 4.701214177999873, \"total_seconds\": 4.703709764999985}\n{\"id\": \"route-1\", \"option_ids\": [\"account_access\", \"billing\", \"sales\"], \"probabilities\": [0.9999629642207696, 2.62066227817402e-05, 1.0829156448702313e-05], \"option_logits\": [28.472476959228516, 17.923015594482422, 17.03924560546875], \"answer_token_ids\": [32, 33, 34], \"input_tokens\": 130, \"allowed_token_mass\": 0.9998695699189398, \"full_vocab_argmax_id\": 32, \"prompt_sha256\": \"016267542103ab91422c425d2234ec7e1b9aef77f9d3e481b96db2de7cca470a\", \"prompt_version\": \"direct-options-v1\", \"model\": {\"source\": \"Qwen/Qwen3.5-4B\", \"revision\": \"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\", \"backend\": \"llamacpp\", \"dtype\": \"gguf-quantized\", \"gguf\": {\"file\": \"Qwen_Qwen3.5-4B-Q4_K_M.gguf\", \"bytes\": 3013027808, \"sha256\": \"13c16f426047e2de38cd075bdade4a7bcbc8c774384876f677740cda65f8a983\"}, \"vocab_size\": 248320, \"threads\": 2, \"n_gpu_layers\": 0, \"max_prompt_tokens\": 4096, \"context_tokens\": 4352, \"decode_chunk\": 512, \"llama_cpp_python_version\": \"0.3.35\", \"transformers_version\": \"5.17.0\", \"serving_config\": \"llamacpp-direct-v1\"}, \"readout\": \"quantized last-position logits restricted to declared answer slots; no generated tokens\", \"probability_status\": \"conditional option score over quantized weights; uncalibrated as decision confidence\", \"forward_seconds\": 3.589164976999882, \"total_seconds\": 3.591024411000035}\n{\"id\": \"policy-1\", \"option_ids\": [\"required\", \"not_required\", \"insufficient\"], \"probabilities\": [0.08280425302875945, 0.9112643235874939, 0.005931423383746777], \"option_logits\": [23.673660278320312, 26.07201385498047, 21.037445068359375], \"answer_token_ids\": [32, 33, 34], \"input_tokens\": 154, \"allowed_token_mass\": 0.9996493322869658, \"full_vocab_argmax_id\": 33, \"prompt_sha256\": \"6884fc4a9117b63369dd79e3042b92f92678f1e08dba2600e93d1da6754b1787\", \"prompt_version\": \"direct-options-v1\", \"model\": {\"source\": \"Qwen/Qwen3.5-4B\", \"revision\": \"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\", \"backend\": \"llamacpp\", \"dtype\": \"gguf-quantized\", \"gguf\": {\"file\": \"Qwen_Qwen3.5-4B-Q4_K_M.gguf\", \"bytes\": 3013027808, \"sha256\": \"13c16f426047e2de38cd075bdade4a7bcbc8c774384876f677740cda65f8a983\"}, \"vocab_size\": 248320, \"threads\": 2, \"n_gpu_layers\": 0, \"max_prompt_tokens\": 4096, \"context_tokens\": 4352, \"decode_chunk\": 512, \"llama_cpp_python_version\": \"0.3.35\", \"transformers_version\": \"5.17.0\", \"serving_config\": \"llamacpp-direct-v1\"}, \"readout\": \"quantized last-position logits restricted to declared answer slots; no generated tokens\", \"probability_status\": \"conditional option score over quantized weights; uncalibrated as decision confidence\", \"forward_seconds\": 4.108065505000013, \"total_seconds\": 4.109861006999836}\n", "needs_credentials": false, "needs_browser": false, "scope": "product operation" } } ], "safety": { "version": "z2x-1", "rules_version": "z2x-1", "summary": "Review signals: 1 connection review, 30 hardcoded secret, 6 known vulnerability. 3 static matches retained as context.", "expected": "Reviewed as expected for a test run: resolver address lookups that send no data.", "findings": [ { "kind": "connection-review", "count": 1, "note": "Repository processes attempted these non-loopback connections; proxies can hide the final hostname.", "location_count": 47, "process_count": 11 }, { "kind": "hardcoded-secret", "count": 18, "note": "Gitleaks found a possible hardcoded secret; value withheld.", "location_count": 7 }, { "kind": "known-vulnerability", "count": 6, "note": "OSV reports a known vulnerability in this dependency version.", "location_count": 3 } ], "context": [ { "kind": "encoded-blob", "count": 2, "note": "Source contains encoded blob; review its purpose.", "location_count": 1, "path": "webgpu-demo/vendor/wllama/index.js" }, { "kind": "encoded-execution", "count": 1, "note": "Source contains encoded execution; review its purpose.", "location_count": 1, "path": "webgpu-demo/vendor/wllama/index.js" } ], "limitations": [ "A short run cannot establish that code is safe.", "Connections show IP addresses; proxies can hide the final hostname.", "Tracing covers dispatched commands and their children, not the whole OS; guest root can interfere with tracing." ], "static": { "dependency_note": "OSV scanned supported lockfiles; unpinned dependencies and unsupported formats are outside this check." }, "evidence_url": "/safety/?run=TheoLeeCJ--SemIf-OpenJev-claude-haiku-5-5-1792791092174823671", "evidence_file": "/data/safety/TheoLeeCJ--SemIf-OpenJev-claude-haiku-5-5-1792791092174823671.json.gz" }, "agent_execution": "native_agent", "agent_runtime": "claude-code", "agent_runtime_version": "2.1.300 (Claude Code)", "tool_transport": "mcp-assigned-vm", "platform": "linux", "reasoning": "low", "service_tier": null, "resources": { "cpus": 2, "memory_mib": 4096, "disk_mib": 32768 }, "native_tool_calls": 22, "native_bundle": null, "architecture": null, "backend": "hyperv", "platform_details": { "backend": "hyperv", "os": "linux" }, "platform_conflicts": null, "sandbox_source": null, "source_pin": { "commit": "23cf1f39fc9534fe81437200959b6dfc7106e45a", "url": "https://github.com/TheoLeeCJ/SemIf-OpenJev" }, "purpose": "initial", "profile": { "model": "claude-haiku-5-5", "reasoning": "low" }, "fault": { "owner": null, "status": "resolved", "reason": null }, "fix_id": null, "verifies_run_id": null, "hosting": null, "operation_scope": "product operation", "tool_calls": 22, "guest_setup": null, "runtime": null, "runtime_version": null, "runtime_revision": null, "source_revision": null, "evidence_version": null, "failure_origin": null, "evidence_strength": "readme_checked", "works": "yes", "install_s": null, "first_use_s": null, "mcp": "none", "needs": [], "needs_which": null, "agent_interface": [ "cli" ], "license": "MIT", "repo_url": "https://github.com/TheoLeeCJ/SemIf-OpenJev", "cost_to_try": "free", "hardware": "GPU", "what_failed": null, "local_or_hosted": "both", "cycle_checks": [ { "kind": "cycle", "claim": "Scheduled or looping main behavior", "source": "repository entry points", "input": "source inspection", "expected": "identify the product trigger", "verdict": "yes", "outcome": "Entry points inspected: pyproject.toml defines only the one-shot console script semif-score (semif_phase1.cli:main). It scores the input file once and exits. A grep of src/, README.md and docs/ found no cron, scheduler, webhook, --once/--now, or polling loop. The only loop-like matches are GPU looped-mode code in shared.py and the README phrase 'decoding loop', which is a negation. Nothing runs repeatedly, so no product cycle exists to trigger.", "evidence": "Entry points inspected: pyproject.toml defines only the one-shot console script semif-score (semif_phase1.cli:main). It scores the input file once and exits. A grep of src/, README.md and docs/ found no cron, scheduler, webhook, --once/--now, or polling loop. The only loop-like matches are GPU looped-mode code in shared.py and the README phrase 'decoding loop', which is a negation. Nothing runs repeatedly, so no product cycle exists to trigger.", "command": "" } ], "claims": [ { "kind": "claim", "claim": "Each result contains typed option scores, timing, the exact model revision, and a prompt hash.", "source": "Each result contains typed option scores, timing, the exact model revision, and a prompt hash.", "input": "examples/decisions.jsonl scored with the llama.cpp CPU backend", "expected": "each row has a prompt hash and the pinned model revision", "command": " --mode direct --backend llamacpp --gguf --model Qwen/Qwen3.5-4B --revision 851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a --input examples/decisions.jsonl --output ", "verdict": "yes", "outcome": "Real input produced the expected result: \"prompt_sha256\": \"7ac35785358f0c656eeb741ca0a51f03e0133753a71e2879a18ad0ab56d0c024\"", "evidence": "\"prompt_sha256\": \"7ac35785358f0c656eeb741ca0a51f03e0133753a71e2879a18ad0ab56d0c024\", \"prompt_version\": \"direct-options-v1\", \"model\": {\"source\": \"Qwen/Qwen3.5-4B\", \"revision\": \"851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a\", \"backend\": \"llamacpp\", \"dtype\": \"gguf-quantized\", \"gguf\": {\"file\": \"Qwen_Qwen3.5-4B-Q4_K_M.gguf\", \"bytes\": 3013027808, \"sha256\": \"13c16f426047e2de38cd075bdade4a7bcbc8c774384876f677740cda65f8a983\"}, \"vocab_size\": 248320, \"threads\": 2, \"n_gpu_layers\": 0, \"max_prompt_tokens\": 4096, \"context_tokens\": 4352, \"decode_chunk\": 512, \"llama_cpp_python_version\": \"0.3.35\", \"transformers_vers", "returncode": 0, "source_checked": true, "source_file": "README.md" }, { "kind": "claim", "claim": "CPU only: the llama.cpp backend scores the same prompts from a local GGUF checkpoint with no CUDA device.", "source": "**CPU only:** the llama.cpp backend scores the same prompts from a local GGUF", "input": "examples/decisions.jsonl on a guest with no CUDA device, using the local Q4_K_M GGUF", "expected": "output rows record backend llamacpp", "command": " --mode direct --backend llamacpp --gguf --model Qwen/Qwen3.5-4B --revision 851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a --input examples/decisions.jsonl --output ", "verdict": "yes", "outcome": "Real input produced the expected result: \"backend\": \"llamacpp\"", "evidence": "\"backend\": \"llamacpp\", \"dtype\": \"gguf-quantized\", \"gguf\": {\"file\": \"Qwen_Qwen3.5-4B-Q4_K_M.gguf\", \"bytes\": 3013027808, \"sha256\": \"13c16f426047e2de38cd075bdade4a7bcbc8c774384876f677740cda65f8a983\"}, \"vocab_size\": 248320, \"threads\": 2, \"n_gpu_layers\": 0, \"max_prompt_tokens\": 4096, \"context_tokens\": 4352, \"decode_chunk\": 512, \"llama_cpp_python_version\": \"0.3.35\", \"transformers_version\": \"5.17.0\", \"serving_config\": \"llamacpp-direct-v1\"}, \"readout\": \"quantized last-position logits restricted to declared answer slots; no generated tokens\", \"probability_status\": \"conditional option score over quantiz", "returncode": 0, "source_checked": true, "source_file": "README.md" } ], "claims_note": null, "example_checks": [ { "kind": "example", "claim": "Score the owned decision examples on CPU with the llama.cpp GGUF backend", "source": "README.md Quick start, CPU only llama.cpp path", "input": "examples/decisions.jsonl and the local Q4_K_M GGUF", "command": " --mode direct --backend llamacpp --gguf --model Qwen/Qwen3.5-4B --revision 851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a --input examples/decisions.jsonl --output ", "evidence": "56\": \"13c16f426047e2de38cd075bdade4a7bcbc8c774384876f677740cda65f8a983\"}, \"vocab_size\": 248320, \"threads\": 2, \"n_gpu_layers\": 0, \"max_prompt_tokens\": 4096, \"context_tokens\": 4352, \"decode_chunk\": 512, \"llama_cpp_python_version\": \"0.3.35\", \"transformers_version\": \"5.17.0\", \"serving_config\": \"llamacpp-direct-v1\"}, \"readout\": \"quantized last-position logits restricted to declared answer slots; no generated tokens\", \"probability_status\": \"conditional option score over quantized weights; uncalibrated as decision confidence\", \"forward_seconds\": 4.295587840999815, \"total_seconds\": 4.298743025000022}\n", "deferred": false, "expected": "three scored rows with a prompt hash", "returncode": 0, "verdict": "yes", "outcome": "command completed successfully" }, { "kind": "example", "claim": "Offline test suite", "source": "docs/REPRODUCE.md, pytest -q", "input": "repository tests", "command": " -q", "evidence": ".....................................................s..........s.... [100%]\n67 passed, 3 skipped in 5.84s", "deferred": false, "expected": "all offline tests pass", "returncode": 0, "verdict": "yes", "outcome": "command completed successfully" } ], "does_what_it_says": "yes", "maintenance": { "last_commit_at": "2026-09-23T18:47:59Z", "commits_90d": 25, "merged_prs_90d": 5, "open_issues": 23, "open_prs": 25, "reply_share": 0.02, "reply_threads": 47, "median_reply_hours": 12.9, "last_release_at": null, "archived": false, "verdict": "maintained", "note": "actively maintained: last commit 30 days ago; 25 commits and 5 merged PRs in 90 days; 23 open issues, 25 open PRs; 1 of 47 recent threads got a maintainer reply, median first reply 12.9h; no releases" }, "maintenance_verdict": "maintained", "maintenance_note": "actively maintained: last commit 30 days ago; 25 commits and 5 merged PRs in 90 days; 23 open issues, 25 open PRs; 1 of 47 recent threads got a maintainer reply, median first reply 12.9h; no releases", "maintained": "active", "maintained_note": "last commit 30 days ago; 12 open issues, 2 answered or closed", "last_commit_at": "2026-09-23T18:48:22Z", "last_release_at": null, "open_issues": 12, "answered_issues": 2, "platforms_claimed": [ "mac" ], "topic": "other", "agent_interfaces": [ { "interface": "cli", "evidence": "independent checker", "observed": true, "operation_checked": true } ], "agent_interfaces_note": "README metadata is a discovery hint, not pinned documentation proof. Observed interfaces were exercised in this run. An empty list means unknown.", "mcp_context": null, "tested_at": "2026-10-23T21:53:03.593019+00:00", "tested_commit": "23cf1f39fc9534fe81437200959b6dfc7106e45a", "tested_with": { "agent": "claude-code", "agent_version": "2.1.300 (Claude Code)", "model": "claude-haiku-5-5", "model_route": "claude-haiku-5-5", "reasoning": "low", "os": "linux", "backend": "hyperv", "transport": "mcp-assigned-vm" }, "setup_identity": { "repo": "TheoLeeCJ/SemIf-OpenJev", "source": "https://github.com/TheoLeeCJ/SemIf-OpenJev", "commit": "23cf1f39fc9534fe81437200959b6dfc7106e45a", "source_revision": null, "os": "linux", "architecture": null, "backend": "hyperv", "model": "claude-haiku-5-5", "model_version": "claude-haiku-5-5", "agent_execution": "native_agent", "agent_runtime": "claude-code", "agent_runtime_version": "2.1.300 (Claude Code)", "tool_transport": "mcp-assigned-vm", "runtime": null, "runtime_version": null, "runtime_revision": null, "sandbox_source": null, "guest_setup": null, "source_pin": { "commit": "23cf1f39fc9534fe81437200959b6dfc7106e45a", "url": "https://github.com/TheoLeeCJ/SemIf-OpenJev" }, "platform_details": { "os": "linux", "backend": "hyperv" }, "platform_conflicts": null, "reasoning": "low", "service_tier": null, "resources": { "cpus": 2, "memory_mib": 4096, "disk_mib": 32768 }, "operation": "Score README decision examples (examples/decisions.jsonl) with the semif-score CLI, using the documented CPU-only llama.cpp backend and the pinned Qwen3.5-4B revision", "operation_scope": "product operation", "hosting": null, "kind": "repo", "test_version": "readme-bdc1b9b3c0d2673aef87e076d14084a8123ddcf09d520c1ee2e364a37c3f4bb9", "evidence_version": null }, "other_model": null, "other_run_id": null, "models_agree": null, "furthest_model": null, "transcript_url": null, "report_url": "/reports/TheoLeeCJ--SemIf-OpenJev-claude-haiku-5-5-1792791092174823671.json" }