{
  "checker": {
    "claims": [
      {
        "claim": "The built-in evaluation harness scores retrieval deterministically and offline.",
        "command": "/tmp/memry-venv/bin/memry eval --dataset evals/datasets/synthetic_v1.jsonl -k 5",
        "evidence": "recall@5:     1.000\nMRR:          0.814\nsearch p50:   0.7 ms   p95: 1.0 ms\nllm:          none   embedder: hash:v1-256\n",
        "expected": "recall@5:     1.000",
        "input": "evals/datasets/synthetic_v1.jsonl -k 5",
        "kind": "claim",
        "outcome": "Real input produced the expected result: recall@5:     1.000",
        "returncode": 0,
        "source": "The harness ingests each case through the full write path, then scores retrieval",
        "source_checked": true,
        "source_file": "README.md",
        "verdict": "yes"
      }
    ],
    "cycle_checks": [
      {
        "claim": "Scheduled or looping main behavior",
        "command": "",
        "evidence": "Inspected README quickstart and CLI entrypoints; they document explicit add/search/eval operations and no cron, scheduled loop, webhook, --once, or --now cycle.",
        "expected": "identify the product trigger",
        "input": "source inspection",
        "kind": "cycle",
        "outcome": "Inspected README quickstart and CLI entrypoints; they document explicit add/search/eval operations and no cron, scheduled loop, webhook, --once, or --now cycle.",
        "source": "repository entry points",
        "verdict": "yes"
      }
    ],
    "details": {
      "argv": [
        "/tmp/memry-venv/bin/memry",
        "eval",
        "--dataset",
        "evals/datasets/synthetic_v1.jsonl",
        "-k",
        "5"
      ],
      "kind": "cli",
      "needs_browser": false,
      "needs_credentials": false,
      "returncode": 0,
      "scope": "product operation",
      "stderr": "",
      "stdout": "dataset:      evals/datasets/synthetic_v1.jsonl\ncases:        8  questions: 17\nmemories:     19\nrecall@5:     1.000\nMRR:          0.814\nsearch p50:   0.7 ms   p95: 0.9 ms\nllm:          none   embedder: hash:v1-256",
      "working": true
    },
    "evidence_strength": "readme_checked",
    "example_checks": [
      {
        "claim": "Run the synthetic retrieval evaluation",
        "command": "/tmp/memry-venv/bin/memry eval --dataset evals/datasets/synthetic_v1.jsonl -k 5",
        "deferred": false,
        "evidence": "dataset:      evals/datasets/synthetic_v1.jsonl\ncases:        8  questions: 17\nmemories:     19\nrecall@5:     1.000\nMRR:          0.814\nsearch p50:   0.7 ms   p95: 0.8 ms\nllm:          none   embedder: hash:v1-256",
        "expected": "retrieval metrics",
        "input": "synthetic_v1.jsonl, k=5",
        "kind": "example",
        "outcome": "command completed successfully",
        "returncode": 0,
        "source": "README.md / Evaluation",
        "verdict": "yes"
      }
    ],
    "status": "PASS"
  },
  "deviations": [],
  "fault_owner": null,
  "main_blocker": null,
  "operation": "Run the synthetic retrieval evaluation",
  "operation_scope": "Local offline evaluation using the built-in dataset, SQLite, and hash embeddings; Python 3.12.3 in a venv",
  "outcome": "working",
  "plain_failure": null,
  "steps": [
    "Read README.md and pyproject.toml; confirmed Python >=3.11.",
    "Created /tmp/memry-venv and installed the documented development package with pip install -e '.[dev]'.",
    "Ran memry eval --dataset evals/datasets/synthetic_v1.jsonl -k 5; observed 8 cases, 17 questions, recall@5 1.000, and MRR 0.814.",
    "Replayed probe, README claim, and documented evaluation example with the coldpass checker; validation passed."
  ],
  "stuck_step": null,
  "suggested_fix": null
}