{
  "tokens_out": 4158,
  "outcome": "pass",
  "suggested_fix": null,
  "cycle_checks": null,
  "fault": {
    "owner": null,
    "status": "resolved",
    "reason": null,
    "evidence": []
  },
  "runtime_revision": null,
  "agent_execution": "native_agent",
  "time_to_working_s": null,
  "usage_complete": true,
  "runtime": null,
  "architecture": null,
  "local_or_hosted": null,
  "agent_interfaces_note": null,
  "platform_details": null,
  "platforms_claimed": null,
  "maintenance": null,
  "evidence_strength": null,
  "purpose": "initial",
  "topic": null,
  "needs_which": null,
  "tested_with": null,
  "native_bundle": null,
  "agent_runtime": "claude-code",
  "maintenance_note": null,
  "runtime_version": null,
  "does_what_it_says": null,
  "commit": "e690f4cf9f35d2bb58f163888bc74cfa0f157038",
  "failure_origin": null,
  "stuck_step": null,
  "repo_url": null,
  "needs": null,
  "checks": [
    {
      "check": "operation",
      "status": "PASS",
      "details": {
        "working": true,
        "argv": [],
        "kind": "cli",
        "returncode": 0,
        "stdout": "\u2713 Evaluation completed\n\nEVALUATION RESULTS\n\nSummary:\n\u256d\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u256e\n\u2502 Metric               \u2502 Value           \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Total Data Points    \u2502 3               \u2502\n\u2502 Failed Data Points   \u2502 0               \u2502\n\u2502 Total Jobs           \u2502 6               \u2502\n\u2502 Failed Jobs          \u2502 0               \u2502\n[excerpt truncated]",
        "stderr": "",
        "needs_credentials": false,
        "needs_browser": false,
        "scope": "product operation"
      }
    }
  ],
  "open_issues": null,
  "agent_runtime_version": "2.1.300 (Claude Code)",
  "verifies_run_id": null,
  "total_s": 108.30377578735352,
  "license": null,
  "tool_calls": 9,
  "model_version": "claude-haiku-5-5",
  "example_checks": [],
  "mcp": null,
  "other_model": null,
  "first_use_s": null,
  "native_tool_calls": 9,
  "last_commit_at": null,
  "source": "https://github.com/orq-ai/evaluatorq",
  "models_agree": null,
  "maintenance_verdict": null,
  "cost_usd": 0.00748411,
  "safety": null,
  "answered_issues": null,
  "what_failed": null,
  "steps": [
    "cd <host-path> && git log -1 --format=%H 2>/dev/null; ls -la; cat README.md | head -200",
    "cd <host-path> && ls -la .github/workflows 2>/dev/null; cat package.json 2>/dev/null | head -60; which node npm python3 pip3; node -v; npm -v",
    "cd <host-path> && sed -n 200,400p README.md | grep -n -E \"^#|uv run|eq |bash\" | head -60; grep -n -E \"uv run eq|eq (--help|version)|eq \" README.md docs/*.md 2>/dev/null | head -20; ls src/evaluatorq; grep -n -A3 \"\\[project.scripts\\]\" pyproject.toml",
    "cd <host-path> && (command -v uv || (curl -LsSf https://astral.sh/uv/install.sh | sh)) 2>&1 | tail -3; export PATH=$HOME/.local/bin:$PATH; export NODE_USE_ENV_PROXY=1; uv --version && uv sync --all-extras 2>&1 | tail -15; ls .venv/bin | grep -E \"^eq$|^python$\"",
    "cd <host-path> && export PATH=$HOME/.local/bin:$PATH NODE_USE_ENV_PROXY=1; V=$(grep -m1 -oE '^## \\[?[0-9]+\\.[0-9]+\\.[0-9]+' CHANGELOG.md | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'); echo \"version=$V\"; export SETUPTOOLS_SCM_PRETEND_VERSION=$V; uv sync --all-extras 2>&1 | tail -8; ls .venv/bin | grep -E \"^eq$\"",
    "cd <host-path> && grep -n -m3 -E \"^## |version\" CHANGELOG.md | head; grep -n -A4 \"hatch.version\" pyproject.toml",
    "cd <host-path> && export PATH=$HOME/.local/bin:$PATH NODE_USE_ENV_PROXY=1 SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0.dev0 SETUPTOOLS_SCM_PRETEND_VERSION_FOR_EVALUATORQ=0.0.0.dev0; uv sync --all-extras 2>&1 | tail -8; ls .venv/bin | grep -E \"^eq$\"",
    "cd <host-path> && unset ORQ_API_KEY; ls examples/lib/basics/support_agent_eval.py && timeout 300 <host-path> examples/lib/basics/support_agent_eval.py 2>&1 | tail -40; echo \"rc=${PIPESTATUS[0]}\"",
    "cd <host-path> && mkdir -p .coldpass && cat > .coldpass/probe.json <<'EOF'\n{\"kind\":\"cli\",\"argv\":[\"<host-path>\",\"<host-path>\"],\"cwd\":\"<host-path>\",\"expected_text\":\"Evaluation completed successfully\"}\nEOF\ncat > .coldpass/report.json <<'EOF'\n{\"outcome\":\"working\",\"operation\":\"Ran the README quick-start example (examples/lib/basics/support_agent_eval.py) with the venv Python: two agents scored by string_contains_evaluator over 3 datapoints; output shows agent-v1 0.33, agent-v2 1.00, 0 failed jobs, 'Evaluation completed successfully'. Run locally with ORQ_API_KEY unset.\",\"operation_scope\":\"Core evaluation only. Red teaming, simulation, dashboard, and coding-agent Docker mode were not exercised; they need an ORQ_API_KEY, Docker, or an LLM provider key. Full pytest/ruff/ty suites intentionally not run.\",\"steps\":[\"Installed uv 0.13.1 via https://astral.sh/uv/install.sh (README 'Install' uses uv)\",\"Set SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0.dev0 and SETUPTOOLS_SCM_PRETEND_VERSION_FOR_EVALUATORQ=0.0.0.dev0, then ran 'uv sync --all-extras' in <host-path> (creates .venv with eq CLI)\",\"Ran examples/lib/basics/support_agent_eval.py (README Quick start) with .venv/bin/python: exit 0\"],\"deviations\":[\"'uv sync --all-extras' (README Install / uv add) failed first with hatch-vcs 'setuptools-scm was unable to detect version' because the checkout has no .git directory. Worked around by setting SETUPTOOLS_SCM_PRETEND_VERSION env vars (no source edits). The version 0.0.0.dev0 is a placeholder.\"],\"stuck_step\":null,\"main_blocker\":null,\"suggested_fix\":null}\nEOF\n<env-var>=<host-path> python3 -I <host-path> --validate"
  ],
  "platform_conflicts": null,
  "claims_note": null,
  "maintained_note": null,
  "repo": "orq-ai/evaluatorq",
  "service_tier": null,
  "maintained": null,
  "source_pin": null,
  "operation_scope": "product operation",
  "reasoning": "low",
  "claims": [],
  "platform": "linux",
  "cost_to_try": null,
  "backend": null,
  "tool_transport": "mcp-assigned-vm",
  "hardware": null,
  "agent_interface": null,
  "profile": {
    "model": "claude-haiku-5-5",
    "reasoning": "low"
  },
  "date": "2026-10-18T17:34:38.309091+00:00",
  "kind": "repo",
  "resources": {
    "cpus": 2,
    "memory_mib": 4096,
    "disk_mib": 32768
  },
  "error_excerpt": null,
  "pass": true,
  "evidence_version": null,
  "source_revision": null,
  "run_id": "orq-ai--evaluatorq-claude-haiku-5-5-1792344770005300413",
  "last_release_at": null,
  "furthest_model": null,
  "model": "claude-haiku-5-5",
  "what_it_did": "Ran the README quick-start example (examples/lib/basics/support_agent_eval.py) with the venv Python: two agents scored by string_contains_evaluator over 3 datapoints; output shows agent-v1 0.33, agent-v2 1.00, 0 failed jobs, 'Evaluation completed successfully'. Run locally with ORQ_API_KEY unset.",
  "sandbox_source": null,
  "other_run_id": null,
  "hosting": null,
  "tested_commit": null,
  "guest_setup": null,
  "mcp_context": null,
  "agent_interfaces": [],
  "setup_identity": null,
  "works": null,
  "install_s": null,
  "fix_id": null,
  "tokens_in": 157670,
  "tested_at": null,
  "test_version": "readme-c5e0ccb431a80a9d60d2e220a9fed646c77ddd4e70bc799cda23957b77e37337",
  "recovery": {
    "source": "legacy-oct16-18",
    "original_run_id": "orq-ai--evaluatorq-claude-haiku-5-5-1792344770005300413",
    "historical": true,
    "ledger_sha256": "c5e6a833c182c1e2aecf69d0bd0020aec63c65d98f3e933e4b32232340312736"
  },
  "transcript_url": "/transcripts/orq-ai--evaluatorq-claude-haiku-5-5-1792344770005300413.txt",
  "report_url": "/reports/orq-ai--evaluatorq-claude-haiku-5-5-1792344770005300413.json"
}