Holoscan Install Wheel

evals/evals.json

[
  {
    "id": "holoscan-install-wheel-5x-001",
    "question": "Review this request and return the installation decision and a command plan only. Host and publication facts below are test fixtures, not claims about this sandbox. Use the supplied facts and local skill guidance; do not execute commands, contact services, install packages, pull images, build, or run examples. Identify any missing prerequisite that must be checked before execution. Request: Install the published Holoscan 5.x full Python SDK in a venv on Ubuntu 24.04, Python 3.12, and a compatible CUDA 13 dGPU.",
    "expected_skill": "holoscan-install-wheel",
    "expected_script": null,
    "ground_truth": "Produce a plan without executing it. Use an exact published 5.x metapackage version in an isolated environment.",
    "expected_behavior": [
      "Return the decision, prerequisite checks, and proposed commands without executing installations or external lookups; use the supplied host and release facts as fixtures.",
      "Check Python, glibc, driver, CUDA runtime, and release availability.",
      "Install the pinned holoscan metapackage and inspect component versions.",
      "Run pip check and verify the 5.x Python API.",
      "Use the matching first_pipeline source for bounded execution coverage."
    ]
  },
  {
    "id": "holoscan-install-wheel-5x-002",
    "question": "Review this request and return the installation decision and a command plan only. Host and publication facts below are test fixtures, not claims about this sandbox. Use the supplied facts and local skill guidance; do not execute commands, contact services, install packages, pull images, build, or run examples. Identify any missing prerequisite that must be checked before execution. Request: I need only Core and Inference for Holoscan 5.x on a supported arm64 CUDA 13 server.",
    "expected_skill": "holoscan-install-wheel",
    "expected_script": null,
    "ground_truth": "Produce a plan without executing it. Install matching Core and Inference component wheels.",
    "expected_behavior": [
      "Return the decision, prerequisite checks, and proposed commands without executing installations or external lookups; use the supplied host and release facts as fixtures.",
      "Use holoscan-core-cu13 and holoscan-inference-cu13 with the same version.",
      "Check regular CPython 3.12\u20133.14 and glibc 2.39 requirements.",
      "Check TensorRT 10 prerequisites and selected provider imports.",
      "Do not claim wheels provide C++ headers or examples."
    ]
  },
  {
    "id": "holoscan-install-wheel-5x-003",
    "question": "Review this request and return the installation decision and a command plan only. Host and publication facts below are test fixtures, not claims about this sandbox. Use the supplied facts and local skill guidance; do not execute commands, contact services, install packages, pull images, build, or run examples. Identify any missing prerequisite that must be checked before execution. Request: My workstation has Python 3.10, glibc 2.35, and a CUDA 12-only driver. Install Holoscan 5.x via pip.",
    "expected_skill": "holoscan-install-wheel",
    "expected_script": null,
    "ground_truth": "Produce a plan without executing it. Identify platform blockers before attempting package installation.",
    "expected_behavior": [
      "Return the decision, prerequisite checks, and proposed commands without executing installations or external lookups; use the supplied host and release facts as fixtures.",
      "Explain the Python, glibc, and CUDA 13 requirements.",
      "Do not choose holoscan-cu12 or fall back to 4.x.",
      "Offer a compatible host or an assessed alternative instead of forcing installation."
    ]
  },
  {
    "id": "holoscan-install-wheel-5x-004",
    "question": "Review this request and return the installation decision and a command plan only. Host and publication facts below are test fixtures, not claims about this sandbox. Use the supplied facts and local skill guidance; do not execute commands, contact services, install packages, pull images, build, or run examples. Identify any missing prerequisite that must be checked before execution. Request: The holoscan name exists on PyPI but our 5.x release wheels are not published yet.",
    "expected_skill": "holoscan-install-wheel",
    "expected_script": null,
    "ground_truth": "Produce a plan without executing it. A registered name is not proof of an installable 5.x release.",
    "expected_behavior": [
      "Return the decision, prerequisite checks, and proposed commands without executing installations or external lookups; use the supplied host and release facts as fixtures.",
      "Check exact distribution versions.",
      "Do not perform an unpinned install that could resolve to 4.x or a placeholder.",
      "Offer a source build while retaining the 5.x target."
    ]
  },
  {
    "id": "holoscan-install-wheel-5x-005",
    "question": "Install native C++ Holoscan headers with apt.",
    "expected_skill": null,
    "expected_script": null,
    "ground_truth": "Route to Debian rather than Python wheels.",
    "expected_behavior": [
      "Do not invoke the wheel installation skill for native C++ files."
    ]
  },
  {
    "id": "holoscan-install-wheel-5x-006",
    "question": "Review this request and return the installation decision and a command plan only. Host and publication facts below are test fixtures, not claims about this sandbox. Use the supplied facts and local skill guidance; do not execute commands, contact services, install packages, pull images, build, or run examples. Identify any missing prerequisite that must be checked before execution. Request: The requested Holoscan 5.0 pip install exited 1 with a dependency-resolution error. An older SDK is still importable in the venv and pip check returns 0. Can we continue to first_pipeline and call the 5.0 installation successful?",
    "expected_skill": "holoscan-install-wheel",
    "expected_script": null,
    "ground_truth": "Stop after the failed requested-version installation and report its command and error. Successful checks against an older environment do not establish a successful 5.0 install.",
    "expected_behavior": [
      "Use only the supplied fixtures and local skill guidance; return a plan without executing commands or contacting services.",
      "Report the failed installation command and supplied dependency-resolution error.",
      "Do not proceed to first_pipeline or declare the requested 5.0 installation successful.",
      "Do not silently substitute the older SDK or repair unrelated system paths."
    ]
  }
]