runtime/lava: add a liveness probe and fail fast on connect

Every device and queue query the scheduler makes against a lab that has
gone away blocks for the full 30s request timeout, once per job and per
platform.  There is no way to ask "is this lab up?" without paying that
cost on an endpoint that also does real work.

Add LAVA.is_alive(), which polls /system/version/.  lava-server answers
it from a constant with no database access (SystemViewSet.version in
lava_rest_app/v02/views.py) and the body is ~20 bytes, against ~16-78kB
and a full device table scan for /devices/, so a caller can poll it
regularly without adding any measurable load to the lab.

Any HTTP answer counts as reachable, including the refusals: labs that
return 401 or 403 to an anonymous or unprivileged request are up, and
treating them as down would stop scheduling against them entirely.
Only a transport failure or a 5xx is reported as unreachable.

Also split the request timeout into a short connect timeout and the
existing response timeout, so an unreachable host fails in seconds
rather than after 30, and give Runtime a default is_alive() so callers
can probe any runtime.

Signed-off-by: Denys Fedoryshchenko <denys.f@collabora.com>

Author: nuclearcat

kernelci/runtime/__init__.py

@@ -287,6 +287,15 @@ def get_job_id(self, job_object):
     def wait(self, job_object):
         """Wait for a job to complete and get the exit status code"""
 
+    def is_alive(self):
+        """Check whether the runtime is reachable
+
+        Return a (alive, detail) tuple where *detail* describes the outcome
+        for logging.  Runtimes that have no cheap way of answering this, or
+        that cannot become unreachable, report themselves as alive.
+        """
+        return True, "liveness check not implemented"
+
 
 def get_runtime(
     config, user=None, token=None, custom_template_dir=None, kcictx=None

kernelci/runtime/lava.py

@@ -346,6 +346,19 @@ class LAVA(Runtime):
     API_VERSION = "v0.2"
     RestAPIServer = namedtuple("RestAPIServer", ["url", "session"])
 
+    # Connecting is capped well below the response timeout so an unreachable
+    # lab fails in seconds instead of tying up the caller for the full
+    # request timeout on every single call.
+    CONNECT_TIMEOUT = 5
+    REQUEST_TIMEOUT = 30
+
+    # Liveness probe.  lava-server answers /system/version/ from a constant
+    # without touching the database (SystemViewSet.version in
+    # lava_rest_app/v02/views.py) and the body is ~20 bytes, so this can be
+    # polled regularly without adding any measurable load to the lab.
+    LIVENESS_PATH = "system/version/"
+    LIVENESS_TIMEOUT = 10
+
     # LAVA supports 'high'/'medium'/'low' (100/50/0), but we define our own
     # values to allow scaling across labs with different priority ranges.
     PRIORITY_HIGHEST = 80
@@ -457,7 +470,9 @@ def wait(self, job_object):
         job_id = int(job_object)
         job_url = urljoin(self._server.url, "/".join(["jobs", str(job_id)]))
         while True:
-            resp = self._server.session.get(job_url, timeout=30)
+            resp = self._server.session.get(
+                job_url, timeout=self._timeout()
+            )
             resp.raise_for_status()
             data = resp.json()
             if data["state"] == "Finished":
@@ -477,8 +492,41 @@ def _connect(self):
         }
         return rest_api
 
+    def _timeout(self, read_timeout=None):
+        """Timeout tuple for requests: fail fast on connect, wait on read"""
+        return (self.CONNECT_TIMEOUT, read_timeout or self.REQUEST_TIMEOUT)
+
+    def is_alive(self):
+        """Check that the LAVA instance is reachable
+
+        Any HTTP answer proves the instance is up, including the ones that
+        refuse the request: 401 and 403 mean the token or the ACL is wrong,
+        not that the lab is down.  Only a transport failure or a server
+        error counts as unreachable.
+        """
+        if self._server.url is None:
+            return True, "no server URL configured"
+        url = urljoin(self._server.url, self.LIVENESS_PATH)
+        try:
+            resp = self._server.session.get(
+                url, timeout=self._timeout(self.LIVENESS_TIMEOUT)
+            )
+        except requests.RequestException as exc:
+            return False, str(exc)
+        if resp.status_code >= 500:
+            return False, f"HTTP {resp.status_code}"
+        if resp.status_code != 200:
+            return True, f"HTTP {resp.status_code} (reachable)"
+        try:
+            version = resp.json().get("version")
+        except ValueError:
+            version = None
+        return True, f"version {version}" if version else "reachable"
+
     def _get_response(self, url, params=None):
-        resp = self._server.session.get(url, params=params, timeout=30)
+        resp = self._server.session.get(
+            url, params=params, timeout=self._timeout()
+        )
         resp.raise_for_status()
         return resp.json()
 
@@ -574,7 +622,7 @@ def _submit(self, job):
             jobs_url,
             json=job_data,
             allow_redirects=False,
-            timeout=30,
+            timeout=self._timeout(),
         )
         if resp.status_code >= 400:
             print(f"Error submitting job: {resp.status_code}, {resp.text}")

tests/test_runtime.py

@@ -12,6 +12,7 @@
 from pathlib import Path
 
 import pytest
+import requests
 import yaml
 from jinja2 import Environment, FileSystemLoader
 from jinja2.exceptions import TemplateRuntimeError
@@ -528,3 +529,88 @@ def test_compute_tuxrun_parameters_missing_branch():
     """Nodes lacking kernel_revision.branch get an empty parameter set."""
     assert compute_tuxrun_parameters("fvp-aemva", {}) == {}
     assert compute_tuxrun_parameters("fvp-aemva", {"data": {}}) == {}
+
+
+class _LivenessSession:
+    """Session recording the liveness request and replaying a canned answer"""
+
+    def __init__(self, response=None, error=None):
+        self.response = response
+        self.error = error
+        self.calls = []
+
+    def get(self, url, params=None, timeout=None):
+        self.calls.append((url, timeout))
+        if self.error:
+            raise self.error
+        return self.response
+
+
+def _liveness_lab(response=None, error=None):
+    config = kernelci.config.load("tests/configs/lava-runtimes.yaml")
+    runtime_config = config["runtimes"]["lab-min-12-max-40-new-runtime"]
+    lab = kernelci.runtime.get_runtime(runtime_config)
+    lab._server = types.SimpleNamespace(
+        url="http://lava/api/v0.2/",
+        session=_LivenessSession(response=response, error=error),
+    )
+    return lab
+
+
+def test_lava_is_alive_reports_the_version():
+    """A 200 from /system/version/ means the lab is up."""
+    lab = _liveness_lab(_FakeResponse({"version": "2026.07"}))
+
+    alive, detail = lab.is_alive()
+
+    assert alive is True
+    assert "2026.07" in detail
+    url, timeout = lab._server.session.calls[0]
+    assert url == "http://lava/api/v0.2/system/version/"
+    # Connect fast, then allow the probe timeout for the answer.
+    assert timeout == (
+        kernelci.runtime.lava.LAVA.CONNECT_TIMEOUT,
+        kernelci.runtime.lava.LAVA.LIVENESS_TIMEOUT,
+    )
+
+
+def test_lava_is_alive_treats_forbidden_as_reachable():
+    """A lab that refuses the request has still answered it."""
+    lab = _liveness_lab(_FakeResponse({}, status_code=403))
+
+    alive, detail = lab.is_alive()
+
+    assert alive is True
+    assert "403" in detail
+
+
+def test_lava_is_alive_reports_server_errors_as_down():
+    """A 5xx means the instance cannot serve requests."""
+    lab = _liveness_lab(_FakeResponse({}, status_code=502))
+
+    alive, detail = lab.is_alive()
+
+    assert alive is False
+    assert "502" in detail
+
+
+def test_lava_is_alive_reports_transport_failures_as_down():
+    """An unreachable host is the case this probe exists for."""
+    lab = _liveness_lab(
+        error=requests.ConnectionError("Network is unreachable")
+    )
+
+    alive, detail = lab.is_alive()
+
+    assert alive is False
+    assert "Network is unreachable" in detail
+
+
+def test_lava_is_alive_without_server_url():
+    """A runtime storing jobs externally has no server to probe."""
+    config = kernelci.config.load("tests/configs/lava-runtimes.yaml")
+    runtime_config = config["runtimes"]["lab-min-12-max-40-new-runtime"]
+    lab = kernelci.runtime.get_runtime(runtime_config)
+    lab._server = types.SimpleNamespace(url=None, session=None)
+
+    assert lab.is_alive() == (True, "no server URL configured")