asu: recover from upstream version metadata failures

asu/main.py

@@ -69,6 +69,12 @@ def store(path: str):
 
 @app.get("/", response_class=HTMLResponse)
 def index(request: Request):
+    if not app.versions:
+        reload_versions(app)
+        if not app.versions:
+            raise HTTPException(
+                status_code=503, detail="Version information unavailable"
+            )
     return templates.TemplateResponse(
         request=request,
         name="overview.html",
@@ -140,6 +146,8 @@ def json_v1_profile(path: str, target: str, profile: str):
 
 def generate_latest():
     reload_versions(app)  # Do a reload in case .versions.json has updated.
+    if not app.versions:
+        raise HTTPException(status_code=503, detail="Version information unavailable")
     return app.latest
 
 
@@ -151,6 +159,8 @@ def json_v1_latest():
 
 def generate_branches():
     # Caller is responsible for calling reload_versions() beforehand.
+    if not app.versions:
+        raise HTTPException(status_code=503, detail="Version information unavailable")
     branches = dict(**settings.branches)
 
     for branch in branches:

asu/routers/api.py

@@ -109,6 +109,8 @@ def validate_request(
 
     if build_request.version not in app.versions:
         reload_versions(app)
+        if not app.versions:
+            return {"detail": "Version information unavailable", "status": 503}, 503
         if build_request.version not in app.versions:
             return validation_failure(f"Unsupported version: {build_request.version}")
 

asu/util.py

@@ -592,11 +592,10 @@ def reload_versions(app: FastAPI) -> bool:
     """Set the values of both `app.versions` and `app.latest` using the
     upstream `.versions.json` file.
 
-    We check for updates to the versions by examining the response's
-    `from_cache` attribute.  This is safe because `reload_versions` is the
-    only function that downloads that file, so no race conditions can exist.
+    Keep the previous data if downloading or parsing fails, so later
+    requests can retry without disrupting an already running server.
 
-    Returns `True` if data has changed, `False` when cache was used.
+    Returns `True` on success, `False` on failure.
     """
 
     def in_supported_branch(version: str) -> bool:
@@ -614,27 +613,47 @@ def add_versions(version_list: list, *versions: str) -> None:
             if in_supported_branch(version):
                 version_list.append(version)
 
-    response = client_get(settings.upstream_url + "/.versions.json")
+    try:
+        response = client_get(settings.upstream_url + "/.versions.json")
+    except httpx.RequestError as exc:
+        log.warning(f".versions.json: failed to download: {exc}")
+        return False
+
     if response.status_code != 200:
         log.info(f".versions.json: failed to download {response.status_code}")
         return False
 
-    versions_upstream = response.json()
-    upcoming_version = versions_upstream["upcoming_version"]
+    try:
+        versions_upstream = response.json()
+        upcoming_version = versions_upstream["upcoming_version"]
+        stable_version = versions_upstream["stable_version"]
+        oldstable_version = versions_upstream["oldstable_version"]
+        versions_list = versions_upstream["versions_list"]
+        if not all(
+            version is None or isinstance(version, str)
+            for version in (upcoming_version, stable_version, oldstable_version)
+        ) or not (
+            isinstance(versions_list, list)
+            and all(isinstance(version, str) for version in versions_list)
+        ):
+            raise ValueError("Invalid version fields")
+    except (ValueError, KeyError, TypeError) as exc:
+        log.warning(f".versions.json: invalid version data: {exc}")
+        return False
 
-    app.latest = []
+    latest = []
     add_versions(
-        app.latest,
+        latest,
         upcoming_version,
-        versions_upstream["stable_version"],
-        versions_upstream["oldstable_version"],
+        stable_version,
+        oldstable_version,
     )
 
-    app.versions = []
+    versions = []
     add_versions(
-        app.versions,
+        versions,
         upcoming_version,
-        *versions_upstream["versions_list"],
+        *versions_list,
         "SNAPSHOT",
         *[
             f"{branch_name}-SNAPSHOT"
@@ -644,7 +663,9 @@ def add_versions(version_list: list, *versions: str) -> None:
     )
 
     # Create a key that puts -rcN between -SNAPSHOT and releases.
-    app.versions.sort(reverse=True, key=lambda v: v.replace(".0-rc", "-rc"))
+    versions.sort(reverse=True, key=lambda v: v.replace(".0-rc", "-rc"))
+    app.latest = latest
+    app.versions = versions
 
     return True
 

tests/test_versions.py

@@ -0,0 +1,144 @@
+import runpy
+from unittest.mock import Mock
+
+import httpx
+import pytest
+from fastapi.testclient import TestClient
+
+from asu.config import settings
+from asu.routers.api import validate_request
+from asu.build_request import BuildRequest
+from asu.util import reload_versions
+
+
+@pytest.fixture
+def version_data():
+    return {
+        "upcoming_version": "",
+        "stable_version": "1.2.3",
+        "oldstable_version": "",
+        "versions_list": ["1.2.3"],
+    }
+
+
+@pytest.fixture
+def load_app(monkeypatch, tmp_path):
+    monkeypatch.setattr(settings, "public_path", tmp_path)
+    monkeypatch.setattr(
+        settings,
+        "branches",
+        {"1.2": {"path": "releases/{version}", "enabled": True}},
+    )
+    get = Mock()
+    monkeypatch.setattr("asu.util.client_get", get)
+
+    def load(result):
+        if isinstance(result, Exception):
+            get.side_effect = result
+        else:
+            get.return_value = result
+        namespace = runpy.run_path("asu/main.py")
+        return namespace["app"], get
+
+    return load
+
+
+@pytest.mark.parametrize(
+    "failure",
+    [
+        httpx.ConnectError("Connection failed"),
+        httpx.ReadTimeout("Timed out"),
+        httpx.Response(503),
+        httpx.Response(200, text="not json"),
+        httpx.Response(200, json={}),
+        httpx.Response(200, json=None),
+        httpx.Response(200, json=[]),
+    ],
+)
+def test_startup_version_failure(load_app, version_data, failure):
+    app, get = load_app(failure)
+    assert app.versions == []
+    assert app.latest == []
+
+    with TestClient(app) as client:
+        assert client.get("/json/v1/latest.json").status_code == 503
+        get.side_effect = None
+        get.return_value = httpx.Response(200, json=version_data)
+        response = client.get("/json/v1/latest.json")
+        assert response.status_code == 200
+        assert response.json() == {"latest": ["1.2.3"]}
+        assert app.versions == ["1.2.3", "1.2-SNAPSHOT"]
+
+
+@pytest.mark.parametrize(
+    "path", ["/", "/json/v1/branches.json", "/json/v1/overview.json"]
+)
+def test_version_endpoints_retry(load_app, version_data, path):
+    app, get = load_app(httpx.ConnectError("Connection failed"))
+    with TestClient(app) as client:
+        assert client.get(path).status_code == 503
+
+        def recovered_get(url):
+            return httpx.Response(
+                200, json=version_data if url.endswith("/.versions.json") else {}
+            )
+
+        get.side_effect = recovered_get
+        assert client.get(path).status_code == 200
+
+
+@pytest.mark.parametrize(
+    "failure",
+    [
+        httpx.ReadTimeout("Timed out"),
+        httpx.Response(502),
+        httpx.Response(200, text="not json"),
+        httpx.Response(200, json={"upcoming_version": "1.2.4"}),
+    ],
+)
+def test_version_refresh_keeps_previous_data(load_app, version_data, failure):
+    app, get = load_app(httpx.Response(200, json=version_data))
+    versions, latest = list(app.versions), list(app.latest)
+    if isinstance(failure, Exception):
+        get.side_effect = failure
+    else:
+        get.return_value = failure
+
+    assert reload_versions(app) is False
+    assert app.versions == versions
+    assert app.latest == latest
+    with TestClient(app) as client:
+        response = client.get("/json/v1/latest.json")
+        assert response.status_code == 200
+        assert response.json() == {"latest": latest}
+
+
+@pytest.mark.parametrize(
+    "field,value",
+    [
+        ("upcoming_version", 123),
+        ("stable_version", {}),
+        ("oldstable_version", []),
+        ("versions_list", "1.2.3"),
+        ("versions_list", None),
+        ("versions_list", ["1.2.4", 123]),
+    ],
+)
+def test_invalid_version_fields_keep_previous_data(
+    load_app, version_data, field, value
+):
+    app, get = load_app(httpx.Response(200, json=version_data))
+    versions, latest = list(app.versions), list(app.latest)
+    get.return_value = httpx.Response(200, json={**version_data, field: value})
+    assert reload_versions(app) is False
+    assert app.versions == versions
+    assert app.latest == latest
+
+
+def test_build_without_version_information(load_app):
+    app, get = load_app(httpx.ConnectError("Connection failed"))
+    request = BuildRequest(version="1.2.3", target="test/target", profile="test")
+    content, status = validate_request(app, request)
+    assert status == 503
+    assert content["detail"] == "Version information unavailable"
+    assert get.call_count == 2