Fix stale OCSP snapshots indefinitely

- Parse and validate from snapshot JSON, reject future/missing timestamps - Enforce snapshot age vs in cache and stale fallback (no lifetime extension on repeat S3 reads) - OCSP: , , - Remove unused from signer - Publisher heartbeats: , republish clean snapshots past interval, preserve dirty-only behavior and concurrency - Config/env/docker/docs updates for heartbeat Rolling upgrade: enable publisher heartbeats before deploying the age-enforcing responder so existing snapshots remain within the age window.
2026-10-25 21:30:00 +00:00
parent ec42cafc28
commit d22ba6662c
12 changed files with 299 additions and 43 deletions
@@ -62,6 +62,9 @@ KCA_OCSP_S3_PREFIX=ocsp
 #KCA_OCSP_S3_ENDPOINT=http://localhost:9000
 KCA_OCSP_S3_FORCE_PATH_STYLE=false
 KCA_OCSP_PUBLISH_INTERVAL_SECONDS=1
+# Republish unchanged snapshots as a liveness heartbeat. Keep this comfortably
+# below the responder's s3.stale_grace_seconds (default 60).
+KCA_OCSP_HEARTBEAT_INTERVAL_SECONDS=30

 # --- CORS (optional) -------------------------------------------------------
 #CORS_ALLOW_ORIGINS=
@@ -191,6 +191,7 @@ class Settings:
    ocsp_s3_endpoint: Optional[str]
    ocsp_s3_force_path_style: bool
    ocsp_publish_interval_seconds: int
+    ocsp_heartbeat_interval_seconds: int

    @staticmethod
    def from_env() -> "Settings":
@@ -245,6 +246,9 @@ class Settings:
            ocsp_publish_interval_seconds=max(
                1, int(_env("KCA_OCSP_PUBLISH_INTERVAL_SECONDS", "1"))
            ),
+            ocsp_heartbeat_interval_seconds=max(
+                1, int(_env("KCA_OCSP_HEARTBEAT_INTERVAL_SECONDS", "30"))
+            ),
        )


@@ -10,11 +10,12 @@ from __future__ import annotations
 import asyncio
 import json
 import logging
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from typing import Any, Awaitable, Callable, Dict, Optional
 from uuid import UUID

 from kaya.core import KayaApp, KayaMixin
+from tortoise.expressions import Q

 from .config import Settings, settings
 from .models import Certificate, CertificateRevocation, OcspSnapshot, Vault
@@ -147,14 +148,16 @@ async def mark_all_cas_dirty(vault: Vault) -> None:


 class OcspPublisherMixin(KayaMixin):
-    """Background publisher for dirty OCSP snapshots."""
+    """Background publisher for dirty and due OCSP snapshots."""

    def __init__(
        self,
        interval_seconds: int,
        ensure_context: Callable[[], Awaitable[None]],
+        heartbeat_interval_seconds: int = 30,
    ) -> None:
        self._interval = max(1, interval_seconds)
+        self._heartbeat_interval = max(1, heartbeat_interval_seconds)
        self._ensure_context = ensure_context
        self._task: Optional[asyncio.Task] = None
        self._bootstrapped = False
@@ -212,7 +215,14 @@ class OcspPublisherMixin(KayaMixin):
        # The publisher runs outside an HTTP request task, so it must bind
        # Tortoise's per-loop context before querying models.
        await self._ensure_context()
-        rows = await OcspSnapshot.filter(dirty=True).limit(25)
+        heartbeat_before = datetime.now(timezone.utc) - timedelta(
+            seconds=self._heartbeat_interval
+        )
+        rows = await OcspSnapshot.filter(
+            Q(dirty=True)
+            | Q(published_at__isnull=True)
+            | Q(published_at__lte=heartbeat_before)
+        ).limit(25)
        for row in rows:
            ca = await Certificate.get_or_none(id=row.ca_id)
            if ca is None:
@@ -226,7 +236,7 @@ class OcspPublisherMixin(KayaMixin):
                published_at = datetime.now(timezone.utc)
                cleared = await OcspSnapshot.filter(
                    ca_id=row.ca_id,
-                    dirty=True,
+                    dirty=row.dirty,
                    updated_at=row.updated_at,
                ).update(
                    dirty=False,
@@ -257,6 +267,7 @@ def publisher_mixin_from_settings(
    return OcspPublisherMixin(
        settings_obj.ocsp_publish_interval_seconds,
        ensure_context,
+        settings_obj.ocsp_heartbeat_interval_seconds,
    )


@@ -1038,6 +1038,79 @@ class OcspSnapshotTests(unittest.TestCase):
                assert row.dirty is False
                assert row.published_at is not None

+    @async_test
+    async def test_publisher_heartbeat_republishes_due_clean_snapshot(self) -> None:
+        from unittest.mock import patch
+
+        from kca import ocsp
+        from kca.models import OcspSnapshot
+
+        async with api_client() as client:
+            with oidc_user("alice"):
+                vault_id = await _vault(client)
+                ca_key = await _key(client, vault_id, "heartbeat due ca")
+                ca = await _cert(client, vault_id, ca_key["id"], "Heartbeat Due CA", ca=True)
+                previous_published_at = datetime.now(timezone.utc) - timedelta(seconds=31)
+                await OcspSnapshot.create(
+                    ca_id=ca["id"],
+                    dirty=False,
+                    published_at=previous_published_at,
+                )
+                published = []
+
+                async def fake_publish(publish_ca, snapshot):
+                    published.append(snapshot)
+                    return "key"
+
+                async def ensure_context():
+                    return None
+
+                mixin = ocsp.OcspPublisherMixin(
+                    1, ensure_context, heartbeat_interval_seconds=30
+                )
+                with patch.object(ocsp, "publish_snapshot", fake_publish):
+                    await mixin.run_once()
+
+                assert len(published) == 1
+                row = await OcspSnapshot.get(ca_id=ca["id"])
+                assert row.dirty is False
+                assert row.published_at is not None
+                assert row.published_at != previous_published_at
+
+    @async_test
+    async def test_publisher_heartbeat_skips_recent_clean_snapshot(self) -> None:
+        from unittest.mock import patch
+
+        from kca import ocsp
+        from kca.models import OcspSnapshot
+
+        async with api_client() as client:
+            with oidc_user("alice"):
+                vault_id = await _vault(client)
+                ca_key = await _key(client, vault_id, "heartbeat recent ca")
+                ca = await _cert(client, vault_id, ca_key["id"], "Heartbeat Recent CA", ca=True)
+                await OcspSnapshot.create(
+                    ca_id=ca["id"],
+                    dirty=False,
+                    published_at=datetime.now(timezone.utc),
+                )
+                published = []
+
+                async def fake_publish(publish_ca, snapshot):
+                    published.append(snapshot)
+                    return "key"
+
+                async def ensure_context():
+                    return None
+
+                mixin = ocsp.OcspPublisherMixin(
+                    1, ensure_context, heartbeat_interval_seconds=30
+                )
+                with patch.object(ocsp, "publish_snapshot", fake_publish):
+                    await mixin.run_once()
+
+                assert published == []
+
    @async_test
    async def test_imported_ca_cannot_overwrite_other_vault_snapshot(self) -> None:
        import json