Fix stale OCSP snapshots indefinitely

- Parse and validate  from snapshot JSON, reject future/missing timestamps
- Enforce snapshot age vs  in cache and stale fallback (no lifetime extension on repeat S3 reads)
- OCSP: , ,
- Remove unused  from signer
- Publisher heartbeats: , republish clean snapshots past interval, preserve dirty-only behavior and concurrency
- Config/env/docker/docs updates for heartbeat

Rolling upgrade: enable publisher heartbeats before deploying the age-enforcing responder so existing snapshots remain within the age window.

ocsp-responder/README.md

@@ -27,8 +27,9 @@ vaults. The responder validates both this ID and the CA certificate fingerprint
 against every downloaded snapshot.
 
 KCA publishes these objects when `KCA_OCSP_S3_BUCKET` is configured. The JSON
-contains all certificates issued by that CA, both valid and revoked. The
-responder never infers status from a partial revoked-only list.
+contains all certificates issued by that CA, both valid and revoked, and a
+required `generated_at` timestamp. The responder never infers status from a
+partial revoked-only list.
 
 ## Upgrading from fingerprint-named snapshots
 
@@ -97,7 +98,19 @@ build a trust store containing the CA or validate with your normal PKI tooling.
 
 - Per-CA cache TTL: `s3.cache_ttl_ms` (default 1000 ms).
 - Concurrent requests for the same CA share one S3 `GetObject`.
-- If S3 fails, a previously cached snapshot is served for
-  `s3.stale_grace_seconds`; otherwise the OCSP response is `tryLater`.
+- `s3.stale_grace_seconds` is also the hard maximum snapshot age, measured from
+  `generated_at`. Expired or future-dated snapshots are rejected even when S3
+  remains readable; if no valid cached snapshot remains, OCSP returns
+  `tryLater`.
+- If a refresh fails, a cached snapshot is served only while both its time since
+  the last successful fetch and its `generated_at` age remain within
+  `s3.stale_grace_seconds`.
+- KCA republishes unchanged snapshots every
+  `KCA_OCSP_HEARTBEAT_INTERVAL_SECONDS` (default 30 seconds). Keep that interval
+  comfortably below the responder's `s3.stale_grace_seconds` (default 60). For
+  a rolling upgrade, enable publisher heartbeats before the freshness-enforcing
+  responder so existing snapshots remain within the age limit.
+- OCSP `thisUpdate` is the snapshot's `generated_at`. `nextUpdate` is capped at
+  the snapshot's hard expiry, while `producedAt` is the response signing time.
 - With multiple CAs, the TTL is per CA. If you need a global one-request-per-
   second limit across all CAs, that requires a shared rate limiter.

ocsp-responder/responder.example.yaml

@@ -11,6 +11,7 @@ s3:
   # endpoint: https://s3.example.com
   # force_path_style: true
   cache_ttl_ms: 1000
+  # Hard maximum snapshot age. Keep comfortably above KCA's heartbeat interval.
   stale_grace_seconds: 60
 
 issuers:

ocsp-responder/src/cache.rs

@@ -3,6 +3,7 @@
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
+use chrono::Utc;
 use tokio::sync::{Mutex, RwLock};
 
 use crate::snapshot::Snapshot;
@@ -27,7 +28,8 @@ impl SnapshotCache {
 
     /// Return a snapshot, refreshing from S3 at most once per `ttl`.
     ///
-    /// On refresh failure, data up to `ttl + stale_grace` old is still used.
+    /// Snapshots older than `stale_grace` are never used. On refresh failure,
+    /// cached data is also limited to `stale_grace` since its last fetch.
     pub async fn get<S: ObjectStore + ?Sized>(
         &self,
         store: &S,
@@ -38,7 +40,8 @@ impl SnapshotCache {
         stale_grace: Duration,
     ) -> Result<Arc<Snapshot>, CacheError> {
         if let Some(value) = self.cached.read().await.as_ref() {
-            if value.fetched_at.elapsed() < ttl {
+            if value.fetched_at.elapsed() < ttl && snapshot_age_valid(&value.snapshot, stale_grace)
+            {
                 return Ok(value.snapshot.clone());
             }
         }
@@ -49,7 +52,9 @@ impl SnapshotCache {
         let stale = {
             let cache = self.cached.read().await;
             if let Some(value) = cache.as_ref() {
-                if value.fetched_at.elapsed() < ttl {
+                if value.fetched_at.elapsed() < ttl
+                    && snapshot_age_valid(&value.snapshot, stale_grace)
+                {
                     return Ok(value.snapshot.clone());
                 }
                 Some(value.clone())
@@ -60,14 +65,17 @@ impl SnapshotCache {
 
         match store.get(key).await {
             Ok(bytes) => match Snapshot::parse(&bytes, expected_sha256, expected_ca_id) {
-                Ok(snapshot) => {
-                    let snapshot = Arc::new(snapshot);
-                    *self.cached.write().await = Some(CachedSnapshot {
-                        snapshot: snapshot.clone(),
-                        fetched_at: Instant::now(),
-                    });
-                    Ok(snapshot)
-                }
+                Ok(snapshot) => match validate_snapshot_age(&snapshot, stale_grace) {
+                    Ok(()) => {
+                        let snapshot = Arc::new(snapshot);
+                        *self.cached.write().await = Some(CachedSnapshot {
+                            snapshot: snapshot.clone(),
+                            fetched_at: Instant::now(),
+                        });
+                        Ok(snapshot)
+                    }
+                    Err(error) => self.fallback(stale, error, stale_grace),
+                },
                 Err(error) => self.fallback(stale, error.to_string(), stale_grace),
             },
             Err(error) => self.fallback(stale, error.to_string(), stale_grace),
@@ -81,7 +89,9 @@ impl SnapshotCache {
         stale_grace: Duration,
     ) -> Result<Arc<Snapshot>, CacheError> {
         if let Some(value) = stale {
-            if value.fetched_at.elapsed() <= stale_grace {
+            if value.fetched_at.elapsed() <= stale_grace
+                && snapshot_age_valid(&value.snapshot, stale_grace)
+            {
                 tracing::warn!(error = %error, "serving stale OCSP snapshot");
                 return Ok(value.snapshot);
             }
@@ -90,6 +100,29 @@ impl SnapshotCache {
     }
 }
 
+fn snapshot_age_valid(snapshot: &Snapshot, stale_grace: Duration) -> bool {
+    validate_snapshot_age(snapshot, stale_grace).is_ok()
+}
+
+fn validate_snapshot_age(snapshot: &Snapshot, stale_grace: Duration) -> Result<(), String> {
+    let age = Utc::now()
+        .signed_duration_since(snapshot.generated_at)
+        .to_std()
+        .map_err(|_| {
+            format!(
+                "snapshot generation time {} is in the future",
+                snapshot.generated_at
+            )
+        })?;
+    if age > stale_grace {
+        return Err(format!(
+            "snapshot generated at {} exceeds maximum age {stale_grace:?}",
+            snapshot.generated_at
+        ));
+    }
+    Ok(())
+}
+
 #[derive(Debug, thiserror::Error)]
 pub enum CacheError {
     #[error("OCSP snapshot unavailable: {0}")]
@@ -134,16 +167,17 @@ mod tests {
         }
     }
 
-    fn body(sha: &str) -> String {
+    fn body(sha: &str, generated_at: chrono::DateTime<Utc>) -> String {
+        let generated_at = generated_at.to_rfc3339();
         format!(
-            r#"{{"schema":"kca-ocsp-status/v1","issuer":{{"id":"ca","sha256":"{sha}"}},"entries":[{{"serial_hex":"01","status":"valid"}}]}}"#
+            r#"{{"schema":"kca-ocsp-status/v1","issuer":{{"id":"ca","sha256":"{sha}"}},"generated_at":"{generated_at}","entries":[{{"serial_hex":"01","status":"valid"}}]}}"#
         )
     }
 
     #[tokio::test]
     async fn single_flight_refreshes_once() {
         let sha = sha256_hex(b"ca");
-        let store = FakeStore::new(body(&sha));
+        let store = FakeStore::new(body(&sha, Utc::now()));
         let cache = SnapshotCache::new();
         let (a, b) = tokio::join!(
             cache.get(
@@ -171,7 +205,7 @@ mod tests {
     #[tokio::test]
     async fn refreshes_after_ttl_and_serves_stale_on_error() {
         let sha = sha256_hex(b"ca");
-        let store = FakeStore::new(body(&sha));
+        let store = FakeStore::new(body(&sha, Utc::now()));
         let cache = SnapshotCache::new();
         cache
             .get(
@@ -200,4 +234,46 @@ mod tests {
         assert_eq!(stale.entries.len(), 1);
         assert_eq!(store.calls.load(Ordering::SeqCst), 2);
     }
+
+    #[tokio::test]
+    async fn rejects_expired_snapshot_from_healthy_store() {
+        let sha = sha256_hex(b"ca");
+        let generated_at = Utc::now() - chrono::Duration::seconds(30);
+        let store = FakeStore::new(body(&sha, generated_at));
+        let cache = SnapshotCache::new();
+        let result = cache
+            .get(
+                &store,
+                "key",
+                &sha,
+                "ca",
+                Duration::from_millis(1),
+                Duration::from_secs(1),
+            )
+            .await;
+        assert!(matches!(result, Err(CacheError::Unavailable(_))));
+        assert_eq!(store.calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn successful_fetches_do_not_extend_snapshot_lifetime() {
+        let sha = sha256_hex(b"ca");
+        let store = FakeStore::new(body(&sha, Utc::now()));
+        let cache = SnapshotCache::new();
+        let ttl = Duration::from_secs(1);
+        let stale_grace = Duration::from_millis(100);
+        cache
+            .get(&store, "key", &sha, "ca", ttl, stale_grace)
+            .await
+            .unwrap();
+
+        tokio::time::sleep(Duration::from_millis(120)).await;
+        let expired = cache.get(&store, "key", &sha, "ca", ttl, stale_grace).await;
+        assert!(matches!(expired, Err(CacheError::Unavailable(_))));
+
+        store.fail.store(true, Ordering::SeqCst);
+        let unavailable = cache.get(&store, "key", &sha, "ca", ttl, stale_grace).await;
+        assert!(matches!(unavailable, Err(CacheError::Unavailable(_))));
+        assert_eq!(store.calls.load(Ordering::SeqCst), 3);
+    }
 }

ocsp-responder/src/ocsp.rs

@@ -8,6 +8,7 @@ use axum::extract::{Path, State};
 use axum::http::{header, HeaderValue, StatusCode};
 use axum::response::{IntoResponse, Response};
 use base64::Engine;
+use chrono::{DateTime, Utc};
 use der::{Decode, Encode};
 use x509_cert::ext::pkix::CrlReason;
 use x509_ocsp::builder::OcspResponseBuilder;
@@ -17,7 +18,7 @@ use x509_ocsp::{
 
 use crate::config::Config;
 use crate::issuer::Issuer;
-use crate::signer::{now_time, system_time};
+use crate::signer::system_time;
 use crate::store::StatusStore;
 
 #[derive(Clone)]
@@ -74,19 +75,28 @@ async fn handle_request(state: &AppState, body: &[u8]) -> Response {
         }
     };
 
-    let now = match now_time() {
+    let now = SystemTime::now();
+    let produced_at = match OcspGeneralizedTime::try_from(now) {
         Ok(now) => now,
         Err(error) => {
             tracing::error!(error = %error, "could not create OCSP time");
             return ocsp_response(OcspResponse::internal_error());
         }
     };
-    let next_update = now_time()
-        .ok()
-        .and_then(|_| {
-            SystemTime::now().checked_add(Duration::from_secs(state.config.next_update_seconds))
-        })
-        .and_then(|value| OcspGeneralizedTime::try_from(value).ok());
+    let (this_update, next_update) = response_update_times(
+        snapshot.generated_at,
+        state.config.next_update_seconds,
+        state.config.s3.stale_grace_seconds,
+        now,
+    );
+    let this_update = match OcspGeneralizedTime::try_from(this_update) {
+        Ok(value) => value,
+        Err(error) => {
+            tracing::error!(error = %error, "could not create OCSP snapshot time");
+            return ocsp_response(OcspResponse::internal_error());
+        }
+    };
+    let next_update = next_update.and_then(|value| OcspGeneralizedTime::try_from(value).ok());
 
     let mut builder = OcspResponseBuilder::new(issuer.signer_cert.tbs_certificate.subject.clone());
     for item in &request.tbs_request.request_list {
@@ -108,7 +118,7 @@ async fn handle_request(state: &AppState, body: &[u8]) -> Response {
             Some(_) => CertStatus::good(),
             None => CertStatus::unknown(),
         };
-        let mut single = SingleResponse::new(item.req_cert.clone(), status, now);
+        let mut single = SingleResponse::new(item.req_cert.clone(), status, this_update);
         if let Some(next_update) = next_update {
             single = single.with_next_update(next_update);
         }
@@ -123,7 +133,7 @@ async fn handle_request(state: &AppState, body: &[u8]) -> Response {
     }
 
     let mut signer = issuer.signer.lock().await;
-    match signer.sign(builder, vec![issuer.signer_cert.clone()], now) {
+    match signer.sign(builder, vec![issuer.signer_cert.clone()], produced_at) {
         Ok(response) => ocsp_response(response),
         Err(error) => {
             tracing::error!(issuer = %issuer.name, error = %error, "OCSP signing failed");
@@ -132,6 +142,20 @@ async fn handle_request(state: &AppState, body: &[u8]) -> Response {
     }
 }
 
+fn response_update_times(
+    generated_at: DateTime<Utc>,
+    next_update_seconds: u64,
+    stale_grace_seconds: u64,
+    now: SystemTime,
+) -> (SystemTime, Option<SystemTime>) {
+    let this_update = system_time(generated_at);
+    let expires_at = this_update.checked_add(Duration::from_secs(stale_grace_seconds));
+    let next_update = now
+        .checked_add(Duration::from_secs(next_update_seconds))
+        .and_then(|next| expires_at.map(|expires| next.min(expires)));
+    (this_update, next_update)
+}
+
 fn ocsp_response(response: OcspResponse) -> Response {
     match response.to_der() {
         Ok(bytes) => (
@@ -149,3 +173,28 @@ fn ocsp_response(response: OcspResponse) -> Response {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn update_times_use_snapshot_age_and_bound_next_update() {
+        let generated_at = DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z")
+            .unwrap()
+            .with_timezone(&Utc);
+        let generated_at_system = system_time(generated_at);
+
+        let now = generated_at_system + Duration::from_secs(5);
+        let (this_update, next_update) = response_update_times(generated_at, 10, 60, now);
+        assert_eq!(this_update, generated_at_system);
+        assert_eq!(next_update, Some(now + Duration::from_secs(10)));
+
+        let now = generated_at_system + Duration::from_secs(55);
+        let (_, next_update) = response_update_times(generated_at, 10, 60, now);
+        assert_eq!(
+            next_update,
+            Some(generated_at_system + Duration::from_secs(60))
+        );
+    }
+}

ocsp-responder/src/signer.rs

@@ -83,10 +83,6 @@ pub fn read_certificate(path: &Path) -> Result<Certificate, SignerError> {
     Certificate::from_der(&data).map_err(|error| SignerError::Certificate(error.to_string()))
 }
 
-pub fn now_time() -> Result<OcspGeneralizedTime, SignerError> {
-    Ok(OcspGeneralizedTime::try_from(SystemTime::now())?)
-}
-
 pub fn system_time(value: chrono::DateTime<chrono::Utc>) -> SystemTime {
     if value.timestamp() >= 0 {
         SystemTime::UNIX_EPOCH
@@ -107,6 +103,4 @@ pub enum SignerError {
     Certificate(String),
     #[error("OCSP signing failed: {0}")]
     Sign(String),
-    #[error("time conversion failed: {0}")]
-    Time(#[from] der::Error),
 }

ocsp-responder/src/snapshot.rs

@@ -25,6 +25,7 @@ pub struct StatusEntry {
 
 #[derive(Debug, Clone)]
 pub struct Snapshot {
+    pub generated_at: DateTime<Utc>,
     pub entries: HashMap<Vec<u8>, StatusEntry>,
 }
 
@@ -32,6 +33,7 @@ pub struct Snapshot {
 struct RawSnapshot {
     schema: String,
     issuer: RawIssuer,
+    generated_at: DateTime<Utc>,
     #[serde(default)]
     entries: Vec<RawEntry>,
 }
@@ -65,6 +67,9 @@ impl Snapshot {
         if raw.schema != SCHEMA {
             return Err(SnapshotError::Schema(raw.schema));
         }
+        if raw.generated_at > Utc::now() {
+            return Err(SnapshotError::FutureGeneration(raw.generated_at));
+        }
         if !raw.issuer.id.eq_ignore_ascii_case(expected_ca_id) {
             return Err(SnapshotError::IssuerIdMismatch {
                 expected: expected_ca_id.to_string(),
@@ -110,7 +115,10 @@ impl Snapshot {
             }
         }
         let _ = raw.issuer.sha256;
-        Ok(Self { entries })
+        Ok(Self {
+            generated_at: raw.generated_at,
+            entries,
+        })
     }
 
     pub fn status_for_serial(&self, serial: &[u8]) -> Option<&StatusEntry> {
@@ -170,6 +178,8 @@ pub enum SnapshotError {
     Json(#[from] serde_json::Error),
     #[error("unsupported snapshot schema {0:?}")]
     Schema(String),
+    #[error("snapshot generation time {0} is in the future")]
+    FutureGeneration(DateTime<Utc>),
     #[error("snapshot issuer ID {actual} does not match configured CA ID {expected}")]
     IssuerIdMismatch { expected: String, actual: String },
     #[error("snapshot issuer {actual} does not match configured CA {expected}")]
@@ -228,6 +238,24 @@ mod tests {
         assert!(Snapshot::parse(duplicate.as_bytes(), &sha, "ca").is_err());
     }
 
+    #[test]
+    fn rejects_missing_or_future_generation_time() {
+        let sha = sha256_hex(b"ca");
+        let missing = body(&sha)
+            .lines()
+            .filter(|line| !line.contains("\"generated_at\""))
+            .collect::<Vec<_>>()
+            .join("\n");
+        assert!(Snapshot::parse(missing.as_bytes(), &sha, "ca").is_err());
+
+        let future = (Utc::now() + chrono::Duration::hours(1)).to_rfc3339();
+        let future = body(&sha).replace("2026-01-01T00:00:00Z", &future);
+        assert!(matches!(
+            Snapshot::parse(future.as_bytes(), &sha, "ca"),
+            Err(SnapshotError::FutureGeneration(_))
+        ));
+    }
+
     #[test]
     fn normalizes_serials() {
         assert_eq!(normalize_serial(&[0, 0, 1, 2]), vec![1, 2]);