Use S3 object expiry for OCSP snapshots

This commit is contained in:
2026-10-28 19:39:51 +00:00
parent 9da68a9625
commit a7cfda4167
17 changed files with 321 additions and 238 deletions
+80 -113
View File
@@ -1,9 +1,8 @@
//! One-second, single-flight cache of per-CA status snapshots.
use std::sync::Arc;
use std::time::{Duration, Instant};
use std::time::{Duration, Instant, SystemTime};
use chrono::Utc;
use tokio::sync::{Mutex, RwLock};
use crate::snapshot::Snapshot;
@@ -11,7 +10,8 @@ use crate::store::ObjectStore;
#[derive(Clone)]
pub struct CachedSnapshot {
snapshot: Arc<Snapshot>,
pub snapshot: Arc<Snapshot>,
pub expires_at: SystemTime,
fetched_at: Instant,
}
@@ -28,8 +28,8 @@ impl SnapshotCache {
/// Return a snapshot, refreshing from S3 at most once per `ttl`.
///
/// Snapshots older than `stale_grace` are never used. On refresh failure,
/// cached data is also limited to `stale_grace` since its last fetch.
/// A cached snapshot is never used after its S3 `Expires` timestamp,
/// including when a refresh fails.
pub async fn get<S: ObjectStore + ?Sized>(
&self,
store: &S,
@@ -37,12 +37,10 @@ impl SnapshotCache {
expected_sha256: &str,
expected_ca_id: &str,
ttl: Duration,
stale_grace: Duration,
) -> Result<Arc<Snapshot>, CacheError> {
) -> Result<CachedSnapshot, CacheError> {
if let Some(value) = self.cached.read().await.as_ref() {
if value.fetched_at.elapsed() < ttl && snapshot_age_valid(&value.snapshot, stale_grace)
{
return Ok(value.snapshot.clone());
if fresh_cache_entry_valid(value, ttl) {
return Ok(value.clone());
}
}
@@ -52,10 +50,8 @@ impl SnapshotCache {
let stale = {
let cache = self.cached.read().await;
if let Some(value) = cache.as_ref() {
if value.fetched_at.elapsed() < ttl
&& snapshot_age_valid(&value.snapshot, stale_grace)
{
return Ok(value.snapshot.clone());
if fresh_cache_entry_valid(value, ttl) {
return Ok(value.clone());
}
Some(value.clone())
} else {
@@ -64,21 +60,25 @@ impl SnapshotCache {
};
match store.get(key).await {
Ok(bytes) => match Snapshot::parse(&bytes, expected_sha256, expected_ca_id) {
Ok(snapshot) => match validate_snapshot_age(&snapshot, stale_grace) {
Ok(()) => {
let snapshot = Arc::new(snapshot);
*self.cached.write().await = Some(CachedSnapshot {
snapshot: snapshot.clone(),
Ok(stored) => {
let parsed = validate_expiry(stored.expires_at).and_then(|()| {
Snapshot::parse(&stored.body, expected_sha256, expected_ca_id)
.map_err(|error| error.to_string())
});
match parsed {
Ok(snapshot) => {
let cached = CachedSnapshot {
snapshot: Arc::new(snapshot),
expires_at: stored.expires_at,
fetched_at: Instant::now(),
});
Ok(snapshot)
};
*self.cached.write().await = Some(cached.clone());
Ok(cached)
}
Err(error) => self.fallback(stale, error, stale_grace),
},
Err(error) => self.fallback(stale, error.to_string(), stale_grace),
},
Err(error) => self.fallback(stale, error.to_string(), stale_grace),
Err(error) => self.fallback(stale, error),
}
}
Err(error) => self.fallback(stale, error.to_string()),
}
}
@@ -86,41 +86,34 @@ impl SnapshotCache {
&self,
stale: Option<CachedSnapshot>,
error: String,
stale_grace: Duration,
) -> Result<Arc<Snapshot>, CacheError> {
) -> Result<CachedSnapshot, CacheError> {
if let Some(value) = stale {
if value.fetched_at.elapsed() <= stale_grace
&& snapshot_age_valid(&value.snapshot, stale_grace)
{
tracing::warn!(error = %error, "serving stale OCSP snapshot");
return Ok(value.snapshot);
if expiry_valid(value.expires_at) {
tracing::warn!(error = %error, "serving unexpired cached OCSP snapshot after refresh failure");
return Ok(value);
}
}
Err(CacheError::Unavailable(error))
}
}
fn snapshot_age_valid(snapshot: &Snapshot, stale_grace: Duration) -> bool {
validate_snapshot_age(snapshot, stale_grace).is_ok()
fn fresh_cache_entry_valid(value: &CachedSnapshot, ttl: Duration) -> bool {
value.fetched_at.elapsed() < ttl && expiry_valid(value.expires_at)
}
fn validate_snapshot_age(snapshot: &Snapshot, stale_grace: Duration) -> Result<(), String> {
let age = Utc::now()
.signed_duration_since(snapshot.generated_at)
.to_std()
.map_err(|_| {
format!(
"snapshot generation time {} is in the future",
snapshot.generated_at
)
})?;
if age > stale_grace {
return Err(format!(
"snapshot generated at {} exceeds maximum age {stale_grace:?}",
snapshot.generated_at
));
fn expiry_valid(expires_at: SystemTime) -> bool {
expires_at > SystemTime::now()
}
fn validate_expiry(expires_at: SystemTime) -> Result<(), String> {
if expiry_valid(expires_at) {
Ok(())
} else {
Err(format!(
"snapshot expired at {}",
httpdate::fmt_http_date(expires_at)
))
}
Ok(())
}
#[derive(Debug, thiserror::Error)]
@@ -138,18 +131,29 @@ mod tests {
use super::*;
use crate::snapshot::sha256_hex;
use crate::store::{ObjectStore, StoreError};
use crate::store::{ObjectStore, StoreError, StoredSnapshot};
struct FakeStore {
body: String,
expires_at: SystemTime,
calls: AtomicUsize,
fail: AtomicBool,
}
impl FakeStore {
fn new(body: String) -> Self {
fn new(body: String, lifetime: Duration) -> Self {
Self {
body,
expires_at: SystemTime::now() + lifetime,
calls: AtomicUsize::new(0),
fail: AtomicBool::new(false),
}
}
fn expired(body: String) -> Self {
Self {
body,
expires_at: SystemTime::now() - Duration::from_secs(1),
calls: AtomicUsize::new(0),
fail: AtomicBool::new(false),
}
@@ -158,98 +162,65 @@ mod tests {
#[async_trait]
impl ObjectStore for FakeStore {
async fn get(&self, _key: &str) -> Result<Bytes, StoreError> {
async fn get(&self, _key: &str) -> Result<StoredSnapshot, StoreError> {
self.calls.fetch_add(1, Ordering::SeqCst);
if self.fail.load(Ordering::SeqCst) {
return Err(StoreError::S3("offline".to_string()));
}
Ok(Bytes::from(self.body.clone()))
Ok(StoredSnapshot {
body: Bytes::from(self.body.clone()),
expires_at: self.expires_at,
})
}
}
fn body(sha: &str, generated_at: chrono::DateTime<Utc>) -> String {
let generated_at = generated_at.to_rfc3339();
fn body(sha: &str) -> String {
format!(
r#"{{"schema":"kca-ocsp-status/v1","issuer":{{"id":"ca","sha256":"{sha}"}},"generated_at":"{generated_at}","entries":[{{"serial_hex":"01","status":"valid"}}]}}"#
r#"{{"schema":"kca-ocsp-status/v2","issuer":{{"id":"ca","sha256":"{sha}"}},"entries":[{{"serial_hex":"01","status":"valid"}}]}}"#
)
}
#[tokio::test]
async fn single_flight_refreshes_once() {
let sha = sha256_hex(b"ca");
let store = FakeStore::new(body(&sha, Utc::now()));
let store = FakeStore::new(body(&sha), Duration::from_secs(1));
let cache = SnapshotCache::new();
let (a, b) = tokio::join!(
cache.get(
&store,
"key",
&sha,
"ca",
Duration::from_millis(100),
Duration::from_secs(1)
),
cache.get(
&store,
"key",
&sha,
"ca",
Duration::from_millis(100),
Duration::from_secs(1)
)
cache.get(&store, "key", &sha, "ca", Duration::from_millis(100),),
cache.get(&store, "key", &sha, "ca", Duration::from_millis(100),)
);
assert!(a.is_ok());
assert!(b.is_ok());
assert_eq!(store.calls.load(Ordering::SeqCst), 1);
assert_eq!(a.unwrap().expires_at, store.expires_at);
}
#[tokio::test]
async fn refreshes_after_ttl_and_serves_stale_on_error() {
async fn refreshes_after_ttl_and_serves_unexpired_cache_on_error() {
let sha = sha256_hex(b"ca");
let store = FakeStore::new(body(&sha, Utc::now()));
let store = FakeStore::new(body(&sha), Duration::from_secs(1));
let cache = SnapshotCache::new();
cache
.get(
&store,
"key",
&sha,
"ca",
Duration::from_millis(20),
Duration::from_secs(1),
)
.get(&store, "key", &sha, "ca", Duration::from_millis(20))
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(30)).await;
store.fail.store(true, Ordering::SeqCst);
let stale = cache
.get(
&store,
"key",
&sha,
"ca",
Duration::from_millis(20),
Duration::from_secs(1),
)
.get(&store, "key", &sha, "ca", Duration::from_millis(20))
.await
.unwrap();
assert_eq!(stale.entries.len(), 1);
assert_eq!(stale.snapshot.entries.len(), 1);
assert_eq!(store.calls.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn rejects_expired_snapshot_from_healthy_store() {
let sha = sha256_hex(b"ca");
let generated_at = Utc::now() - chrono::Duration::seconds(30);
let store = FakeStore::new(body(&sha, generated_at));
let store = FakeStore::expired(body(&sha));
let cache = SnapshotCache::new();
let result = cache
.get(
&store,
"key",
&sha,
"ca",
Duration::from_millis(1),
Duration::from_secs(1),
)
.get(&store, "key", &sha, "ca", Duration::from_millis(1))
.await;
assert!(matches!(result, Err(CacheError::Unavailable(_))));
assert_eq!(store.calls.load(Ordering::SeqCst), 1);
@@ -258,21 +229,17 @@ mod tests {
#[tokio::test]
async fn successful_fetches_do_not_extend_snapshot_lifetime() {
let sha = sha256_hex(b"ca");
let store = FakeStore::new(body(&sha, Utc::now()));
let store = FakeStore::new(body(&sha), Duration::from_millis(100));
let cache = SnapshotCache::new();
let ttl = Duration::from_secs(1);
let stale_grace = Duration::from_millis(100);
cache
.get(&store, "key", &sha, "ca", ttl, stale_grace)
.await
.unwrap();
cache.get(&store, "key", &sha, "ca", ttl).await.unwrap();
tokio::time::sleep(Duration::from_millis(120)).await;
let expired = cache.get(&store, "key", &sha, "ca", ttl, stale_grace).await;
let expired = cache.get(&store, "key", &sha, "ca", ttl).await;
assert!(matches!(expired, Err(CacheError::Unavailable(_))));
store.fail.store(true, Ordering::SeqCst);
let unavailable = cache.get(&store, "key", &sha, "ca", ttl, stale_grace).await;
let unavailable = cache.get(&store, "key", &sha, "ca", ttl).await;
assert!(matches!(unavailable, Err(CacheError::Unavailable(_))));
assert_eq!(store.calls.load(Ordering::SeqCst), 3);
}
-7
View File
@@ -37,8 +37,6 @@ pub struct S3Config {
pub force_path_style: bool,
#[serde(default = "default_cache_ttl_ms")]
pub cache_ttl_ms: u64,
#[serde(default = "default_stale_grace_seconds")]
pub stale_grace_seconds: u64,
}
impl Default for S3Config {
@@ -50,7 +48,6 @@ impl Default for S3Config {
endpoint: None,
force_path_style: false,
cache_ttl_ms: default_cache_ttl_ms(),
stale_grace_seconds: default_stale_grace_seconds(),
}
}
}
@@ -81,10 +78,6 @@ fn default_cache_ttl_ms() -> u64 {
1000
}
fn default_stale_grace_seconds() -> u64 {
60
}
fn default_ocsp_next_update_seconds() -> u64 {
10
}
+1 -2
View File
@@ -112,7 +112,7 @@ impl Issuer {
&self,
store: &S,
config: &crate::config::S3Config,
) -> Result<Arc<crate::snapshot::Snapshot>, crate::cache::CacheError> {
) -> Result<crate::cache::CachedSnapshot, crate::cache::CacheError> {
let key = config.object_key(&self.ca_id);
self.cache
.get(
@@ -121,7 +121,6 @@ impl Issuer {
&self.fingerprint,
&self.ca_id,
Duration::from_millis(config.cache_ttl_ms),
Duration::from_secs(config.stale_grace_seconds),
)
.await
}
+17 -29
View File
@@ -8,7 +8,6 @@ use axum::extract::{Path, State};
use axum::http::{header, HeaderValue, StatusCode};
use axum::response::{IntoResponse, Response};
use base64::Engine;
use chrono::{DateTime, Utc};
use der::{Decode, Encode};
use x509_cert::ext::pkix::CrlReason;
use x509_ocsp::builder::OcspResponseBuilder;
@@ -67,13 +66,14 @@ async fn handle_request(state: &AppState, body: &[u8]) -> Response {
return ocsp_response(OcspResponse::unauthorized());
}
let snapshot = match issuer.snapshot(&state.store, &state.config.s3).await {
Ok(snapshot) => snapshot,
let cached = match issuer.snapshot(&state.store, &state.config.s3).await {
Ok(cached) => cached,
Err(error) => {
tracing::warn!(issuer = %issuer.name, error = %error, "OCSP snapshot unavailable");
return ocsp_response(OcspResponse::try_later());
}
};
let snapshot = &cached.snapshot;
let now = SystemTime::now();
let produced_at = match OcspGeneralizedTime::try_from(now) {
@@ -83,12 +83,8 @@ async fn handle_request(state: &AppState, body: &[u8]) -> Response {
return ocsp_response(OcspResponse::internal_error());
}
};
let (this_update, next_update) = response_update_times(
snapshot.generated_at,
state.config.next_update_seconds,
state.config.s3.stale_grace_seconds,
now,
);
let (this_update, next_update) =
response_update_times(cached.expires_at, state.config.next_update_seconds, now);
let this_update = match OcspGeneralizedTime::try_from(this_update) {
Ok(value) => value,
Err(error) => {
@@ -143,17 +139,14 @@ async fn handle_request(state: &AppState, body: &[u8]) -> Response {
}
fn response_update_times(
generated_at: DateTime<Utc>,
expires_at: SystemTime,
next_update_seconds: u64,
stale_grace_seconds: u64,
now: SystemTime,
) -> (SystemTime, Option<SystemTime>) {
let this_update = system_time(generated_at);
let expires_at = this_update.checked_add(Duration::from_secs(stale_grace_seconds));
let next_update = now
.checked_add(Duration::from_secs(next_update_seconds))
.and_then(|next| expires_at.map(|expires| next.min(expires)));
(this_update, next_update)
.map(|next| next.min(expires_at));
(now, next_update)
}
fn ocsp_response(response: OcspResponse) -> Response {
@@ -179,22 +172,17 @@ mod tests {
use super::*;
#[test]
fn update_times_use_snapshot_age_and_bound_next_update() {
let generated_at = DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z")
.unwrap()
.with_timezone(&Utc);
let generated_at_system = system_time(generated_at);
fn update_times_use_response_time_and_bound_next_update_to_expiry() {
let base = SystemTime::UNIX_EPOCH + Duration::from_secs(1_767_225_600);
let expires_at = base + Duration::from_secs(60);
let now = generated_at_system + Duration::from_secs(5);
let (this_update, next_update) = response_update_times(generated_at, 10, 60, now);
assert_eq!(this_update, generated_at_system);
let now = base + Duration::from_secs(5);
let (this_update, next_update) = response_update_times(expires_at, 10, now);
assert_eq!(this_update, now);
assert_eq!(next_update, Some(now + Duration::from_secs(10)));
let now = generated_at_system + Duration::from_secs(55);
let (_, next_update) = response_update_times(generated_at, 10, 60, now);
assert_eq!(
next_update,
Some(generated_at_system + Duration::from_secs(60))
);
let now = base + Duration::from_secs(55);
let (_, next_update) = response_update_times(expires_at, 10, now);
assert_eq!(next_update, Some(expires_at));
}
}
+3 -32
View File
@@ -8,7 +8,7 @@ use sha1::{Digest as _, Sha1};
use sha2::Sha256;
use x509_cert::ext::pkix::CrlReason;
pub const SCHEMA: &str = "kca-ocsp-status/v1";
pub const SCHEMA: &str = "kca-ocsp-status/v2";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Status {
@@ -25,7 +25,6 @@ pub struct StatusEntry {
#[derive(Debug, Clone)]
pub struct Snapshot {
pub generated_at: DateTime<Utc>,
pub entries: HashMap<Vec<u8>, StatusEntry>,
}
@@ -33,7 +32,6 @@ pub struct Snapshot {
struct RawSnapshot {
schema: String,
issuer: RawIssuer,
generated_at: DateTime<Utc>,
#[serde(default)]
entries: Vec<RawEntry>,
}
@@ -67,9 +65,6 @@ impl Snapshot {
if raw.schema != SCHEMA {
return Err(SnapshotError::Schema(raw.schema));
}
if raw.generated_at > Utc::now() {
return Err(SnapshotError::FutureGeneration(raw.generated_at));
}
if !raw.issuer.id.eq_ignore_ascii_case(expected_ca_id) {
return Err(SnapshotError::IssuerIdMismatch {
expected: expected_ca_id.to_string(),
@@ -115,10 +110,7 @@ impl Snapshot {
}
}
let _ = raw.issuer.sha256;
Ok(Self {
generated_at: raw.generated_at,
entries,
})
Ok(Self { entries })
}
pub fn status_for_serial(&self, serial: &[u8]) -> Option<&StatusEntry> {
@@ -178,8 +170,6 @@ pub enum SnapshotError {
Json(#[from] serde_json::Error),
#[error("unsupported snapshot schema {0:?}")]
Schema(String),
#[error("snapshot generation time {0} is in the future")]
FutureGeneration(DateTime<Utc>),
#[error("snapshot issuer ID {actual} does not match configured CA ID {expected}")]
IssuerIdMismatch { expected: String, actual: String },
#[error("snapshot issuer {actual} does not match configured CA {expected}")]
@@ -201,9 +191,8 @@ mod tests {
fn body(sha: &str) -> String {
format!(
r#"{{
"schema": "kca-ocsp-status/v1",
"schema": "kca-ocsp-status/v2",
"issuer": {{"id": "ca", "sha256": "{sha}", "subject": "CN=CA"}},
"generated_at": "2026-01-01T00:00:00Z",
"entries": [
{{"serial_hex": "01ab", "status": "valid", "not_after": "2027-01-01T00:00:00Z"}},
{{"serial_hex": "00ff", "status": "revoked", "revoked_at": "2026-02-03T04:05:06Z", "reason": "keyCompromise"}}
@@ -238,24 +227,6 @@ mod tests {
assert!(Snapshot::parse(duplicate.as_bytes(), &sha, "ca").is_err());
}
#[test]
fn rejects_missing_or_future_generation_time() {
let sha = sha256_hex(b"ca");
let missing = body(&sha)
.lines()
.filter(|line| !line.contains("\"generated_at\""))
.collect::<Vec<_>>()
.join("\n");
assert!(Snapshot::parse(missing.as_bytes(), &sha, "ca").is_err());
let future = (Utc::now() + chrono::Duration::hours(1)).to_rfc3339();
let future = body(&sha).replace("2026-01-01T00:00:00Z", &future);
assert!(matches!(
Snapshot::parse(future.as_bytes(), &sha, "ca"),
Err(SnapshotError::FutureGeneration(_))
));
}
#[test]
fn normalizes_serials() {
assert_eq!(normalize_serial(&[0, 0, 1, 2]), vec![1, 2]);
+112 -18
View File
@@ -1,11 +1,17 @@
//! S3-compatible object store access.
use std::time::SystemTime;
use async_trait::async_trait;
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::operation::get_object::GetObjectError;
use aws_sdk_s3::Client;
use bytes::Bytes;
use crate::config::S3Config;
const MAX_FETCH_ATTEMPTS: usize = 3;
#[derive(Clone)]
pub struct StatusStore {
client: Client,
@@ -31,34 +37,122 @@ impl StatusStore {
}
}
#[derive(Debug)]
pub struct StoredSnapshot {
pub body: Bytes,
pub expires_at: SystemTime,
}
#[async_trait]
pub trait ObjectStore: Send + Sync {
async fn get(&self, key: &str) -> Result<Bytes, StoreError>;
async fn get(&self, key: &str) -> Result<StoredSnapshot, StoreError>;
}
#[async_trait]
impl ObjectStore for StatusStore {
async fn get(&self, key: &str) -> Result<Bytes, StoreError> {
let response = self
.client
.get_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
.map_err(|error| StoreError::S3(format!("{error:?}")))?;
let data = response
.body
.collect()
.await
.map_err(|error| StoreError::S3(format!("{error:?}")))?
.into_bytes();
Ok(data)
async fn get(&self, key: &str) -> Result<StoredSnapshot, StoreError> {
// RustFS returns Expires from HeadObject but not GetObject. A
// conditional GET binds the body to the metadata returned by HEAD,
// even if the publisher overwrites the object between the two calls.
for attempt in 1..=MAX_FETCH_ATTEMPTS {
let head = self
.client
.head_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
.map_err(|error| StoreError::S3(format!("HeadObject failed: {error:?}")))?;
let expires_at = parse_expiry(head.expires_string.as_deref())?;
let etag = head
.e_tag
.as_deref()
.ok_or(StoreError::MissingEtag)?
.to_string();
let response = match self
.client
.get_object()
.bucket(&self.bucket)
.key(key)
.if_match(etag)
.send()
.await
{
Ok(response) => response,
Err(error) if attempt < MAX_FETCH_ATTEMPTS && is_precondition_failed(&error) => {
continue;
}
Err(error) => {
return Err(StoreError::S3(format!("GetObject failed: {error:?}")));
}
};
let body = response
.body
.collect()
.await
.map_err(|error| StoreError::S3(format!("GetObject body failed: {error:?}")))?
.into_bytes();
return Ok(StoredSnapshot { body, expires_at });
}
unreachable!("the final GET attempt either returns or fails")
}
}
fn parse_expiry(value: Option<&str>) -> Result<SystemTime, StoreError> {
let value = value.ok_or(StoreError::MissingExpiry)?;
let expires_at = httpdate::parse_http_date(value)
.map_err(|_| StoreError::InvalidExpiry(value.to_string()))?;
if expires_at <= SystemTime::now() {
return Err(StoreError::Expired(value.to_string()));
}
Ok(expires_at)
}
fn is_precondition_failed(error: &SdkError<GetObjectError>) -> bool {
error
.as_service_error()
.and_then(|error| error.meta().code())
== Some("PreconditionFailed")
}
#[derive(Debug, thiserror::Error)]
pub enum StoreError {
#[error("S3 GetObject failed: {0}")]
#[error("S3 request failed: {0}")]
S3(String),
#[error("S3 object has no Expires header")]
MissingExpiry,
#[error("S3 object has an invalid Expires header: {0}")]
InvalidExpiry(String),
#[error("S3 object expired at {0}")]
Expired(String),
#[error("S3 object has no ETag")]
MissingEtag,
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use super::*;
#[test]
fn parses_future_http_expiry() {
let expires = httpdate::fmt_http_date(SystemTime::now() + Duration::from_secs(60));
assert!(parse_expiry(Some(&expires)).is_ok());
}
#[test]
fn rejects_missing_invalid_and_expired_expiry() {
assert!(matches!(parse_expiry(None), Err(StoreError::MissingExpiry)));
assert!(matches!(
parse_expiry(Some("not-a-date")),
Err(StoreError::InvalidExpiry(_))
));
let expired = httpdate::fmt_http_date(SystemTime::now() - Duration::from_secs(1));
assert!(matches!(
parse_expiry(Some(&expired)),
Err(StoreError::Expired(_))
));
}
}