Use aioboto3 for asynchronous OCSP S3 uploads
This commit is contained in:
@@ -32,7 +32,7 @@ dependencies = [
|
|||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
s3 = [
|
s3 = [
|
||||||
"boto3>=1.34",
|
"aioboto3>=13.0",
|
||||||
]
|
]
|
||||||
otel = [
|
otel = [
|
||||||
"kaya-otel>=0.0.7",
|
"kaya-otel>=0.0.7",
|
||||||
|
|||||||
@@ -85,7 +85,7 @@ def cert_subject(ca: Certificate) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def _s3_client() -> Any:
|
def _s3_client() -> Any:
|
||||||
import boto3
|
import aioboto3
|
||||||
from botocore.config import Config
|
from botocore.config import Config
|
||||||
|
|
||||||
kwargs: Dict[str, Any] = {
|
kwargs: Dict[str, Any] = {
|
||||||
@@ -94,10 +94,9 @@ def _s3_client() -> Any:
|
|||||||
}
|
}
|
||||||
if settings.ocsp_s3_force_path_style:
|
if settings.ocsp_s3_force_path_style:
|
||||||
kwargs["config"] = Config(s3={"addressing_style": "path"})
|
kwargs["config"] = Config(s3={"addressing_style": "path"})
|
||||||
# boto3's create_client rejects an explicit None region/endpoint in some
|
# Only pass region and endpoint values that are explicitly configured.
|
||||||
# versions, so only pass values that are set.
|
|
||||||
kwargs = {key: value for key, value in kwargs.items() if value is not None}
|
kwargs = {key: value for key, value in kwargs.items() if value is not None}
|
||||||
return boto3.client("s3", **kwargs)
|
return aioboto3.Session().client("s3", **kwargs)
|
||||||
|
|
||||||
|
|
||||||
async def publish_snapshot(ca: Certificate, snapshot: Optional[Dict[str, Any]] = None) -> str:
|
async def publish_snapshot(ca: Certificate, snapshot: Optional[Dict[str, Any]] = None) -> str:
|
||||||
@@ -109,11 +108,8 @@ async def publish_snapshot(ca: Certificate, snapshot: Optional[Dict[str, Any]] =
|
|||||||
body = json.dumps(snapshot, indent=2, sort_keys=True).encode("utf-8")
|
body = json.dumps(snapshot, indent=2, sort_keys=True).encode("utf-8")
|
||||||
key = object_key(str(ca.id))
|
key = object_key(str(ca.id))
|
||||||
|
|
||||||
# boto3 is synchronous and credential resolution can block; do both the
|
async with _s3_client() as client:
|
||||||
# client creation and the PUT outside the event loop.
|
await client.put_object(
|
||||||
def upload() -> None:
|
|
||||||
client = _s3_client()
|
|
||||||
client.put_object(
|
|
||||||
Bucket=settings.ocsp_s3_bucket,
|
Bucket=settings.ocsp_s3_bucket,
|
||||||
Key=key,
|
Key=key,
|
||||||
Body=body,
|
Body=body,
|
||||||
@@ -121,7 +117,6 @@ async def publish_snapshot(ca: Certificate, snapshot: Optional[Dict[str, Any]] =
|
|||||||
CacheControl="no-store",
|
CacheControl="no-store",
|
||||||
)
|
)
|
||||||
|
|
||||||
await asyncio.to_thread(upload)
|
|
||||||
return key
|
return key
|
||||||
|
|
||||||
|
|
||||||
@@ -171,10 +166,10 @@ class OcspPublisherMixin(KayaMixin):
|
|||||||
log.info("OCSP S3 publication disabled (KCA_OCSP_S3_BUCKET unset)")
|
log.info("OCSP S3 publication disabled (KCA_OCSP_S3_BUCKET unset)")
|
||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
import boto3 # noqa: F401
|
import aioboto3 # noqa: F401
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"KCA_OCSP_S3_BUCKET is set but boto3 is not installed; "
|
"KCA_OCSP_S3_BUCKET is set but aioboto3 is not installed; "
|
||||||
"install kca-app with the 's3' extra"
|
"install kca-app with the 's3' extra"
|
||||||
) from exc
|
) from exc
|
||||||
self._task = loop.create_task(self._poll())
|
self._task = loop.create_task(self._poll())
|
||||||
|
|||||||
@@ -1222,10 +1222,16 @@ class OcspSnapshotTests(unittest.TestCase):
|
|||||||
from kca.models import OcspSnapshot
|
from kca.models import OcspSnapshot
|
||||||
|
|
||||||
class FakeS3:
|
class FakeS3:
|
||||||
|
async def __aenter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, exc_type, exc, traceback):
|
||||||
|
return None
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.calls = []
|
self.calls = []
|
||||||
|
|
||||||
def put_object(self, **kwargs):
|
async def put_object(self, **kwargs):
|
||||||
self.calls.append(kwargs)
|
self.calls.append(kwargs)
|
||||||
|
|
||||||
fake = FakeS3()
|
fake = FakeS3()
|
||||||
@@ -1342,10 +1348,16 @@ class OcspSnapshotTests(unittest.TestCase):
|
|||||||
from kca.models import Certificate, OcspSnapshot
|
from kca.models import Certificate, OcspSnapshot
|
||||||
|
|
||||||
class FakeS3:
|
class FakeS3:
|
||||||
|
async def __aenter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, exc_type, exc, traceback):
|
||||||
|
return None
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.objects = {}
|
self.objects = {}
|
||||||
|
|
||||||
def put_object(self, **kwargs):
|
async def put_object(self, **kwargs):
|
||||||
self.objects[kwargs["Key"]] = kwargs["Body"]
|
self.objects[kwargs["Key"]] = kwargs["Body"]
|
||||||
|
|
||||||
fake = FakeS3()
|
fake = FakeS3()
|
||||||
|
|||||||
Generated
+935
-102
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user