diff --git a/documentcloud/common/environment/minio/storage.py b/documentcloud/common/environment/minio/storage.py index 25781c67..aea92575 100644 --- a/documentcloud/common/environment/minio/storage.py +++ b/documentcloud/common/environment/minio/storage.py @@ -1,3 +1,7 @@ +# Standard Library +import base64 +import hashlib + # Third Party import environ from botocore.client import Config @@ -8,6 +12,19 @@ env = environ.Env() +def _add_content_md5(params, **_kwargs): + """boto3 >= 1.36 sends a CRC32 checksum instead of Content-MD5 for + operations that require a checksum (e.g. DeleteObjects), but older MinIO + releases reject those requests with MissingContentMD5. Add it back. + """ + body = params.get("body") or b"" + if isinstance(body, str): + body = body.encode("utf-8") + params["headers"]["Content-MD5"] = base64.b64encode( + hashlib.md5(body).digest() + ).decode("ascii") + + class MinIOStorage(AwsStorage): def __init__(self, resource_kwargs=None, minio=True): if resource_kwargs is None: @@ -19,6 +36,10 @@ def __init__(self, resource_kwargs=None, minio=True): "region_name": "us-east-1", } super().__init__(resource_kwargs, minio) + for client in (self.s3_client, self.s3_resource.meta.client): + client.meta.events.register( + "before-call.s3.DeleteObjects", _add_content_md5 + ) storage = MinIOStorage()