Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions documentcloud/common/environment/minio/storage.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# Standard Library
import base64
import hashlib

# Third Party
import environ
from botocore.client import Config
Expand All @@ -8,6 +12,19 @@
env = environ.Env()


def _add_content_md5(params, **_kwargs):
"""boto3 >= 1.36 sends a CRC32 checksum instead of Content-MD5 for
operations that require a checksum (e.g. DeleteObjects), but older MinIO
releases reject those requests with MissingContentMD5. Add it back.
"""
body = params.get("body") or b""
if isinstance(body, str):
body = body.encode("utf-8")
params["headers"]["Content-MD5"] = base64.b64encode(
hashlib.md5(body).digest()
).decode("ascii")


class MinIOStorage(AwsStorage):
def __init__(self, resource_kwargs=None, minio=True):
if resource_kwargs is None:
Expand All @@ -19,6 +36,10 @@ def __init__(self, resource_kwargs=None, minio=True):
"region_name": "us-east-1",
}
super().__init__(resource_kwargs, minio)
for client in (self.s3_client, self.s3_resource.meta.client):
client.meta.events.register(
"before-call.s3.DeleteObjects", _add_content_md5
)


storage = MinIOStorage()
Loading