diff --git a/go.mod b/go.mod
index 2fab9ee..bb115e2 100644
--- a/go.mod
+++ b/go.mod
@@ -5,10 +5,23 @@ go 1.26.0
require (
github.com/alecthomas/errors v0.9.1
github.com/alecthomas/kong v1.16.1
+ github.com/aws/aws-sdk-go-v2 v1.47.3
+ github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3
+ github.com/aws/smithy-go v1.28.5
github.com/klauspost/compress v1.20.3
github.com/minio/crc64nvme v1.1.1
golang.org/x/sync v0.24.0
golang.org/x/sys v0.49.0
)
-require github.com/klauspost/cpuid/v2 v2.2.9 // indirect
+require (
+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect
+ github.com/klauspost/cpuid/v2 v2.2.9 // indirect
+)
diff --git a/go.sum b/go.sum
index 1a42164..c8de818 100644
--- a/go.sum
+++ b/go.sum
@@ -2,55 +2,41 @@ github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8v
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
github.com/alecthomas/errors v0.9.1 h1:JNXtU30rtMNARCkW41OTZ4yL6Lyocq20xIJgIw2raqI=
github.com/alecthomas/errors v0.9.1/go.mod h1:l8mjMEHMGUdIWPMNtvDyRYPVS1fQFXHFXc/iVCCLGkI=
-github.com/alecthomas/kong v1.13.0 h1:5e/7XC3ugvhP1DQBmTS+WuHtCbcv44hsohMgcvVxSrA=
-github.com/alecthomas/kong v1.13.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I=
-github.com/alecthomas/kong v1.15.0 h1:BVJstKbpO73zKpmIu+m/aLRrNmWwxXPIGTNin9VmLVI=
-github.com/alecthomas/kong v1.15.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I=
-github.com/alecthomas/kong v1.16.0 h1:g92/kUxBcdcTPOM79yE63viJgtcp5dNyrB3/O2cjYT4=
-github.com/alecthomas/kong v1.16.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I=
github.com/alecthomas/kong v1.16.1 h1:ixhCt93XkJ98kGposQ54+bl0IK6XwqB40AsMynU7Z8E=
github.com/alecthomas/kong v1.16.1/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I=
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk=
+github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 h1:oiEmPz/YMMeh/JvAE8xSxNCIsvWE6lpFe4ZPlyFykQI=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 h1:CwvFlKw0uUScfgOMflXSRZa+ayXgOf6wk7lhyMemQYU=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7/go.mod h1:4SbD8BB+p+QEnG0aX6DoWwrtk9SufyYXRsUv0nve8LE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 h1:NyeuyXkZL476uoAowNActm4XJO1QPKf7aassgXHpiqA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3/go.mod h1:GfoBeKMDJecgRUsGf/0RGdx0NJAh81ObTd922h8pQys=
+github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k=
+github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
-github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
-github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
-github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
-github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
-github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
-github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ=
-github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
-github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
-github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
-github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
-github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
-github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
-github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
-golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
-golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
-golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
-golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
-golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
-golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
-golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
-golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
-golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
-golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
-golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
diff --git a/gradlecache/gradlecache_test.go b/gradlecache/gradlecache_test.go
index c2265b4..f174028 100644
--- a/gradlecache/gradlecache_test.go
+++ b/gradlecache/gradlecache_test.go
@@ -17,6 +17,7 @@ import (
"strconv"
"strings"
"sync"
+ "sync/atomic"
"testing"
"time"
)
@@ -1699,11 +1700,13 @@ func (f *fakeS3) ServeHTTP(w http.ResponseWriter, r *http.Request) {
func newTestS3BundleStore(server *httptest.Server) *s3BundleStore {
client := &s3Client{
region: "us-east-1",
+ creds: awsCreds{AccessKeyID: "test-id", SecretAccessKey: "test-secret", SessionToken: "test-token"},
http: server.Client(),
chunkSize: defaultDownloadChunkSize,
dlWorkers: defaultDownloadWorkers,
testBaseURL: server.URL,
}
+ client.initReads()
return &s3BundleStore{client: client, bucket: "test-bucket"}
}
@@ -1867,7 +1870,11 @@ func TestS3ParallelDownloadInvalidRange(t *testing.T) {
{"server-error", http.StatusInternalServerError, chunkSize, false},
} {
t.Run(tc.name, func(t *testing.T) {
+ var attempts atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Range") == "bytes=0-256" {
+ attempts.Add(1)
+ }
w.WriteHeader(tc.status)
if tc.chunked {
w.(http.Flusher).Flush()
@@ -1877,7 +1884,7 @@ func TestS3ParallelDownloadInvalidRange(t *testing.T) {
defer srv.Close()
client := newTestS3BundleStore(srv).client
client.chunkSize, client.dlWorkers = chunkSize, 2
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
r, err := client.get(ctx, "bucket", "key", s3ObjInfo{Size: 5 * chunkSize})
must(t, err)
@@ -1885,6 +1892,13 @@ func TestS3ParallelDownloadInvalidRange(t *testing.T) {
if _, err := io.ReadAll(r); err == nil || stderrors.Is(err, context.DeadlineExceeded) {
t.Fatalf("expected a range error, got %v", err)
}
+ wantAttempts := int32(1)
+ if tc.status == http.StatusInternalServerError {
+ wantAttempts = 3
+ }
+ if attempts.Load() != wantAttempts {
+ t.Fatalf("attempts=%d, want %d (body failures must not resume)", attempts.Load(), wantAttempts)
+ }
})
}
}
@@ -1933,6 +1947,81 @@ func TestS3ParallelDownloadCancellation(t *testing.T) {
}
}
+func TestS3ReadRetries(t *testing.T) {
+ for _, operation := range []string{"head", "get", "range"} {
+ for _, tc := range []struct {
+ name string
+ status int
+ failures int32
+ attempts int32
+ wantErr bool
+ }{
+ {"recover", http.StatusServiceUnavailable, 1, 2, false},
+ {"exhausted", http.StatusServiceUnavailable, 3, 3, true},
+ {"forbidden", http.StatusForbidden, 1, 1, true},
+ } {
+ t.Run(operation+"/"+tc.name, func(t *testing.T) {
+ const chunkSize = 257
+ payload := bytes.Repeat([]byte("abc"), chunkSize)
+ var attempts atomic.Int32
+ server := newFakeS3()
+ server.objects["bucket/path with spaces/+key"] = payload
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if !strings.Contains(r.Header.Get("Authorization"), "Credential=test-id/") || r.Header.Get("X-Amz-Security-Token") != "test-token" {
+ t.Error("request did not use the resolved credentials")
+ }
+ // Count attempts for the first range only, not subsequent chunks.
+ if r.Header.Get("Range") == "" || r.Header.Get("Range") == "bytes=0-256" {
+ if attempts.Add(1) <= tc.failures {
+ w.WriteHeader(tc.status)
+ _, _ = io.WriteString(w, "InjectedFailure")
+ return
+ }
+ }
+ if r.Method == http.MethodHead {
+ w.Header().Set("Content-Length", strconv.Itoa(len(payload)))
+ w.Header().Set("ETag", `"etag"`)
+ w.Header().Set("X-Amz-Meta-Base-Commit", "abc123")
+ }
+ server.ServeHTTP(w, r)
+ }))
+ defer srv.Close()
+ client := newTestS3BundleStore(srv).client
+ ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+ defer cancel()
+ var err error
+ if operation == "head" {
+ var info s3ObjInfo
+ info, err = client.stat(ctx, "bucket", "path with spaces/+key")
+ if err == nil && (info.Size != int64(len(payload)) || info.ETag != `"etag"` || info.Metadata["base-commit"] != "abc123") {
+ t.Fatalf("incorrect object metadata: %+v", info)
+ }
+ } else {
+ if operation == "range" {
+ client.chunkSize, client.dlWorkers = chunkSize, 1
+ }
+ var r io.ReadCloser
+ r, err = client.get(ctx, "bucket", "path with spaces/+key", s3ObjInfo{Size: int64(len(payload))})
+ if err == nil {
+ defer r.Close() //nolint:errcheck
+ var got []byte
+ got, err = io.ReadAll(r)
+ if err == nil && !bytes.Equal(got, payload) {
+ t.Fatal("download differs from source")
+ }
+ }
+ }
+ if (err != nil) != tc.wantErr || ctx.Err() != nil {
+ t.Fatalf("error=%v, want error=%v, context=%v", err, tc.wantErr, ctx.Err())
+ }
+ if attempts.Load() != tc.attempts {
+ t.Fatalf("attempts=%d, want %d", attempts.Load(), tc.attempts)
+ }
+ })
+ }
+ }
+}
+
// ─── Helpers ─────────────────────────────────────────────────────────────────
func must(t *testing.T, err error) {
diff --git a/gradlecache/s3.go b/gradlecache/s3.go
index 40491e3..1d0bd7e 100644
--- a/gradlecache/s3.go
+++ b/gradlecache/s3.go
@@ -21,6 +21,10 @@ import (
"time"
"github.com/alecthomas/errors"
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ smithyhttp "github.com/aws/smithy-go/transport/http"
)
type awsCreds struct {
@@ -33,6 +37,7 @@ type s3Client struct {
region string
creds awsCreds
http *http.Client
+ reads *s3.Client
chunkSize int64
dlWorkers int
testBaseURL string
@@ -54,13 +59,36 @@ func newS3Client(region string) (*s3Client, error) {
WriteBufferSize: 128 << 10,
ReadBufferSize: 128 << 10,
}
- return &s3Client{
+ client := &s3Client{
region: region,
creds: creds,
http: &http.Client{Transport: transport},
chunkSize: defaultDownloadChunkSize,
dlWorkers: defaultDownloadWorkers,
- }, nil
+ }
+ client.initReads()
+ return client, nil
+}
+
+func (c *s3Client) initReads() {
+ // Preserve the existing credential resolution and HTTP transport. The SDK
+ // owns signing and retries for reads, not credential refresh or uploads.
+ opts := s3.Options{
+ Region: c.region,
+ HTTPClient: c.http,
+ Credentials: aws.CredentialsProviderFunc(func(context.Context) (aws.Credentials, error) {
+ return aws.Credentials{
+ AccessKeyID: c.creds.AccessKeyID, SecretAccessKey: c.creds.SecretAccessKey,
+ SessionToken: c.creds.SessionToken,
+ }, nil
+ }),
+ ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired,
+ }
+ if c.testBaseURL != "" {
+ opts.BaseEndpoint = aws.String(c.testBaseURL)
+ opts.UsePathStyle = true
+ }
+ c.reads = s3.New(opts)
}
type s3ObjInfo struct {
@@ -70,54 +98,36 @@ type s3ObjInfo struct {
}
func (c *s3Client) stat(ctx context.Context, bucket, key string) (s3ObjInfo, error) {
- req, err := http.NewRequestWithContext(ctx, http.MethodHead, c.objectURL(bucket, key), nil)
+ resp, err := c.reads.HeadObject(ctx, &s3.HeadObjectInput{
+ Bucket: aws.String(bucket), Key: aws.String(key),
+ })
if err != nil {
return s3ObjInfo{}, err
}
- c.sign(req)
- resp, err := c.http.Do(req)
- if err != nil {
- return s3ObjInfo{}, err
- }
- io.Copy(io.Discard, resp.Body) //nolint:errcheck,gosec
- resp.Body.Close() //nolint:errcheck,gosec
- if resp.StatusCode != http.StatusOK {
- return s3ObjInfo{}, errors.Errorf("status %d", resp.StatusCode)
- }
info := s3ObjInfo{
- Size: resp.ContentLength,
- ETag: resp.Header.Get("ETag"),
- }
- // net/http canonicalizes response header keys, so user metadata always
- // arrives as "X-Amz-Meta-". Collect it keyed by lowercased name.
- for k, vs := range resp.Header {
- name, ok := strings.CutPrefix(k, "X-Amz-Meta-")
- if !ok || len(vs) == 0 {
- continue
- }
- if info.Metadata == nil {
- info.Metadata = make(map[string]string)
- }
- info.Metadata[strings.ToLower(name)] = vs[0]
+ Size: -1,
+ ETag: aws.ToString(resp.ETag),
+ Metadata: resp.Metadata,
+ }
+ if resp.ContentLength != nil {
+ info.Size = *resp.ContentLength
}
return info, nil
}
func (c *s3Client) get(ctx context.Context, bucket, key string, info s3ObjInfo) (io.ReadCloser, error) {
if info.Size <= c.chunkSize {
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.objectURL(bucket, key), nil)
- if err != nil {
- return nil, err
- }
- c.sign(req)
- resp, err := c.http.Do(req) //nolint:gosec
+ resp, err := c.reads.GetObject(ctx, &s3.GetObjectInput{
+ Bucket: aws.String(bucket), Key: aws.String(key),
+ })
if err != nil {
return nil, err
}
- if resp.StatusCode != http.StatusOK {
+ status := awsmiddleware.GetRawResponse(resp.ResultMetadata).(*smithyhttp.Response).StatusCode
+ if status != http.StatusOK {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
resp.Body.Close() //nolint:errcheck,gosec
- return nil, errors.Errorf("s3 GET: status %d: %s", resp.StatusCode, body)
+ return nil, errors.Errorf("s3 GET: status %d: %s", status, body)
}
return resp.Body, nil
}
@@ -192,26 +202,25 @@ func (c *s3Client) parallelGet(ctx context.Context, bucket, key string, info s3O
}
start := int64(seq) * c.chunkSize
end := min(start+c.chunkSize-1, info.Size-1)
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.objectURL(bucket, key), nil)
- if err != nil {
- return err
+ input := &s3.GetObjectInput{
+ Bucket: aws.String(bucket), Key: aws.String(key),
+ Range: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)),
}
- req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
if info.ETag != "" {
- req.Header.Set("If-Match", info.ETag)
+ input.IfMatch = aws.String(info.ETag)
}
- c.sign(req)
- resp, err := c.http.Do(req) //nolint:gosec
+ resp, err := c.reads.GetObject(ctx, input)
if err != nil {
return err
}
defer resp.Body.Close() //nolint:errcheck,gosec
- if resp.StatusCode != http.StatusPartialContent {
+ status := awsmiddleware.GetRawResponse(resp.ResultMetadata).(*smithyhttp.Response).StatusCode
+ if status != http.StatusPartialContent {
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
- return errors.Errorf("s3 GET range %d-%d: status %d: %s", start, end, resp.StatusCode, msg)
+ return errors.Errorf("s3 GET range %d-%d: status %d: %s", start, end, status, msg)
}
- if resp.ContentLength >= 0 && resp.ContentLength != end-start+1 {
- return errors.Errorf("s3 GET range %d-%d: unexpected length %d", start, end, resp.ContentLength)
+ if resp.ContentLength != nil && *resp.ContentLength != end-start+1 {
+ return errors.Errorf("s3 GET range %d-%d: unexpected length %d", start, end, *resp.ContentLength)
}
for remaining := end - start + 1; remaining > 0; {
data = data[:min(blockSize, remaining)]