use a custom http client for s3 gets

quickwit/Cargo.toml

@@ -17,6 +17,7 @@ members = [
   "quickwit-directories",
   "quickwit-doc-mapper",
   "quickwit-dst",
+  "quickwit-http-client",
   "quickwit-index-management",
   "quickwit-indexing",
   "quickwit-ingest",
@@ -64,6 +65,7 @@ default-members = [
   "quickwit-datetime",
   "quickwit-directories",
   "quickwit-doc-mapper",
+  "quickwit-http-client",
   "quickwit-index-management",
   "quickwit-indexing",
   "quickwit-ingest",
@@ -154,6 +156,8 @@ http = "1.4"
 http-body = "1.0"
 http-body-util = "0.1"
 http-serde = "2.1"
+httparse = "1.10"
+socket2 = { version = "0.6", features = ["all"] }
 humantime = "2.3"
 hyper = { version = "1.9", features = ["client", "http1", "http2", "server"] }
 hyper-rustls = "0.27"
@@ -253,6 +257,7 @@ reqwest-retry = "0.8"
 rust-embed = "8.11"
 rustc-hash = "2.1"
 rustls = "0.23"
+rustls-native-certs = "0.8"
 rustls-pemfile = "2.2"
 sea-query = { version = "0.32" }
 sea-query-binder = { version = "0.7", features = [
@@ -296,6 +301,7 @@ tokio-metrics = { version = "0.5", features = ["rt"] }
 tokio-rustls = { version = "0.26", default-features = false }
 tokio-stream = { version = "0.1", features = ["sync"] }
 tokio-util = { version = "0.7", default-features = false, features = [
+  "codec",
   "compat",
   "io-util",
 ] }
@@ -403,6 +409,7 @@ quickwit-df-core = { path = "quickwit-df-core" }
 quickwit-directories = { path = "quickwit-directories" }
 quickwit-doc-mapper = { path = "quickwit-doc-mapper" }
 quickwit-dst = { path = "quickwit-dst" }
+quickwit-http-client = { path = "quickwit-http-client" }
 quickwit-index-management = { path = "quickwit-index-management" }
 quickwit-indexing = { path = "quickwit-indexing" }
 quickwit-ingest = { path = "quickwit-ingest" }

quickwit/quickwit-http-client/Cargo.toml

@@ -0,0 +1,37 @@
+[package]
+name = "quickwit-http-client"
+description = "A fast HTTP/1.1 client for Quickwit object storage"
+version.workspace = true
+edition.workspace = true
+homepage.workspace = true
+documentation.workspace = true
+repository.workspace = true
+authors.workspace = true
+license.workspace = true
+
+[features]
+# enable s3 adaptor
+s3 = ["dep:aws-smithy-runtime-api", "dep:aws-smithy-types"]
+
+[dependencies]
+bytes = { workspace = true }
+futures = { workspace = true }
+http = { workspace = true }
+http-body = { workspace = true }
+http-body-util = { workspace = true }
+httparse = { workspace = true }
+rustls = { workspace = true }
+rustls-native-certs = { workspace = true }
+socket2 = { workspace = true }
+thiserror = { workspace = true }
+tokio = { workspace = true }
+tokio-rustls = { workspace = true, features = ["aws_lc_rs", "tls12"] }
+tokio-util = { workspace = true, features = ["rt"] }
+tracing = { workspace = true }
+
+aws-smithy-runtime-api = { workspace = true, features = ["client", "http-1x"], optional = true }
+aws-smithy-types = { workspace = true, optional = true }
+
+[dev-dependencies]
+rustls-pemfile = { workspace = true }
+tokio = { workspace = true, features = ["test-util"] }