diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30a4205..b6e33a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -263,8 +263,8 @@ jobs: apt-get-add-architecture: ${{ matrix.x86 && 'i386' || '' }} apt-get: >- ${{ matrix.install }} - build-essential libssl-dev zlib1g-dev libbrotli-dev - ${{ matrix.x86 && 'libssl-dev:i386 zlib1g-dev:i386 libbrotli-dev:i386' || '' }} + build-essential libssl-dev zlib1g-dev libbrotli-dev libzstd-dev + ${{ matrix.x86 && 'libssl-dev:i386 zlib1g-dev:i386 libbrotli-dev:i386 libzstd-dev:i386' || '' }} - name: Clone Boost uses: alandefreitas/cpp-actions/boost-clone@v1.9.4 @@ -283,7 +283,7 @@ jobs: if: ${{ startsWith(matrix.runs-on, 'windows') }} id: package-install-windows with: - vcpkg: zlib brotli openssl + vcpkg: zlib brotli zstd openssl vcpkg-dir: vcpkg-root vcpkg-triplet: ${{ matrix.x86 && 'x86-windows-static' || 'x64-windows' }} @@ -303,6 +303,7 @@ jobs: echo "using zlib : : \"${home}/vcpkg-root/installed/${triplet}/include\" \"${home}/vcpkg-root/installed/${triplet}/lib\" \"${home}/vcpkg-root/installed/${triplet}/bin\" zlib : ${addrmdl} ;" | sed 's/\/d\//D:\//g' >> user-config.jam echo "using brotli : : \"${home}/vcpkg-root/installed/${triplet}/include\" \"${home}/vcpkg-root/installed/${triplet}/lib\" \"${home}/vcpkg-root/installed/${triplet}/bin\" : ${addrmdl} ;" | sed 's/\/d\//D:\//g' >> user-config.jam + echo "using zstd : : \"${home}/vcpkg-root/installed/${triplet}/include\" \"${home}/vcpkg-root/installed/${triplet}/lib\" \"${home}/vcpkg-root/installed/${triplet}/bin\" : ${addrmdl} ;" | sed 's/\/d\//D:\//g' >> user-config.jam echo "using openssl : : \"${home}/vcpkg-root/installed/${triplet}/include\" \"${home}/vcpkg-root/installed/${triplet}/lib\" \"${home}/vcpkg-root/installed/${triplet}/bin\" libssl libcrypto : ${addrmdl} ;" | sed 's/\/d\//D:\//g' >> user-config.jam cat user-config.jam diff --git a/CMakeLists.txt b/CMakeLists.txt index f140aee..41fc9a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -193,33 +193,6 @@ if (Libpsl_FOUND) target_compile_definitions(boost_burl PRIVATE BOOST_BURL_HAS_LIBPSL) endif () -#------------------------------------------------- -# -# Compression backends (optional) -# -#------------------------------------------------- - -# zlib: deflate and gzip content codings -find_package(ZLIB) -if (ZLIB_FOUND) - target_link_libraries(boost_burl PRIVATE ZLIB::ZLIB) - target_compile_definitions(boost_burl PUBLIC BOOST_BURL_HAS_ZLIB) -endif () - -# Brotli: br content coding (decoding only) -find_package(Brotli) -if (Brotli_FOUND) - target_link_libraries(boost_burl PRIVATE Brotli::common Brotli::decoder) - target_compile_definitions(boost_burl PUBLIC BOOST_BURL_HAS_BROTLI) -endif () - -# Zstandard: zstd content coding -find_package(Zstd) -if (Zstd_FOUND) - target_link_libraries(boost_burl PRIVATE Zstd::Zstd) - target_compile_definitions(boost_burl PUBLIC BOOST_BURL_HAS_ZSTD) -endif () - #------------------------------------------------- # # Tests diff --git a/README.md b/README.md index 872abfc..b7dd9ab 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,8 @@ One line to make a request. - **Automatic redirects** — 301/302/303/307/308 with standards-compliant method changes, `Referer` handling, and credential stripping on cross-origin hops. -- **Content encodings** — transparent `gzip`, `deflate`, and `br` decoding - when the corresponding decode service is installed. +- **Content encodings** — transparent `gzip`, `deflate`, `br`, and `zstd` + decoding when the corresponding decode service is installed. - **Cookies** — RFC 6265bis jar with optional public-suffix validation (libpsl). - **Authentication** — Basic and Bearer, per client or per request. - **Proxies** — `http`, `socks5`, with credentials. diff --git a/build/Jamfile b/build/Jamfile index 2a3e016..fa0e4d3 100644 --- a/build/Jamfile +++ b/build/Jamfile @@ -11,9 +11,6 @@ import ac ; import config : requires ; using openssl ; -using zlib ; -using brotli ; -using zstd ; constant c11-requires : [ requires @@ -41,9 +38,6 @@ lib boost_burl /boost//capy /boost//corosio [ ac.check-library /boost/corosio//boost_corosio_openssl : /boost/corosio//boost_corosio_openssl : ] - [ ac.check-library /zlib//zlib : /zlib//zlib BOOST_BURL_HAS_ZLIB : ] - [ ac.check-library /brotli//brotlidec : /brotli//brotlicommon /brotli//brotlidec BOOST_BURL_HAS_BROTLI : ] - [ ac.check-library /zstd//zstd : /zstd//zstd BOOST_BURL_HAS_ZSTD : ] /boost//http /boost//json /boost//url @@ -53,9 +47,6 @@ lib boost_burl /boost//capy /boost//corosio [ ac.check-library /boost/corosio//boost_corosio_openssl : /boost/corosio//boost_corosio_openssl : ] - [ ac.check-library /zlib//zlib : /zlib//zlib BOOST_BURL_HAS_ZLIB : ] - [ ac.check-library /brotli//brotlidec : /brotli//brotlicommon /brotli//brotlidec BOOST_BURL_HAS_BROTLI : ] - [ ac.check-library /zstd//zstd : /zstd//zstd BOOST_BURL_HAS_ZSTD : ] /boost//http /boost//json /boost//url diff --git a/cmake/FindBrotli.cmake b/cmake/FindBrotli.cmake deleted file mode 100644 index cb6f884..0000000 --- a/cmake/FindBrotli.cmake +++ /dev/null @@ -1,50 +0,0 @@ -# -# Copyright (c) 2026 Mohammad Nejati -# -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -# -# Official repository: https://github.com/cppalliance/burl -# - -# Provides imported targets: -# Brotli::common -# Brotli::decoder -# Brotli::encoder - -find_path(Brotli_INCLUDE_DIR NAMES "brotli/decode.h") -find_library(Brotli_COMMON_LIBRARY NAMES "brotlicommon") -find_library(Brotli_DEC_LIBRARY NAMES "brotlidec") -find_library(Brotli_ENC_LIBRARY NAMES "brotlienc") - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Brotli - REQUIRED_VARS - Brotli_INCLUDE_DIR - Brotli_COMMON_LIBRARY - Brotli_DEC_LIBRARY - Brotli_ENC_LIBRARY -) - -if(Brotli_FOUND) - add_library(Brotli::common UNKNOWN IMPORTED) - set_target_properties(Brotli::common PROPERTIES - IMPORTED_LOCATION "${Brotli_COMMON_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${Brotli_INCLUDE_DIR}") - - add_library(Brotli::decoder UNKNOWN IMPORTED) - set_target_properties(Brotli::decoder PROPERTIES - IMPORTED_LOCATION "${Brotli_DEC_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${Brotli_INCLUDE_DIR}") - - add_library(Brotli::encoder UNKNOWN IMPORTED) - set_target_properties(Brotli::encoder PROPERTIES - IMPORTED_LOCATION "${Brotli_ENC_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${Brotli_INCLUDE_DIR}") -endif() - -mark_as_advanced( - Brotli_INCLUDE_DIR - Brotli_COMMON_LIBRARY - Brotli_DEC_LIBRARY - Brotli_ENC_LIBRARY) diff --git a/cmake/FindZstd.cmake b/cmake/FindZstd.cmake deleted file mode 100644 index 740ea43..0000000 --- a/cmake/FindZstd.cmake +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright (c) 2026 Mohammad Nejati -# -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -# -# Official repository: https://github.com/cppalliance/burl -# - -# Provides imported targets: -# Zstd::Zstd - -find_path(Zstd_INCLUDE_DIR NAMES "zstd.h") -find_library(Zstd_LIBRARY NAMES zstd libzstd zstd_static) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Zstd - REQUIRED_VARS - Zstd_INCLUDE_DIR - Zstd_LIBRARY -) - -if(Zstd_FOUND) - add_library(Zstd::Zstd UNKNOWN IMPORTED) - set_target_properties(Zstd::Zstd PROPERTIES - IMPORTED_LOCATION "${Zstd_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${Zstd_INCLUDE_DIR}" - ) -endif() - -mark_as_advanced( - Zstd_INCLUDE_DIR - Zstd_LIBRARY) diff --git a/doc/modules/ROOT/pages/2.guide/2m.compression.adoc b/doc/modules/ROOT/pages/2.guide/2m.compression.adoc index bdbc467..0f88812 100644 --- a/doc/modules/ROOT/pages/2.guide/2m.compression.adoc +++ b/doc/modules/ROOT/pages/2.guide/2m.compression.adoc @@ -11,21 +11,40 @@ Burl can advertise the content codings it accepts and transparently decode a compressed response body, so `gzip`, `deflate`, `br`, and `zstd` responses -arrive already decompressed. Decoding for each coding is compiled into the -library when it is built with the corresponding compression library. +arrive already decompressed. Decoding depends on the relevant Boost.Http +service being installed in the system context. -== Build Support +== Installing the Decode Services -The build detects zlib, Brotli, and zstd and enables the codings they provide: -zlib handles `gzip` and `deflate`, Brotli handles `br`, and zstd handles -`zstd`. The macros `BOOST_BURL_HAS_ZLIB`, `BOOST_BURL_HAS_BROTLI`, and -`BOOST_BURL_HAS_ZSTD` indicate which codings the library was built with. +Decoding is provided by Boost.Http and must be registered in the system context once at startup, before any clients are constructed. + +The decoder services are provided by the `Boost::http_brotli`, `Boost::http_zlib`, and `Boost::http_zstd` libraries. Each library is available only when Boost.Http is built with the corresponding compression library. Link the libraries for the compression formats you use, and guard the installation calls with the feature macros they define. This keeps the program portable across builds that do not include all compression libraries. + +[source,cpp] +---- +// Install the available decoder services. +[[maybe_unused]] auto& ctx = capy::get_system_context(); + +#ifdef BOOST_HTTP_HAS_BROTLI +http::brotli::install_decode_service(ctx); +#endif + +#ifdef BOOST_HTTP_HAS_ZLIB +http::zlib::install_inflate_service(ctx); +#endif + +#ifdef BOOST_HTTP_HAS_ZSTD +http::zstd::install_decompress_service(ctx); +#endif +---- + +Burl does not link against any compression library directly. Instead, it looks up the corresponding decoder services at run time. == How It Works -Every supported coding is enabled by default: the client adds it to the -`Accept-Encoding` request header and decodes a response encoded with it. The -configuration controls each coding individually. +With a service installed and the corresponding setting enabled, the client adds +that coding to the `Accept-Encoding` request header and decodes a response +encoded with it. [source,cpp] ---- @@ -40,8 +59,9 @@ burl::client client(co_await capy::this_coro::executor, tls_ctx, cfg); [NOTE] ==== -A coding whose decoder was not compiled into the library is disabled regardless -of the setting, since the client cannot honor what it advertises. +A coding whose decode service is not installed when the client is constructed +is disabled regardless of the setting, since the client cannot honor what it +advertises. ==== The decoding is transparent: the body you read through diff --git a/doc/modules/ROOT/pages/index.adoc b/doc/modules/ROOT/pages/index.adoc index cc25d88..c767546 100644 --- a/doc/modules/ROOT/pages/index.adoc +++ b/doc/modules/ROOT/pages/index.adoc @@ -36,8 +36,8 @@ non-throwing alternatives. timeouts and per-host caps * *Automatic redirects* — 301/302/303/307/308 with standards-compliant method changes, `Referer` handling, and credential stripping on cross-origin hops -* *Content decoding* — transparent `gzip`, `deflate`, and `br` when the - corresponding decode service is installed +* *Content decoding* — transparent `gzip`, `deflate`, `br`, and `zstd` when + the corresponding decode service is installed * *Cookies* — an RFC 6265bis jar with optional public-suffix validation, and persistence in the Netscape cookie file format * *Authentication* — Basic and Bearer, set per client or per request @@ -90,8 +90,9 @@ Burl is a focused HTTP/1.1 client. It does not provide: Boost.Burl is built on Boost.Capy, Boost.Corosio, and Boost.Http, and uses Boost.URL, Boost.JSON, and Boost.System. Content decoding requires Boost.Http -built with zlib (`gzip`, `deflate`) and brotli (`br`). Public-suffix -validation of cookies requires libpsl. +built with zlib (`gzip`, `deflate`), brotli (`br`), and zstd (`zstd`), with +the corresponding decode services installed in the system context. +Public-suffix validation of cookies requires libpsl. == Code Conventions diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 740604d..20e005e 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -8,7 +8,12 @@ # add_executable(burl_example_usage usage.cpp) -target_link_libraries(burl_example_usage PRIVATE Boost::burl Boost::hash2) +target_link_libraries(burl_example_usage PRIVATE + Boost::burl + Boost::hash2 + $ + $ + $) find_package(nlohmann_json QUIET) if (nlohmann_json_FOUND) diff --git a/example/Jamfile b/example/Jamfile index dbe99ee..6b7363b 100644 --- a/example/Jamfile +++ b/example/Jamfile @@ -7,6 +7,8 @@ # Official repository: https://github.com/cppalliance/burl # +import ac ; + lib advapi32 ; lib crypt32 ; lib gdi32 ; @@ -16,6 +18,9 @@ project : requirements /boost/burl//boost_burl /boost/hash2//boost_hash2 + [ ac.check-library /boost/http//boost_http_zlib : /boost/http//boost_http_zlib : ] + [ ac.check-library /boost/http//boost_http_brotli : /boost/http//boost_http_brotli : ] + [ ac.check-library /boost/http//boost_http_zstd : /boost/http//boost_http_zstd : ] windows:advapi32 windows:crypt32 windows:gdi32 diff --git a/example/usage.cpp b/example/usage.cpp index 42cee00..1e77806 100644 --- a/example/usage.cpp +++ b/example/usage.cpp @@ -11,6 +11,9 @@ #include #include #include +#include +#include +#include #include #include @@ -567,6 +570,18 @@ main(int argc, char* argv[]) corosio::io_context ioc; corosio::tls_context tls_ctx; + // Install avialable decoder services. + [[maybe_unused]] auto& ctx = capy::get_system_context(); +#ifdef BOOST_HTTP_HAS_BROTLI + http::brotli::install_decode_service(ctx); +#endif +#ifdef BOOST_HTTP_HAS_ZLIB + http::zlib::install_inflate_service(ctx); +#endif +#ifdef BOOST_HTTP_HAS_ZSTD + http::zstd::install_decompress_service(ctx); +#endif + capy::run_async( ioc.get_executor(), [] {}, diff --git a/include/boost/burl/client.hpp b/include/boost/burl/client.hpp index 7143e30..133baac 100644 --- a/include/boost/burl/client.hpp +++ b/include/boost/burl/client.hpp @@ -139,11 +139,11 @@ class client When enabled, `br` is included in the `Accept-Encoding` header and response bodies are decoded transparently. - Effective only when the library was built - with Brotli support - (`BOOST_BURL_HAS_BROTLI`). Not applied - when the request carries an explicit - `Accept-Encoding` header. + Effective only when the Brotli decode + service is installed in the system + context. Not applied when the request + carries an explicit `Accept-Encoding` + header. */ bool brotli = true; @@ -153,9 +153,9 @@ class client the `Accept-Encoding` header and response bodies are decoded transparently. Effective only when the - library was built with zlib support - (`BOOST_BURL_HAS_ZLIB`). Not applied when - the request carries an explicit + zlib inflate service is installed in the + system context. Not applied when the + request carries an explicit `Accept-Encoding` header. */ bool deflate = true; @@ -165,11 +165,11 @@ class client When enabled, `gzip` is included in the `Accept-Encoding` header and response bodies are decoded transparently. - Effective only when the library was built - with zlib support - (`BOOST_BURL_HAS_ZLIB`). Not applied when - the request carries an explicit - `Accept-Encoding` header. + Effective only when the zlib inflate + service is installed in the system + context. Not applied when the request + carries an explicit `Accept-Encoding` + header. */ bool gzip = true; @@ -178,11 +178,11 @@ class client When enabled, `zstd` is included in the `Accept-Encoding` header and response bodies are decoded transparently. - Effective only when the library was built - with zstd support - (`BOOST_BURL_HAS_ZSTD`). Not applied when - the request carries an explicit - `Accept-Encoding` header. + Effective only when the zstd decompress + service is installed in the system + context. Not applied when the request + carries an explicit `Accept-Encoding` + header. */ bool zstd = true; @@ -335,9 +335,9 @@ class client /** Constructor. Constructs a client with the provided - configuration. Content codings whose - decoder was not compiled into the library - are disabled, regardless of the + configuration. Content codings whose decode + service is not installed in the system + context are disabled, regardless of the configuration. @param exec The executor used to perform diff --git a/include/boost/burl/detail/circular_buffer.hpp b/include/boost/burl/detail/circular_buffer.hpp index 178654d..bc3669c 100644 --- a/include/boost/burl/detail/circular_buffer.hpp +++ b/include/boost/burl/detail/circular_buffer.hpp @@ -50,6 +50,9 @@ struct circular_buffer std::array prepare() const noexcept; + capy::mutable_buffer + prepare_one() const noexcept; + void commit(std::size_t n) noexcept; diff --git a/include/boost/burl/detail/decoder.hpp b/include/boost/burl/detail/decoder.hpp new file mode 100644 index 0000000..cbc76dc --- /dev/null +++ b/include/boost/burl/detail/decoder.hpp @@ -0,0 +1,58 @@ +// +// Copyright (c) 2026 Mohammad Nejati +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/burl +// + +#ifndef BOOST_BURL_DETAIL_DECODER_HPP +#define BOOST_BURL_DETAIL_DECODER_HPP + +#include + +#include +#include + +namespace boost +{ +namespace burl +{ +namespace detail +{ + +// A content decoder, transforming the payload +// octets as they arrive. +struct decoder +{ + struct result + { + // The number of input octets consumed. + std::size_t consumed; + + // The number of output octets produced. + std::size_t produced; + + // The error, if any. Set to `capy::error::eof` + // once the decoder has produced the complete + // output. + std::error_code ec; + }; + + virtual ~decoder() = default; + + // Transform payload octets. `more` is false when + // `in` ends the payload. + virtual result + process( + capy::mutable_buffer out, + capy::const_buffer in, + bool more) = 0; +}; + +} // namespace detail +} // namespace burl +} // namespace boost + +#endif diff --git a/include/boost/burl/error.hpp b/include/boost/burl/error.hpp index a1b511f..ce58b56 100644 --- a/include/boost/burl/error.hpp +++ b/include/boost/burl/error.hpp @@ -72,14 +72,6 @@ enum class error or a body was supplied for a message that has none. */ body_size_mismatch, - - /** The response body could not be decoded. - - Decoding the response body according to its - `Content-Encoding` failed because the compressed - data was corrupt or invalid. - */ - decode_error, }; /** Error conditions corresponding to sets of error codes. diff --git a/include/boost/burl/message_reader.hpp b/include/boost/burl/message_reader.hpp index 2d71a12..3fa6ef1 100644 --- a/include/boost/burl/message_reader.hpp +++ b/include/boost/burl/message_reader.hpp @@ -13,7 +13,6 @@ #include #include -#include #include #include #include @@ -72,12 +71,7 @@ namespace burl Every operation parses the header first, so reading a body without having read the header - explicitly works as expected. This serves the - caller who has no interest in the header: - anything decided from it, installing a decoder - above all, needs @ref read_header called - explicitly, because @ref parser::set_decoder - requires a parsed header and an untouched body. + explicitly works as expected. This type satisfies @ref capy::ReadStream, @ref http::ReadSource, and @ref http::BufferSource, @@ -310,20 +304,17 @@ read_some_( parser& pr, MB buffers) { - capy::buffer_param bp(buffers); - for(;;) { system::error_code ec; - auto const n = pr.read_some(bp.data(), ec); + auto const n = pr.read_some(buffers, ec); if(ec != http::error::need_data) co_return { std::error_code(ec), n }; if(auto const lim = pr.direct_capacity(); lim != 0) { - auto const mbs = bp.data(); auto [rec, rn] = co_await stream.read_some( - capy::buffer_slice(mbs, 0, lim)); + capy::buffer_slice(buffers, 0, lim)); pr.commit_direct(rn); if(rec == capy::cond::eof) pr.commit_eof(); diff --git a/include/boost/burl/parser.hpp b/include/boost/burl/parser.hpp index 7bca09f..ded6fda 100644 --- a/include/boost/burl/parser.hpp +++ b/include/boost/burl/parser.hpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -51,8 +52,7 @@ namespace burl @li the message header, with O(1) access to the start line, @li all or part of the message body, and - @li decoded output when a @ref decoder is - installed. + @li decoded output when the body is encoded. @par Operations @@ -62,7 +62,7 @@ namespace burl @li @ref flatten_body returns the whole body in place, without copying, @li @ref read_some copies into caller-supplied - memory, or lets an installed decoder write + memory, or lets the content decoder write into it directly, and @li @ref pull borrows the parser's own buffers, which @ref consume then releases. @@ -70,9 +70,17 @@ namespace burl Each parses the header first when it has not been parsed already, so a caller with no interest in the header never has to call @ref - parse_header. Installing a decoder does require - it, because @ref set_decoder must run after the - header and before any body octet. + parse_header. + + @par Content Decoding + + When @ref config::decode is set, a body whose + `Content-Encoding` is `gzip`, `deflate`, `br`, + or `zstd` is decoded as it is parsed, using the + decode service installed for that coding in the + system context. A body whose coding has no + installed service, or which the parser does not + know, is delivered as sent. @par Errors @@ -101,55 +109,6 @@ namespace burl class parser { public: - /** A content decoder. - - Installed with @ref set_decoder before body - parsing begins, a decoder transforms the - payload octets as they arrive. - */ - struct decoder - { - /// The outcome of a call to @ref process. - struct result - { - /// The number of input octets consumed. - std::size_t consumed; - - /// The number of output octets produced. - std::size_t produced; - - /** The error, if any. - - Set to `capy::error::eof` once the - decoder has produced the complete - output. - */ - std::error_code ec; - }; - - /// Destructor. - virtual ~decoder() = default; - - /** Transform payload octets. - - @param out The destination for decoded - output. - - @param in The octets to decode. - - @param more False when `in` ends the - payload. - - @return The octets consumed and - produced, and the error if any. - */ - virtual result - process( - capy::mutable_buffer out, - capy::const_buffer in, - bool more) = 0; - }; - /// Settings which apply for the life of the parser. struct config { @@ -164,6 +123,15 @@ class parser /// The default maximum body size. std::uint64_t body_limit = std::uint64_t(-1); + + /** Whether to decode the body. + + When true, a body whose `Content-Encoding` + names a coding with a decode service + installed in the system context is decoded + as it is parsed. + */ + bool decode = true; }; //-------------------------------------------- @@ -244,22 +212,6 @@ class parser void reset() noexcept; - /** Install a content decoder. - - The decoder must remain valid until the - message has been parsed. Passing `nullptr` - removes a previously installed decoder. - - @par Preconditions - `this->got_header() == true` and no body - octet has been parsed. - - @param dec The decoder to install. - */ - BOOST_BURL_DECL - void - set_decoder(decoder* dec) noexcept; - /** Set the maximum body size. Overrides @ref config::body_limit. The limit @@ -383,10 +335,9 @@ class parser /** Parse the message header. Returns as soon as the header is complete, - so that @ref set_decoder and @ref - set_body_limit can be called before any body - octet is parsed. Has no effect once @ref - got_header returns true. + so that @ref set_body_limit can be called + before any body octet is parsed. Has no + effect once @ref got_header returns true. @par Preconditions @ref start has been called. @@ -426,8 +377,8 @@ class parser /** Copy body octets into caller-supplied memory. - When a decoder is installed, it writes its - output into `buffers` directly. Parses the + When the body is decoded, the decoder writes + its output into `buffers` directly. Parses the header first if @ref got_header returns false. @@ -442,10 +393,10 @@ class parser @return The number of octets written. */ - BOOST_BURL_DECL + template std::size_t read_some( - std::span buffers, + MB const& buffers, system::error_code& ec); /** Return available body octets in place. @@ -547,29 +498,40 @@ class parser burl::request_head_base const& get_request() const; + BOOST_BURL_DECL + void + set_decoder( + std::unique_ptr dec) noexcept; + private: struct chunk_fn; std::error_code - need_more() const noexcept; + need_more_() const noexcept; std::size_t - trailer_extent() const noexcept; + trailer_extent_() const noexcept; std::error_code - walk_chunks(chunk_fn f, bool dry = false); + walk_chunks_(chunk_fn f, bool dry = false); std::error_code - flatten_chunks(); + flatten_chunks_(); + + BOOST_BURL_DECL + std::size_t + read_some_( + capy::mutable_buffer dest, + system::error_code& ec); std::size_t - decode_some( - std::span buffers, + decode_some_( + capy::mutable_buffer dest, system::error_code& ec); std::unique_ptr buf_; head_parser hp_; - decoder * dec_ = nullptr; + std::unique_ptr dec_; detail::circular_buffer in_; detail::circular_buffer out_; std::uint64_t rem_ = 0; @@ -577,6 +539,7 @@ class parser std::uint64_t limit_rem_ = 0; std::error_code dec_err_; http::payload payload_ = http::payload::none; + bool decode_ : 1 = false; bool head_ : 1 = false; bool started_ : 1 = false; bool got_header_ : 1 = false; @@ -586,6 +549,37 @@ class parser bool eof_ : 1 = false; }; +//------------------------------------------------ + +template +std::size_t +parser:: +read_some( + MB const& buffers, + system::error_code& ec) +{ + std::size_t n = 0; + auto const end = capy::end(buffers); + for(auto it = capy::begin(buffers); it != end; ++it) + { + capy::mutable_buffer const b(*it); + if(b.size() == 0) + continue; + auto const m = read_some_(b, ec); + if(ec) + { + // reported on the next call + if(n != 0) + ec = {}; + break; + } + n += m; + if(m < b.size()) + break; + } + return n; +} + } // namespace burl } // namespace boost diff --git a/include/boost/burl/response.hpp b/include/boost/burl/response.hpp index 38813db..9d8b080 100644 --- a/include/boost/burl/response.hpp +++ b/include/boost/burl/response.hpp @@ -83,7 +83,6 @@ class response urls::url url_; detail::pooled_connection conn_; response_parser parser_; - std::unique_ptr decoder_; std::optional deadline_; BOOST_BURL_DECL @@ -91,7 +90,6 @@ class response urls::url url, detail::pooled_connection conn, response_parser parser, - std::unique_ptr dec, std::optional deadline); public: diff --git a/include/boost/burl/test/response_factory.hpp b/include/boost/burl/test/response_factory.hpp index e706dba..d4bf377 100644 --- a/include/boost/burl/test/response_factory.hpp +++ b/include/boost/burl/test/response_factory.hpp @@ -280,7 +280,7 @@ class response_factory }()); return response{ - url_, std::move(conn), std::move(parser), {}, deadline_ }; + url_, std::move(conn), std::move(parser), deadline_ }; } private: diff --git a/src/client.cpp b/src/client.cpp index e6c7890..91e6e2f 100644 --- a/src/client.cpp +++ b/src/client.cpp @@ -14,18 +14,20 @@ #include "detail/base64.hpp" #include "detail/can_reuse_conn.hpp" #include "detail/connection_pool.hpp" -#include "detail/content_coding.hpp" -#include "detail/decoders.hpp" #include "detail/drain_body.hpp" #include "detail/redirect.hpp" #include #include +#include #include #include #include #include #include +#include +#include +#include #include #include @@ -95,16 +97,14 @@ client::client( std::make_shared( exec, std::move(tls_ctx), cfg)) { - // Disable codings whose decoder was not compiled in. -#ifndef BOOST_BURL_HAS_BROTLI - config_.brotli = false; -#endif -#ifndef BOOST_BURL_HAS_ZLIB - config_.deflate = config_.gzip = false; -#endif -#ifndef BOOST_BURL_HAS_ZSTD - config_.zstd = false; -#endif + // Disable codings whose decoder service is unavailable. + auto const& ctx = capy::get_system_context(); + if(!ctx.has_service()) + config_.brotli = false; + if(!ctx.has_service()) + config_.deflate = config_.gzip = false; + if(!ctx.has_service()) + config_.zstd = false; } void @@ -226,7 +226,8 @@ client::execute_impl( .hdr_limits = {}, .in_buffer = config_.response_inplace_buffer, .dec_buffer = config_.response_inplace_buffer, - .body_limit = config_.response_body_limit + .body_limit = config_.response_body_limit, + .decode = auto_decode }); serializer sr({}); @@ -305,18 +306,9 @@ client::execute_impl( ? std::error_code(status_int, burl_category()) : std::error_code(); - std::unique_ptr dec; - if(auto_decode && !is_head) - { - dec = detail::make_decoder( - detail::content_coding(parser.get())); - parser.set_decoder(dec.get()); - } - co_return { ec, - response{ url, std::move(conn), std::move(parser), - std::move(dec), deadline } + response{ url, std::move(conn), std::move(parser), deadline } }; } diff --git a/src/detail/circular_buffer.cpp b/src/detail/circular_buffer.cpp index de05976..df18e3e 100644 --- a/src/detail/circular_buffer.cpp +++ b/src/detail/circular_buffer.cpp @@ -83,6 +83,16 @@ prepare() const noexcept { ptr, free - (cap - w) } } }; } +capy::mutable_buffer +circular_buffer:: +prepare_one() const noexcept +{ + std::size_t w = pos + len; + if(w >= cap) + w -= cap; + return { ptr + w, clamp(cap - len, cap - w) }; +} + void circular_buffer:: commit(std::size_t n) noexcept diff --git a/src/detail/decoders.cpp b/src/detail/decoders.cpp index 0b04587..4962543 100644 --- a/src/detail/decoders.cpp +++ b/src/detail/decoders.cpp @@ -9,27 +9,21 @@ #include "decoders.hpp" -#include - #include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include #include -#ifdef BOOST_BURL_HAS_ZLIB -#include -#endif - -#ifdef BOOST_BURL_HAS_BROTLI -#include -#endif - -#ifdef BOOST_BURL_HAS_ZSTD -#include -#endif - namespace boost { namespace burl @@ -40,24 +34,27 @@ namespace detail namespace { -#ifdef BOOST_BURL_HAS_ZLIB class zlib_decoder final - : public parser::decoder + : public decoder { - z_stream strm_ = {}; + http::zlib::inflate_service& svc_; + http::zlib::stream strm_ = {}; public: // window_bits: 15 for zlib/deflate, 15 + 16 for gzip. - explicit - zlib_decoder(int window_bits) + zlib_decoder( + http::zlib::inflate_service& svc, + int window_bits) + : svc_(svc) { - if(inflateInit2(&strm_, window_bits) != Z_OK) + if(svc_.init2(strm_, window_bits) != + static_cast(http::zlib::error::ok)) throw std::bad_alloc(); } ~zlib_decoder() override { - inflateEnd(&strm_); + svc_.inflate_end(strm_); } result @@ -73,15 +70,19 @@ class zlib_decoder final static_cast(out.data()); strm_.avail_out = saturate(out.size()); - auto const rs = ::inflate( - &strm_, more ? Z_NO_FLUSH : Z_FINISH); + auto const rs = static_cast( + svc_.inflate( + strm_, + more ? http::zlib::no_flush + : http::zlib::finish)); auto const ec = [&]() -> std::error_code { - if(rs == Z_STREAM_END) + if(rs == http::zlib::error::stream_end) return capy::error::eof; - if(rs != Z_OK && rs != Z_BUF_ERROR) - return error::decode_error; + if(rs != http::zlib::error::ok && + rs != http::zlib::error::buf_err) + return rs; return {}; }(); @@ -102,17 +103,18 @@ class zlib_decoder final return static_cast(n); } }; -#endif // BOOST_BURL_HAS_ZLIB -#ifdef BOOST_BURL_HAS_BROTLI class brotli_decoder final - : public parser::decoder + : public decoder { - BrotliDecoderState* state_; + http::brotli::decode_service& svc_; + http::brotli::decoder_state* state_; public: - brotli_decoder() - : state_(BrotliDecoderCreateInstance( + explicit + brotli_decoder(http::brotli::decode_service& svc) + : svc_(svc) + , state_(svc.create_instance( nullptr, nullptr, nullptr)) { if(!state_) @@ -121,7 +123,7 @@ class brotli_decoder final ~brotli_decoder() override { - BrotliDecoderDestroyInstance(state_); + svc_.destroy_instance(state_); } result @@ -135,7 +137,7 @@ class brotli_decoder final auto* next_out = static_cast(out.data()); auto available_out = out.size(); - auto const rs = BrotliDecoderDecompressStream( + auto const rs = svc_.decompress_stream( state_, &available_in, &next_in, @@ -145,10 +147,10 @@ class brotli_decoder final auto const ec = [&]() -> std::error_code { - if(BrotliDecoderIsFinished(state_)) + if(svc_.is_finished(state_)) return capy::error::eof; - if(rs == BROTLI_DECODER_RESULT_ERROR) - return error::decode_error; + if(rs == http::brotli::decoder_result::error) + return svc_.get_error_code(state_); return {}; }(); @@ -158,46 +160,56 @@ class brotli_decoder final .ec = ec }; } }; -#endif // BOOST_BURL_HAS_BROTLI -#ifdef BOOST_BURL_HAS_ZSTD class zstd_decoder final - : public parser::decoder + : public decoder { - ZSTD_DStream* strm_; + http::zstd::decompress_service& svc_; + http::zstd::dctx* ctx_; + // A payload is a sequence of frames (RFC 8878 + // Section 3.1.1); it ends at a frame boundary. + bool at_boundary_ = true; public: - zstd_decoder() - : strm_(ZSTD_createDStream()) + explicit + zstd_decoder(http::zstd::decompress_service& svc) + : svc_(svc) + , ctx_(svc.create_dctx()) { - if(!strm_) + if(!ctx_) throw std::bad_alloc(); - ZSTD_initDStream(strm_); } ~zstd_decoder() override { - ZSTD_freeDStream(strm_); + svc_.free_dctx(ctx_); } result process( capy::mutable_buffer out, capy::const_buffer in, - bool) override + bool more) override { - ZSTD_inBuffer in_buf{ in.data(), in.size(), 0 }; - ZSTD_outBuffer out_buf{ out.data(), out.size(), 0 }; + http::zstd::in_buffer in_buf{ in.data(), in.size(), 0 }; + http::zstd::out_buffer out_buf{ out.data(), out.size(), 0 }; auto const rs = - ZSTD_decompressStream(strm_, &out_buf, &in_buf); + svc_.decompress_stream(ctx_, out_buf, in_buf); auto const ec = [&]() -> std::error_code { + if(svc_.is_error(rs)) + return svc_.get_error_code(rs); + // zero marks the end of a frame; decoding + // resumes on the next one, skippable + // frames included if(rs == 0) + at_boundary_ = true; + else if(in_buf.pos != 0) + at_boundary_ = false; + if(!more && in_buf.pos == in_buf.size && at_boundary_) return capy::error::eof; - if(ZSTD_isError(rs)) - return error::decode_error; return {}; }(); @@ -207,29 +219,35 @@ class zstd_decoder final .ec = ec }; } }; -#endif // BOOST_BURL_HAS_ZSTD } // namespace -std::unique_ptr +std::unique_ptr make_decoder(http::content_coding coding) { + auto const& ctx = capy::get_system_context(); switch(coding) { -#ifdef BOOST_BURL_HAS_ZLIB case http::content_coding::deflate: - return std::make_unique(15); + if(auto* svc = ctx.find_service< + http::zlib::inflate_service>()) + return std::make_unique(*svc, 15); + break; case http::content_coding::gzip: - return std::make_unique(15 + 16); -#endif -#ifdef BOOST_BURL_HAS_BROTLI + if(auto* svc = ctx.find_service< + http::zlib::inflate_service>()) + return std::make_unique(*svc, 15 + 16); + break; case http::content_coding::br: - return std::make_unique(); -#endif -#ifdef BOOST_BURL_HAS_ZSTD + if(auto* svc = ctx.find_service< + http::brotli::decode_service>()) + return std::make_unique(*svc); + break; case http::content_coding::zstd: - return std::make_unique(); -#endif + if(auto* svc = ctx.find_service< + http::zstd::decompress_service>()) + return std::make_unique(*svc); + break; default: break; } diff --git a/src/detail/decoders.hpp b/src/detail/decoders.hpp index 4502c54..879577f 100644 --- a/src/detail/decoders.hpp +++ b/src/detail/decoders.hpp @@ -10,7 +10,7 @@ #ifndef BOOST_BURL_SRC_DETAIL_DECODERS_HPP #define BOOST_BURL_SRC_DETAIL_DECODERS_HPP -#include +#include #include @@ -23,7 +23,7 @@ namespace burl namespace detail { -std::unique_ptr +std::unique_ptr make_decoder(http::content_coding coding); } // namespace detail diff --git a/src/detail/send_file.cpp b/src/detail/send_file.cpp index 7b11dab..98ea3ed 100644 --- a/src/detail/send_file.cpp +++ b/src/detail/send_file.cpp @@ -35,15 +35,8 @@ send_file( bool call_eof) { corosio::stream_file f(co_await capy::this_coro::executor); - // TODO: switch to a non-throwing open() overload once available. - try - { - f.open(path, corosio::file_base::read_only); - } - catch(std::system_error const& e) - { - co_return { e.code() }; - } + if(auto ec = f.open(path, corosio::file_base::read_only)) + co_return { ec }; auto remaining = size; while(remaining > 0) diff --git a/src/error.cpp b/src/error.cpp index ae39109..ec3f798 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -52,8 +52,6 @@ error_category::message(int ev) const return "unsupported proxy protocol version"; case error::body_size_mismatch: return "message body size did not match content length"; - case error::decode_error: - return "response body could not be decoded"; default: return "unknown error"; } diff --git a/src/file.cpp b/src/file.cpp index 13287b3..51b045c 100644 --- a/src/file.cpp +++ b/src/file.cpp @@ -81,17 +81,12 @@ tag_invoke( response& resp, std::filesystem::path dest) { + using enum corosio::file_base::flags; + corosio::stream_file f(co_await capy::this_coro::executor); - // TODO: switch to a non-throwing open() overload once available. - try - { - using enum corosio::file_base::flags; - f.open(dest, write_only | create | exclusive); - } - catch(std::system_error const& e) - { - co_return { e.code(), {} }; - } + + if(auto ec = f.open(dest, write_only | create | exclusive)) + co_return { ec, {} }; auto src = resp.as_buffer_source(); auto [ec, n] = co_await http::push_to(src, f); diff --git a/src/parser.cpp b/src/parser.cpp index c289a55..6d20ad5 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -11,12 +11,13 @@ #include +#include "detail/content_coding.hpp" +#include "detail/decoders.hpp" #include "detail/grammar.hpp" #include "detail/util.hpp" #include #include -#include #include #include #include @@ -33,9 +34,11 @@ namespace burl { using detail::clamp; +using detail::content_coding; +using detail::distance; +using detail::make_decoder; using detail::parse_field; using detail::parse_limited; -using detail::distance; using http::condition::need_more_input; using http::error::bad_payload; @@ -249,16 +252,6 @@ prefix(auto buf, std::size_t n) noexcept return { buf.data(), clamp(buf.size(), n) }; }; -auto -first(auto const& bs) noexcept - -> decltype(*begin(bs)) -{ - for(auto b : bs) - if(b.size() != 0) - return b; - return {}; -} - } // namespace struct parser::chunk_fn @@ -292,6 +285,7 @@ parser( config const& cfg, bool is_req) : body_limit_(cfg.body_limit) + , decode_(cfg.decode) { auto const h_cap = head_parser::bytes_needed( cfg.hdr_limits, cfg.in_buffer); @@ -321,7 +315,7 @@ bool parser:: has_buffered_data() const noexcept { - return in_.size() > rem_ + trailer_extent(); + return in_.size() > rem_ + trailer_extent_(); } std::array @@ -333,7 +327,7 @@ buffered_data() const noexcept std::error_code parser:: -need_more() const noexcept +need_more_() const noexcept { if(eof_) return incomplete; @@ -344,7 +338,7 @@ need_more() const noexcept std::size_t parser:: -trailer_extent() const noexcept +trailer_extent_() const noexcept { if(!fin_chunk_) return 0; @@ -362,11 +356,11 @@ start(bool head) BOOST_ASSERT(!started_ || got_body_); in_.consume( - clamp(rem_) + trailer_extent()); + clamp(rem_) + trailer_extent_()); hp_.reset( in_.linearize(buf_.get())); - dec_ = nullptr; + dec_.reset(); rem_ = 0; limit_rem_ = body_limit_; dec_err_ = {}; @@ -452,14 +446,14 @@ commit_direct(std::size_t n) noexcept std::error_code parser:: -walk_chunks(chunk_fn f, bool dry) +walk_chunks_(chunk_fn f, bool dry) { chained_sequence cs = in_.data(); std::uint64_t size = rem_; if(fin_chunk_) { - // from flatten_chunks + // from flatten_chunks_ auto const b = in_.first(clamp(rem_)); auto const [ec, n] = f(b, true); if(!dry) @@ -522,7 +516,7 @@ walk_chunks(chunk_fn f, bool dry) std::error_code parser:: -flatten_chunks() +flatten_chunks_() { if(fin_chunk_) return {}; @@ -647,6 +641,9 @@ parse_header(system::error_code& ec) break; } + if(decode_ && payload_ != payload::none) + dec_ = make_decoder(content_coding(h)); + if(!got_body_) { in_.slide(buf_.get()); @@ -658,9 +655,10 @@ parse_header(system::error_code& ec) void parser:: -set_decoder(decoder* dec) noexcept +set_decoder( + std::unique_ptr dec) noexcept { - dec_ = dec; + dec_ = std::move(dec); } void @@ -689,8 +687,8 @@ flatten_body(system::error_code& ec) ec = in_place_overflow; break; } - auto pb = out_.prepare(); - auto const n = decode_some(pb, ec); + auto const n = decode_some_( + out_.prepare_one(), ec); out_.commit(n); if(ec == capy::cond::eof) { @@ -717,12 +715,12 @@ flatten_body(system::error_code& ec) } if(fin_chunk_) break; - if(auto fec = flatten_chunks(); fec) + if(auto fec = flatten_chunks_(); fec) { if(fec != need_more_input) ec = fec; else - ec = need_more(); + ec = need_more_(); break; } } @@ -733,7 +731,7 @@ flatten_body(system::error_code& ec) if(rem_ > limit_rem_) ec = body_too_large; else if(!got_body_) - ec = need_more(); + ec = need_more_(); return { in_.linearize(in_.ptr), clamp(rem_, in_.len) }; } case payload::to_eof: @@ -741,7 +739,7 @@ flatten_body(system::error_code& ec) if(in_.size() > limit_rem_) ec = body_too_large; else if(!got_body_) - ec = need_more(); + ec = need_more_(); return { in_.linearize(in_.ptr), in_.len }; } default: @@ -765,14 +763,13 @@ get_request() const std::size_t parser:: -decode_some( - std::span buffers, +decode_some_( + capy::mutable_buffer dest, system::error_code& ec) { - if(capy::buffer_empty(buffers)) + if(dest.size() == 0) return 0; - auto outbufs = capy::consuming_buffers(buffers); std::size_t prod = 0; auto decode = [&](capy::const_buffer in, bool last) @@ -790,17 +787,14 @@ decode_some( std::size_t cons = 0; for(;;) { - auto const out = first(outbufs.data()); - if(out.size() == 0) - return { std::error_code(), cons }; auto const lim = clamp(limit_rem_); auto const r = dec_->process( - prefix(out, lim), in, !last); + prefix(dest, lim), in, !last); + dest += r.produced; in += r.consumed; cons += r.consumed; prod += r.produced; limit_rem_ -= r.produced; - outbufs.consume(r.produced); if(r.ec) { dec_err_ = r.ec; @@ -810,10 +804,10 @@ decode_some( { if(lim == 0) return { body_too_large, cons }; - dec_err_ = error::decode_error; + dec_err_ = bad_payload; return { std::error_code(), cons }; } - if(in.size() == 0) + if(dest.size() == 0 || in.size() == 0) return { std::error_code(), cons }; } }; @@ -822,13 +816,13 @@ decode_some( { case payload::chunked: { - auto const wec = walk_chunks(decode); + auto const wec = walk_chunks_(decode); if(prod != 0) return prod; if(wec != need_more_input) ec = wec; else - ec = need_more(); + ec = need_more_(); return 0; } case payload::size: @@ -839,7 +833,7 @@ decode_some( auto const in = in_.first(clamp(rem_)); if(in.size() == 0 && !got_body_) { - ec = need_more(); + ec = need_more_(); return 0; } auto [dec_ec, cons] = decode( @@ -865,8 +859,8 @@ decode_some( std::size_t parser:: -read_some( - std::span buffers, +read_some_( + capy::mutable_buffer dest, system::error_code& ec) { parse_header(ec); @@ -878,17 +872,17 @@ read_some( if(!out_.empty()) { auto const n = capy::buffer_copy( - buffers, out_.data()); + dest, out_.data()); out_.consume(n); return n; } - return decode_some(buffers, ec); + return decode_some_(dest, ec); } auto copy = [&](std::size_t at_most) { auto const n = capy::buffer_copy( - buffers, in_.data(), at_most); + dest, in_.data(), at_most); in_.consume(n); rem_ -= n; limit_rem_ -= n; @@ -901,17 +895,15 @@ read_some( { std::size_t read = 0; std::size_t lim = clamp(limit_rem_); - auto outbufs = capy::consuming_buffers(buffers); - auto const wec = walk_chunks( + auto const wec = walk_chunks_( [&](capy::const_buffer b, bool) -> capy::io_result { auto const take = clamp(b.size(), lim); lim -= take; - auto const n = capy::buffer_copy( - outbufs.data(), b, take); + auto const n = capy::buffer_copy(dest, b, take); + dest += n; read += n; - outbufs.consume(n); if(take < b.size()) return { body_too_large, n }; return { std::error_code(), n }; @@ -920,7 +912,7 @@ read_some( return read; if(wec == need_more_input) { - ec = need_more(); + ec = need_more_(); return 0; } if(wec) @@ -949,7 +941,7 @@ read_some( ec = capy::error::eof; return 0; } - ec = need_more(); + ec = need_more_(); return 0; } case payload::to_eof: @@ -968,7 +960,7 @@ read_some( ec = capy::error::eof; return 0; } - ec = need_more(); + ec = need_more_(); return 0; } default: @@ -993,8 +985,8 @@ pull( { if(!out_.empty()) return collect(dest, out_.data()); - auto pb = out_.prepare(); - auto const n = decode_some(pb, ec); + auto const n = decode_some_( + out_.prepare_one(), ec); out_.commit(n); if(ec && n == 0) return {}; @@ -1008,7 +1000,7 @@ pull( { std::size_t n = 0; std::size_t lim = clamp(limit_rem_); - auto const wec = walk_chunks( + auto const wec = walk_chunks_( [&](capy::const_buffer b, bool) -> capy::io_result { @@ -1036,7 +1028,7 @@ pull( ec = wec; return {}; } - ec = need_more(); + ec = need_more_(); return {}; } case payload::size: @@ -1056,7 +1048,7 @@ pull( ec = capy::error::eof; return {}; } - ec = need_more(); + ec = need_more_(); return {}; } case payload::to_eof: @@ -1075,7 +1067,7 @@ pull( ec = capy::error::eof; return {}; } - ec = need_more(); + ec = need_more_(); return {}; } default: @@ -1096,7 +1088,7 @@ consume(std::size_t n) noexcept switch(payload_) { case payload::chunked: - walk_chunks( + walk_chunks_( [&](capy::const_buffer b, bool) -> capy::io_result { diff --git a/src/response.cpp b/src/response.cpp index a1f11cf..e00a7fc 100644 --- a/src/response.cpp +++ b/src/response.cpp @@ -26,12 +26,10 @@ response::response( urls::url url, detail::pooled_connection conn, response_parser parser, - std::unique_ptr dec, std::optional deadline) : url_(std::move(url)) , conn_(std::move(conn)) , parser_(std::move(parser)) - , decoder_(std::move(dec)) , deadline_(deadline) { } @@ -40,7 +38,6 @@ response::response(response&& other) noexcept : url_(std::move(other.url_)) , conn_(std::move(other.conn_)) , parser_(std::move(other.parser_)) - , decoder_(std::move(other.decoder_)) , deadline_(other.deadline_) { } @@ -55,7 +52,6 @@ response::operator=(response&& other) noexcept url_ = std::move(other.url_); conn_ = std::move(other.conn_); parser_ = std::move(other.parser_); - decoder_ = std::move(other.decoder_); deadline_ = other.deadline_; } return *this; diff --git a/src/serializer.cpp b/src/serializer.cpp index 24c8014..d4c58c8 100644 --- a/src/serializer.cpp +++ b/src/serializer.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -246,7 +247,6 @@ decide_framing_(std::uint64_t total) noexcept return; if(payload_ == payload::chunked || payload_ == payload::to_eof) { - msg_->erase(http::field::transfer_encoding); msg_->set_content_length(total); payload_ = payload::size; owed_ = total; diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index ee418b6..5c1ca8b 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -22,10 +22,9 @@ add_executable(boost_burl_tests ${PFILES}) target_link_libraries(boost_burl_tests PRIVATE boost_capy_test_suite_main Boost::burl - $ - $ - $ - $) + $ + $ + $) target_include_directories(boost_burl_tests PRIVATE . ../../) diff --git a/test/unit/Jamfile b/test/unit/Jamfile index 5b3ee72..70e2f62 100644 --- a/test/unit/Jamfile +++ b/test/unit/Jamfile @@ -10,13 +10,13 @@ import testing ; import ac ; -using brotli ; - project : requirements $(c20-requires) /boost/burl//boost_burl - [ ac.check-library /brotli//brotlienc : /brotli//brotlienc : ] + [ ac.check-library /boost/http//boost_http_zlib : /boost/http//boost_http_zlib : ] + [ ac.check-library /boost/http//boost_http_brotli : /boost/http//boost_http_brotli : ] + [ ac.check-library /boost/http//boost_http_zstd : /boost/http//boost_http_zstd : ] ../../../capy/extra/test_suite/test_main.cpp ../../../capy/extra/test_suite/test_suite.cpp . diff --git a/test/unit/client.cpp b/test/unit/client.cpp index ce03c98..94d17f0 100644 --- a/test/unit/client.cpp +++ b/test/unit/client.cpp @@ -22,6 +22,9 @@ #include #include #include +#include +#include +#include #include #include "scripted_net.hpp" @@ -697,7 +700,12 @@ class client_test void testGzipDecode() { -#ifdef BOOST_BURL_HAS_ZLIB +#ifdef BOOST_HTTP_HAS_ZLIB + if(!capy::get_system_context() + .has_service()) + http::zlib::install_inflate_service( + capy::get_system_context()); + scripted_net net; net.run([&]() -> capy::task<> { @@ -742,7 +750,12 @@ class client_test void testBrotliDecode() { -#ifdef BOOST_BURL_HAS_BROTLI +#ifdef BOOST_HTTP_HAS_BROTLI + if(!capy::get_system_context() + .has_service()) + http::brotli::install_decode_service( + capy::get_system_context()); + scripted_net net; net.run([&]() -> capy::task<> { @@ -785,7 +798,12 @@ class client_test void testZstdDecode() { -#ifdef BOOST_BURL_HAS_ZSTD +#ifdef BOOST_HTTP_HAS_ZSTD + if(!capy::get_system_context() + .has_service()) + http::zstd::install_decompress_service( + capy::get_system_context()); + scripted_net net; net.run([&]() -> capy::task<> { @@ -825,6 +843,51 @@ class client_test #endif } + void + testZstdDecodeTruncated() + { +#ifdef BOOST_HTTP_HAS_ZSTD + if(!capy::get_system_context() + .has_service()) + http::zstd::install_decompress_service( + capy::get_system_context()); + + scripted_net net; + net.run([&]() -> capy::task<> + { + // zstd("hello world") minus its last three + // octets; the framing is intact, the frame + // is not. + static char const zs[] = + "\x28\xb5\x2f\xfd\x20\x0b\x59\x00\x00\x68\x65\x6c" + "\x6c\x6f\x20\x77\x6f"; + auto const body = std::string(zs, sizeof(zs) - 1); + + net.scripts = { std::string( + "HTTP/1.1 200 OK\r\n" + "Content-Encoding: zstd\r\n" + "Content-Length: " + + std::to_string(body.size()) + + "\r\n\r\n" + body) }; + + auto cfg = net.config(); + cfg.zstd = true; + client c( + co_await capy::this_coro::executor, + corosio::tls_context(), + cfg); + + auto [ec, r] = co_await c + .get("http://example.com/z") + .send(); + BOOST_TEST(!ec); + + auto [ec2, text] = co_await r.try_as_view(); + BOOST_TEST(ec2 == http::error::bad_payload); + }); +#endif + } + void testTimeoutHeader() { @@ -1067,6 +1130,7 @@ class client_test testGzipDecode(); testBrotliDecode(); testZstdDecode(); + testZstdDecodeTruncated(); testTimeoutHeader(); testTimeoutBody(); testTimeoutOverride(); diff --git a/test/unit/detail/circular_buffer.cpp b/test/unit/detail/circular_buffer.cpp index 00e5a53..b19e645 100644 --- a/test/unit/detail/circular_buffer.cpp +++ b/test/unit/detail/circular_buffer.cpp @@ -131,6 +131,60 @@ class circular_buffer_test BOOST_TEST(cb.empty()); } + void + testPrepare() + { + char store[8]; + circular_buffer cb{ store, sizeof(store) }; + + // prepare_one is always the first prepare region + auto check = [&]( + std::size_t off, std::size_t n0, std::size_t n1) + { + auto const pb = cb.prepare(); + BOOST_TEST_EQ(pb[0].data(), store + off); + BOOST_TEST_EQ(pb[0].size(), n0); + BOOST_TEST_EQ(pb[1].size(), n1); + if(n1 != 0) + BOOST_TEST_EQ(pb[1].data(), store); + auto const mb = cb.prepare_one(); + BOOST_TEST_EQ(mb.data(), pb[0].data()); + BOOST_TEST_EQ(mb.size(), pb[0].size()); + }; + + // empty: the whole store, in one region + check(0, 8, 0); + + // contents at the front: the free region is + // the contiguous tail + put(cb, "abcde"); + check(5, 3, 0); + + // contents in the middle: the free region is split + // around the end of the store + cb.consume(2); + check(5, 3, 2); + + // the write position lands exactly on the end: + // the free region is contiguous at the front + put(cb, "fgh"); + BOOST_TEST(!cb.wrapped()); + check(0, 2, 0); + + // wrapped contents: the free region is contiguous + // between the write and read positions + put(cb, "X"); + BOOST_TEST(cb.wrapped()); + check(1, 1, 0); + + // full: nothing to prepare + put(cb, "Y"); + BOOST_TEST(cb.full()); + check(2, 0, 0); + BOOST_TEST_EQ(cb.prepare_one().size(), 0); + BOOST_TEST(str(cb.data()) == "cdefghXY"); + } + void testReset() { @@ -351,6 +405,7 @@ class circular_buffer_test testEmpty(); testFillAndDrain(); testWrapAround(); + testPrepare(); testReset(); testShedAndSlide(); testLinearizeEmpty(); diff --git a/test/unit/detail/connection_pool.cpp b/test/unit/detail/connection_pool.cpp index 7d91a32..bc72d97 100644 --- a/test/unit/detail/connection_pool.cpp +++ b/test/unit/detail/connection_pool.cpp @@ -87,7 +87,8 @@ class connection_pool_test explicit loopback_server(corosio::io_context& ioc) : ioc_{ ioc } { - acceptor_.open(); + if(auto ec = acceptor_.open()) + throw std::system_error(ec); acceptor_.set_option( corosio::socket_option::reuse_address(true)); if(auto ec = acceptor_.bind({})) diff --git a/test/unit/detail/decoders.cpp b/test/unit/detail/decoders.cpp index 7845145..75f90d6 100644 --- a/test/unit/detail/decoders.cpp +++ b/test/unit/detail/decoders.cpp @@ -14,6 +14,10 @@ #include #include +#include +#include +#include +#include #include "test_suite.hpp" @@ -22,18 +26,6 @@ #include #include -#ifdef BOOST_BURL_HAS_ZLIB -#include -#endif - -#ifdef BOOST_BURL_HAS_BROTLI -#include -#endif - -#ifdef BOOST_BURL_HAS_ZSTD -#include -#endif - namespace boost { namespace burl @@ -53,7 +45,7 @@ class decoders_test static decode_result run_decoder( - parser::decoder& dec, + decoder& dec, std::string_view input, std::size_t in_step, std::size_t out_step) @@ -102,49 +94,55 @@ class decoders_test return body; } -#ifdef BOOST_BURL_HAS_ZLIB +#ifdef BOOST_HTTP_HAS_ZLIB static std::string zlib_compress(std::string_view body, int window_bits) { - z_stream st = {}; - if(deflateInit2( - &st, - Z_DEFAULT_COMPRESSION, - Z_DEFLATED, + auto& svc = *capy::get_system_context() + .find_service(); + + http::zlib::stream st = {}; + if(svc.init2( + st, + http::zlib::default_compression, + http::zlib::deflated, window_bits, 8, - Z_DEFAULT_STRATEGY) != Z_OK) + http::zlib::default_strategy) != + static_cast(http::zlib::error::ok)) return {}; std::string out( - deflateBound( - &st, static_cast(body.size())), + svc.bound(st, static_cast(body.size())), '\0'); st.next_in = reinterpret_cast( const_cast(body.data())); st.avail_in = static_cast(body.size()); st.next_out = reinterpret_cast(out.data()); st.avail_out = static_cast(out.size()); - auto const rs = deflate(&st, Z_FINISH); + auto const rs = svc.deflate(st, http::zlib::finish); out.resize(out.size() - st.avail_out); - deflateEnd(&st); - if(rs != Z_STREAM_END) + svc.deflate_end(st); + if(rs != static_cast(http::zlib::error::stream_end)) return {}; return out; } #endif -#ifdef BOOST_BURL_HAS_BROTLI +#ifdef BOOST_HTTP_HAS_BROTLI static std::string brotli_compress(std::string_view body) { + auto& svc = *capy::get_system_context() + .find_service(); + std::size_t encoded_size = - BrotliEncoderMaxCompressedSize(body.size()) + 64; + svc.max_compressed_size(body.size()) + 64; std::string out(encoded_size, '\0'); - if(!BrotliEncoderCompress( - BROTLI_DEFAULT_QUALITY, - BROTLI_DEFAULT_WINDOW, - BROTLI_MODE_GENERIC, + if(!svc.compress( + http::brotli::default_quality, + http::brotli::default_window, + http::brotli::encoder_mode::generic, body.size(), reinterpret_cast(body.data()), &encoded_size, @@ -155,18 +153,21 @@ class decoders_test } #endif -#ifdef BOOST_BURL_HAS_ZSTD +#ifdef BOOST_HTTP_HAS_ZSTD static std::string zstd_compress(std::string_view body) { - std::string out(ZSTD_compressBound(body.size()), '\0'); - auto const n = ZSTD_compress( + auto& svc = *capy::get_system_context() + .find_service(); + + std::string out(svc.compress_bound(body.size()), '\0'); + auto const n = svc.compress( out.data(), out.size(), body.data(), body.size(), - ZSTD_CLEVEL_DEFAULT); - if(ZSTD_isError(n)) + svc.default_level()); + if(svc.is_error(n)) return {}; out.resize(n); return out; @@ -184,8 +185,8 @@ class decoders_test BOOST_TEST( make_decoder(http::content_coding::compress) == nullptr); - // Availability of each decoder follows the build configuration. -#ifdef BOOST_BURL_HAS_ZLIB + // Availability of each decoder follows the installed services. +#ifdef BOOST_HTTP_HAS_ZLIB BOOST_TEST( make_decoder(http::content_coding::deflate) != nullptr); BOOST_TEST( @@ -196,14 +197,14 @@ class decoders_test BOOST_TEST( make_decoder(http::content_coding::gzip) == nullptr); #endif -#ifdef BOOST_BURL_HAS_BROTLI +#ifdef BOOST_HTTP_HAS_BROTLI BOOST_TEST( make_decoder(http::content_coding::br) != nullptr); #else BOOST_TEST( make_decoder(http::content_coding::br) == nullptr); #endif -#ifdef BOOST_BURL_HAS_ZSTD +#ifdef BOOST_HTTP_HAS_ZSTD BOOST_TEST( make_decoder(http::content_coding::zstd) != nullptr); #else @@ -212,7 +213,7 @@ class decoders_test #endif } -#ifdef BOOST_BURL_HAS_ZLIB +#ifdef BOOST_HTTP_HAS_ZLIB void test_zlib_round_trip() { @@ -321,6 +322,24 @@ class decoders_test BOOST_TEST_EQ(r.leftover, 9u); } + void + test_zlib_second_member() + { + // A gzip payload is a single member. As with + // curl and the browsers, a second member is + // not decoded; the parser rejects it as + // trailing octets. + auto const first = make_body(200); + auto const second = zlib_compress(make_body(100), 15 + 16); + auto const input = zlib_compress(first, 15 + 16) + second; + + auto dec = make_decoder(http::content_coding::gzip); + auto const r = run_decoder(*dec, input, 3, 64); + BOOST_TEST(r.finished); + BOOST_TEST(r.body == first); + BOOST_TEST_EQ(r.leftover, second.size()); + } + void test_zlib_invalid() { @@ -331,11 +350,11 @@ class decoders_test auto const r = run_decoder( *dec, input, input.size(), 64); BOOST_TEST(!r.finished); - BOOST_TEST(r.ec == error::decode_error); + BOOST_TEST(r.ec == http::zlib::error::data_err); } -#endif // BOOST_BURL_HAS_ZLIB +#endif // BOOST_HTTP_HAS_ZLIB -#ifdef BOOST_BURL_HAS_BROTLI +#ifdef BOOST_HTTP_HAS_BROTLI void test_brotli_round_trip() { @@ -409,11 +428,11 @@ class decoders_test auto const r = run_decoder( *dec, input, input.size(), 64); BOOST_TEST(!r.finished); - BOOST_TEST(r.ec == error::decode_error); + BOOST_TEST(r.ec == http::brotli::error::format_padding_1); } -#endif // BOOST_BURL_HAS_BROTLI +#endif // BOOST_HTTP_HAS_BROTLI -#ifdef BOOST_BURL_HAS_ZSTD +#ifdef BOOST_HTTP_HAS_ZSTD void test_zstd_round_trip() { @@ -480,32 +499,216 @@ class decoders_test auto const r = run_decoder( *dec, input, input.size(), 64); BOOST_TEST(!r.finished); - BOOST_TEST(r.ec == error::decode_error); + BOOST_TEST(r.ec == http::zstd::error::prefix_unknown); + } + + static std::string + zstd_skippable_frame(std::string_view content) + { + // magic 0x184D2A50 and the content size, + // both little-endian (RFC 8878 Section 3.1.2) + std::string frame = "\x50\x2A\x4D\x18"; + auto const n = content.size(); + frame += static_cast(n & 0xFF); + frame += static_cast((n >> 8) & 0xFF); + frame += static_cast((n >> 16) & 0xFF); + frame += static_cast((n >> 24) & 0xFF); + frame += content; + return frame; + } + + void + test_zstd_multi_frame() + { + // concatenated frames decode to the + // concatenation of their contents + auto const first = make_body(200); + auto const second = make_body(300); + auto const third = make_body(10); + auto const compressed = zstd_compress(first) + + zstd_compress(second) + zstd_compress(third); + + // single pass + { + auto dec = make_decoder(http::content_coding::zstd); + auto const r = run_decoder( + *dec, compressed, compressed.size(), 1024); + BOOST_TEST(!r.ec); + BOOST_TEST(r.finished); + BOOST_TEST(r.body == first + second + third); + BOOST_TEST_EQ(r.leftover, 0u); + } + + // starved input and output + { + auto dec = make_decoder(http::content_coding::zstd); + auto const r = run_decoder(*dec, compressed, 3, 7); + BOOST_TEST(!r.ec); + BOOST_TEST(r.finished); + BOOST_TEST(r.body == first + second + third); + BOOST_TEST_EQ(r.leftover, 0u); + } + } + + void + test_zstd_skippable_frame() + { + auto const body = make_body(200); + + // leading, trailing, and in between + { + auto const compressed = + zstd_skippable_frame("pzstd") + + zstd_compress(body) + + zstd_skippable_frame({}) + + zstd_compress(body) + + zstd_skippable_frame("seek table"); + + auto dec = make_decoder(http::content_coding::zstd); + auto const r = run_decoder(*dec, compressed, 5, 64); + BOOST_TEST(!r.ec); + BOOST_TEST(r.finished); + BOOST_TEST(r.body == body + body); + BOOST_TEST_EQ(r.leftover, 0u); + } + + // nothing but a skippable frame is an empty body + { + auto const compressed = zstd_skippable_frame("meta"); + + auto dec = make_decoder(http::content_coding::zstd); + auto const r = run_decoder( + *dec, compressed, compressed.size(), 64); + BOOST_TEST(!r.ec); + BOOST_TEST(r.finished); + BOOST_TEST(r.body.empty()); + BOOST_TEST_EQ(r.leftover, 0u); + } + } + + void + test_zstd_frame_end_before_payload_end() + { + // The frame completes while more payload may + // follow; the end of the payload is signalled + // afterwards with no further octets. + auto const body = make_body(50); + auto const compressed = zstd_compress(body); + + auto dec = make_decoder(http::content_coding::zstd); + char buf[1024]; + auto res = dec->process( + capy::mutable_buffer(buf, sizeof(buf)), + capy::const_buffer(compressed.data(), compressed.size()), + true); + BOOST_TEST(!res.ec); + BOOST_TEST_EQ(res.consumed, compressed.size()); + BOOST_TEST(std::string_view(buf, res.produced) == body); + + res = dec->process( + capy::mutable_buffer(buf, sizeof(buf)), + capy::const_buffer(), + false); + BOOST_TEST(res.ec == capy::cond::eof); + BOOST_TEST_EQ(res.consumed, 0u); + BOOST_TEST_EQ(res.produced, 0u); + } + + void + test_zstd_truncated() + { + auto const body = make_body(200); + auto compressed = zstd_compress(body); + compressed.resize(compressed.size() - 3); + + // single pass + { + auto dec = make_decoder(http::content_coding::zstd); + auto const r = run_decoder( + *dec, compressed, compressed.size(), 1024); + BOOST_TEST(!r.finished); + BOOST_TEST(!r.ec); + BOOST_TEST_EQ(r.leftover, 0u); + } + + // starved input and output + { + auto dec = make_decoder(http::content_coding::zstd); + auto const r = run_decoder(*dec, compressed, 3, 7); + BOOST_TEST(!r.finished); + BOOST_TEST(!r.ec); + BOOST_TEST_EQ(r.leftover, 0u); + } + + // cut inside the second frame + { + auto input = zstd_compress(body) + compressed; + + auto dec = make_decoder(http::content_coding::zstd); + auto const r = run_decoder(*dec, input, input.size(), 1024); + BOOST_TEST(!r.finished); + BOOST_TEST(!r.ec); + BOOST_TEST_EQ(r.leftover, 0u); + BOOST_TEST(r.body == body); + } + } + + void + test_zstd_trailing_garbage() + { + // octets after the last frame are not a frame + auto const body = make_body(200); + auto const input = zstd_compress(body) + "PIPELINED"; + + auto dec = make_decoder(http::content_coding::zstd); + auto const r = run_decoder(*dec, input, input.size(), 1024); + BOOST_TEST(!r.finished); + BOOST_TEST(r.ec == http::zstd::error::prefix_unknown); + BOOST_TEST(r.body == body); } -#endif // BOOST_BURL_HAS_ZSTD +#endif // BOOST_HTTP_HAS_ZSTD void run() { -#ifdef BOOST_BURL_HAS_ZLIB + [[maybe_unused]] auto& ctx = capy::get_system_context(); +#ifdef BOOST_HTTP_HAS_ZLIB + if(!ctx.has_service()) + http::zlib::install_inflate_service(ctx); + if(!ctx.has_service()) + http::zlib::install_deflate_service(ctx); test_zlib_round_trip(); test_zlib_large(); test_zlib_empty_body(); test_zlib_finished_latch(); test_zlib_trailing_garbage(); + test_zlib_second_member(); test_zlib_invalid(); #endif -#ifdef BOOST_BURL_HAS_BROTLI +#ifdef BOOST_HTTP_HAS_BROTLI + if(!ctx.has_service()) + http::brotli::install_decode_service(ctx); + if(!ctx.has_service()) + http::brotli::install_encode_service(ctx); test_brotli_round_trip(); test_brotli_large(); test_brotli_finished_latch(); test_brotli_invalid(); #endif -#ifdef BOOST_BURL_HAS_ZSTD +#ifdef BOOST_HTTP_HAS_ZSTD + if(!ctx.has_service()) + http::zstd::install_decompress_service(ctx); + if(!ctx.has_service()) + http::zstd::install_compress_service(ctx); test_zstd_round_trip(); test_zstd_large(); test_zstd_empty_body(); test_zstd_invalid(); + test_zstd_multi_frame(); + test_zstd_skippable_frame(); + test_zstd_frame_end_before_payload_end(); + test_zstd_truncated(); + test_zstd_trailing_garbage(); #endif test_make_decoder(); } diff --git a/test/unit/parser.cpp b/test/unit/parser.cpp index ddf092d..cd545f8 100644 --- a/test/unit/parser.cpp +++ b/test/unit/parser.cpp @@ -16,13 +16,14 @@ #include #include -#include #include #include #include #include #include +#include #include +#include #include #include @@ -59,6 +60,34 @@ struct test_parser : parser { } + // Install a test decoder, which the test keeps + // owning so that it can inspect it afterwards. + template + void + set_decoder(D& dec) + { + struct forwarder : detail::decoder + { + D& d; + + explicit + forwarder(D& d_) noexcept + : d(d_) + { + } + + result + process( + capy::mutable_buffer out, + capy::const_buffer in, + bool more) override + { + return d.process(out, in, more); + } + }; + parser::set_decoder(std::make_unique(dec)); + } + //-------------------------------------------- // // the wire @@ -174,20 +203,17 @@ struct test_parser : parser if(auto ec = read_header(); ec) return { ec, 0 }; - capy::buffer_param bp(buffers); - for(;;) { system::error_code ec; - auto const n = parser::read_some(bp.data(), ec); + auto const n = parser::read_some(buffers, ec); if(ec != http::error::need_data) return { ec, n }; if(auto const lim = direct_capacity(); lim != 0) { - auto const mbs = bp.data(); auto const rn = direct_read( - capy::buffer_slice(mbs, 0, lim)); + capy::buffer_slice(buffers, 0, lim)); if(rn != 0) return { std::error_code(), rn }; continue; @@ -197,25 +223,6 @@ struct test_parser : parser } } - // hand the parser the span exactly as given, with no - // buffer_param in between to drop the empty buffers - std::pair - read_some_raw(std::span buffers) - { - if(! got_header()) - if(auto ec = read_header(); ec) - return { ec, 0 }; - - for(;;) - { - system::error_code ec; - auto const n = parser::read_some(buffers, ec); - if(ec != http::error::need_data) - return { ec, n }; - refill(); - } - } - template std::pair read(MB buffers) @@ -295,7 +302,7 @@ class parser_test // trailer first), finishing exactly at `eof_at` consumed octets // (zlib-style, or early), failing at `fail_at` alongside partial // output, or never finishing at all (a truncated stream). - struct test_decoder : parser::decoder + struct test_decoder : detail::decoder { std::string trailer; std::error_code fail_ec; @@ -2077,7 +2084,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); char buf[16]; { @@ -2112,7 +2119,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); auto [ec, body] = pr.read_body(); BOOST_TEST(!ec); @@ -2161,7 +2168,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); char buf[16]; { @@ -2533,7 +2540,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); char buf[16]; { @@ -2566,7 +2573,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); auto [ec, body] = pr.read_body(); BOOST_TEST(!ec); @@ -2713,7 +2720,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); char buf[16]; { @@ -2818,7 +2825,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); char buf[16]; { @@ -2851,7 +2858,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); char buf[16]; { @@ -2883,7 +2890,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); std::string got; for(;;) @@ -2916,7 +2923,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); char buf[16]; { @@ -2928,12 +2935,12 @@ class parser_test // the raw side ended and the decoder stalls: this // must be an error, not a wait for more input auto [ec, n] = pr.read_some(capy::make_buffer(buf)); - BOOST_TEST(ec == error::decode_error); + BOOST_TEST(ec == http::error::bad_payload); BOOST_TEST_EQ(n, 0); } { auto [ec, n] = pr.read_some(capy::make_buffer(buf)); - BOOST_TEST(ec == error::decode_error); + BOOST_TEST(ec == http::error::bad_payload); BOOST_TEST_EQ(n, 0); } } @@ -2954,7 +2961,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); char buf[16]; // the cleanly decoded bytes are delivered first @@ -2984,7 +2991,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); char buf[16]; // the bytes decoded before the failure are delivered first @@ -3014,7 +3021,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); capy::const_buffer arr[2]; { @@ -3047,7 +3054,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); // a bodiless message ends the decode path without // ever invoking the decoder @@ -3073,7 +3080,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); // an empty destination transfers nothing and is // not an error @@ -3117,9 +3124,9 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); - auto [ec, n] = pr.read_some_raw(dest()); + auto [ec, n] = pr.read_some(dest()); BOOST_TEST(!ec); BOOST_TEST_EQ(n, 5); BOOST_TEST(std::string_view(buf, n) == decoded("hello")); @@ -3138,15 +3145,69 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); - auto [ec, n] = pr.read_some_raw(dest()); + auto [ec, n] = pr.read_some(dest()); BOOST_TEST(!ec); BOOST_TEST_EQ(n, 5); BOOST_TEST(std::string_view(buf, n) == decoded("hello")); } } + void + testReadSomeSplitDestination() + { + // the body lands contiguously across a destination + // split into buffers smaller than the body, including + // when the split falls in the middle of a chunk + char b1[3]; + char b2[8]; + auto dest = [&] + { + return std::array{ + capy::mutable_buffer{ b1, sizeof(b1) }, + capy::mutable_buffer{ b2, sizeof(b2) } }; + }; + { + test_parser pr; + pr.provide( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 5\r\n" + "\r\n" + "hello"); + + pr.start(); + auto hec = pr.read_header(); + BOOST_TEST(!hec); + + auto [ec, n] = pr.read_some(dest()); + BOOST_TEST(!ec); + BOOST_TEST_EQ(n, 5); + BOOST_TEST(std::string_view(b1, 3) == "hel"); + BOOST_TEST(std::string_view(b2, 2) == "lo"); + } + { + test_parser pr; + pr.provide( + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "2\r\nhe\r\n" + "3\r\nllo\r\n" + "0\r\n\r\n"); + + pr.start(); + auto hec = pr.read_header(); + BOOST_TEST(!hec); + + auto [ec, n] = pr.read_some(dest()); + BOOST_TEST(!ec); + BOOST_TEST_EQ(n, 5); + BOOST_TEST(std::string_view(b1, 3) == "hel"); + BOOST_TEST(std::string_view(b2, 2) == "lo"); + } + } + void testDecoderBodyAfterStreaming() { @@ -3162,7 +3223,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); char buf[3]; { @@ -3194,7 +3255,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); // the in-place body cannot be completed; the output // produced before the failure is still viewable @@ -3218,7 +3279,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); capy::const_buffer arr[2]; { @@ -3259,7 +3320,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); std::string got; for(;;) @@ -3298,7 +3359,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); std::string got; for(;;) @@ -3340,7 +3401,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); char buf[16]; // the cleanly decoded bytes are delivered first @@ -3369,7 +3430,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); char buf[16]; // the cleanly decoded bytes are delivered first @@ -3383,6 +3444,71 @@ class parser_test BOOST_TEST_EQ(n2, 0); } + void + testDecodeGzip() + { +#ifdef BOOST_HTTP_HAS_ZLIB + auto& ctx = capy::get_system_context(); + if(!ctx.has_service()) + http::zlib::install_inflate_service(ctx); + + // gzip("hello world") + static char const gz[] = + "\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xcb\x48" + "\xcd\xc9\xc9\x57\x28\xcf\x2f\xca\x49\x01\x00\x85" + "\x11\x4a\x0d\x0b\x00\x00\x00"; + std::string const body(gz, sizeof(gz) - 1); + auto const msg = std::string( + "HTTP/1.1 200 OK\r\n" + "Content-Encoding: gzip\r\n" + "Content-Length: ") + + std::to_string(body.size()) + "\r\n\r\n" + body; + + auto read_all = [](test_parser& pr) + { + std::string got; + char buf[16]; + for(;;) + { + auto [ec, n] = pr.read_some(capy::make_buffer(buf)); + got.append(buf, n); + if(ec) + { + BOOST_TEST(ec == capy::cond::eof); + break; + } + } + return got; + }; + + // the parser selects the decoder from the header + { + test_parser pr; + pr.provide(msg); + pr.start(); + BOOST_TEST(read_all(pr) == "hello world"); + } + + // decoding disabled: the body is delivered as sent + { + parser::config cfg; + cfg.decode = false; + test_parser pr(cfg); + pr.provide(msg); + pr.start(); + BOOST_TEST(read_all(pr) == body); + } + + // HEAD: no body, nothing to decode + { + test_parser pr; + pr.provide(msg); + pr.start(true); + BOOST_TEST(read_all(pr).empty()); + } +#endif + } + void testDecoderToEof() { @@ -3402,7 +3528,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); char buf[16]; { @@ -3426,7 +3552,7 @@ class parser_test // decoder that echoes 5 body bytes (+1), then treats any further // input as a trailer it consumes WITHOUT producing output, and // finishes (like a gzip CRC/ISIZE trailer) - struct trailer_consuming_decoder : parser::decoder + struct trailer_consuming_decoder : detail::decoder { std::uint64_t consumed_total = 0; bool finished = false; @@ -3465,7 +3591,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); char buf[16]; { @@ -3502,7 +3628,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); auto [ec, body] = pr.read_body(); BOOST_TEST(!ec); @@ -3531,7 +3657,7 @@ class parser_test pr.start(); auto hec = pr.read_header(); BOOST_TEST(!hec); - pr.set_decoder(&dec); + pr.set_decoder(dec); std::string got; { @@ -4000,7 +4126,7 @@ class parser_test pr.start(); auto ec = pr.read_header(); BOOST_TEST(!ec); - pr.set_decoder(&dec); + pr.set_decoder(dec); BOOST_TEST_EQ(pr.direct_capacity(), 0u); } { @@ -4160,7 +4286,7 @@ class parser_test pr.start(); auto ec = pr.read_header(); BOOST_TEST(!ec); - pr.set_decoder(&dec); + pr.set_decoder(dec); auto [bec, body] = pr.read_body(); BOOST_TEST(!bec); @@ -4265,6 +4391,7 @@ class parser_test testDecoderNoPayload(); testDecoderReadSomeEmptyBuffer(); testDecoderReadSomeLeadingEmptyBuffer(); + testReadSomeSplitDestination(); testDecoderBodyAfterStreaming(); testDecoderReadBodyHardError(); testDecoderPullTwiceWithoutConsume(); @@ -4273,6 +4400,7 @@ class parser_test testDecoderChunkedEarlyEof(); testDecoderChunkedIncomplete(); testDecoderToEof(); + testDecodeGzip(); testDecoderToEofTrailerSeparateRead(); testDecoderReadBody(); testDecoderReadBodyOverflowThenStream(); diff --git a/test/unit/request_parser.cpp b/test/unit/request_parser.cpp index 08c2dd7..6a6d03f 100644 --- a/test/unit/request_parser.cpp +++ b/test/unit/request_parser.cpp @@ -62,12 +62,12 @@ class request_parser_test char buf[8]; capy::mutable_buffer mb(buf, sizeof(buf)); - auto n = pr.read_some({ &mb, 1 }, ec); + auto n = pr.read_some(mb, ec); BOOST_TEST(!ec); BOOST_TEST_EQ(n, 5); BOOST_TEST(std::string_view(buf, n) == "hello"); - n = pr.read_some({ &mb, 1 }, ec); + n = pr.read_some(mb, ec); BOOST_TEST(ec == capy::cond::eof); BOOST_TEST_EQ(n, 0); } diff --git a/test/unit/response_parser.cpp b/test/unit/response_parser.cpp index 8f046ed..7d62c58 100644 --- a/test/unit/response_parser.cpp +++ b/test/unit/response_parser.cpp @@ -65,12 +65,12 @@ class response_parser_test char buf[8]; capy::mutable_buffer mb(buf, sizeof(buf)); - auto n = pr.read_some({ &mb, 1 }, ec); + auto n = pr.read_some(mb, ec); BOOST_TEST(!ec); BOOST_TEST_EQ(n, 5); BOOST_TEST(std::string_view(buf, n) == "hello"); - n = pr.read_some({ &mb, 1 }, ec); + n = pr.read_some(mb, ec); BOOST_TEST(ec == capy::cond::eof); BOOST_TEST_EQ(n, 0); }