boost::beast http 分块响应缓冲区

Question

我有响应解析器

boost::beast::http::parser<false, boost::beast::http::buffer_body>

。据我了解， buffer_body 意味着响应主体数据应存储在用户提供的缓冲区中。但是，当我使用解析器的

on_chunk_body

方法在解析器上设置块回调时，解析器似乎没有使用提供的缓冲区。在这种情况下，它也可以在没有提供缓冲区的情况下工作

所以，我需要了解 http 解析器在接收块时如何管理内存？它使用一些内部缓冲区还是什么？

解析器似乎只对非分块响应使用提供的缓冲区。如果是，那么不为分块响应提供缓冲区是正确的吗？

Answer 1

Beast 支持分块编码。你不需要处理它。让我们通过从

httpbin.org

下载分块响应来证明：

首先与

vector_body

删除混淆部分：

住在Coliru

void using_vector_body() {
    tcp::socket conn = send_get();

    http::response<http::vector_body<uint8_t>> res;
    beast::flat_buffer buf;
    read(conn, buf, res);

    std::cout << "response: " << res.base() << "\n";

    std::span body = res.body();
    size_t const n = body.size();
    fmt::print("body, {} bytes: {::0x} ... {::0x}\n", n, body.first(10), body.last(10));

    auto checksum = reduce(begin(body), end(body), 0ull, std::bit_xor<>{});
    fmt::print("body checksum: {:#0x}\n", checksum);
}

打印例如

response: HTTP/1.1 200 OK
Date: Wed, 22 Mar 2023 00:43:25 GMT
Content-Type: application/octet-stream
Transfer-Encoding: chunked
Connection: keep-alive
Server: gunicorn/19.9.0
Access-Control-Allow-Origin: *
Access-Control-Allow-Credentials: true

body, 2000 bytes: [39, c, 8c, 7d, 72, 47, 34, 2c, d8, 10] ... [fd, 18, b0, c3, a3, d5, d1, 4c, 99, c0]
body checksum: 0xa1

转换为

buffer_body

我们需要使用解析器接口，因为它会驱动body-reader，我们需要在解析器上监控

is_done()

为了好看的风格，我们可能会更换首字母

read(conn, buf, p, ec);

更故意的：

read_header(conn, buf, p, ec);

我们会收到

need_buffer

错误，所以我们需要处理它们。然后，我们需要重复将 body reader 值设置到我们的缓冲区，并查看实际解码的内容。

注意不要不使用从

bytes_transferred

调用返回的

[async_]read

，因为这将包括线路中的所有内容，包括块头和尾部头。

计算解码缓冲区字节的界面是残暴非常不友好。但这是你需要的。

事不宜迟：

void using_buffer_body() {
    tcp::socket conn = send_get();

    http::response_parser<http::buffer_body> p;
    auto& res      = p.get(); // convenience shorthands
    auto& body_val = res.body();

    beast::flat_buffer buf;
    error_code ec;
    read_header(conn, buf, p, ec);
    //read(conn, buf, p, ec);

    if (ec && ec != http::error::need_buffer) // expected
        throw boost::system::system_error(ec);

    assert(p.is_header_done());
    std::cout << "\n---\nresponse headers: " << res.base() << std::endl;

    size_t checksum = 0;
    size_t n = 0;

    while (!p.is_done()) {
        std::array<uint8_t, 512> block;
        body_val.data = block.data();
        body_val.size = block.size();
        read(conn, buf, p, ec);

        if (ec && ec != http::error::need_buffer) // expected
            throw boost::system::system_error(ec);

        auto curr = block.size() - body_val.size;
        n += curr;

        std::cout << "parsed " << curr << " body bytes\n";

        for (auto b : std::span(block).first(curr))
            checksum ^= b;
    }

    fmt::print("body, {} bytes streaming decoded, chunked? {}\n", n, p.chunked());
    fmt::print("body checksum: {:#0x}\n", checksum);
}

完整的现场演示

演示确认这两种方法产生相同的主体长度和相同的校验和：

住在Coliru

#include <boost/beast.hpp>
#include <fmt/ranges.h>
#include <iostream>
#include <span>
namespace net   = boost::asio;
namespace beast = boost::beast;
namespace http  = beast::http;
using boost::system::error_code;
using net::ip::tcp;

tcp::socket send_get() {
    net::system_executor ex;
    tcp::socket          s(ex);
    // connect(s, tcp::resolver(ex).resolve("httpbin.org", "http"));
    connect(s, tcp::resolver(ex).resolve("52.86.68.46", "80")); // For COLIRU, DNS is not available

    http::request<http::empty_body> req{http::verb::get, "/stream-bytes/2000?seed=42", 11};
    req.set(http::field::host, "httpbin.org");
    write(s, req);

    return s;
}

void using_vector_body() {
    tcp::socket conn = send_get();

    http::response<http::vector_body<uint8_t>> res;
    beast::flat_buffer buf;
    read(conn, buf, res);

    std::cout << "response: " << res.base() << "\n";

    std::span body = res.body();
    size_t const n = body.size();
    fmt::print("body, {} bytes: {::0x} ... {::0x}\n", n, body.first(10), body.last(10));

    auto checksum = reduce(begin(body), end(body), 0ull, std::bit_xor<>{});
    fmt::print("body checksum: {:#0x}\n", checksum);
}

void using_buffer_body() {
    tcp::socket conn = send_get();

    http::response_parser<http::buffer_body> p;
    auto& res      = p.get(); // convenience shorthands
    auto& body_val = res.body();

    beast::flat_buffer buf;
    error_code ec;
    read_header(conn, buf, p, ec);
    //read(conn, buf, p, ec);

    if (ec && ec != http::error::need_buffer) // expected
        throw boost::system::system_error(ec);

    assert(p.is_header_done());
    std::cout << "\n---\nresponse headers: " << res.base() << std::endl;

    size_t checksum = 0;
    size_t n = 0;

    while (!p.is_done()) {
        std::array<uint8_t, 512> block;
        body_val.data = block.data();
        body_val.size = block.size();
        read(conn, buf, p, ec);

        if (ec && ec != http::error::need_buffer) // expected
            throw boost::system::system_error(ec);

        auto curr = block.size() - body_val.size;
        n += curr;

        std::cout << "parsed " << curr << " body bytes\n";

        for (auto b : std::span(block).first(curr))
            checksum ^= b;
    }

    fmt::print("body, {} bytes streaming decoded, chunked? {}\n", n, p.chunked());
    fmt::print("body checksum: {:#0x}\n", checksum);
}

int main() {
    using_vector_body();
    using_buffer_body();
}

打印例如

response: HTTP/1.1 200 OK
Date: Wed, 22 Mar 2023 00:52:32 GMT
Content-Type: application/octet-stream
Transfer-Encoding: chunked
Connection: keep-alive
Server: gunicorn/19.9.0
Access-Control-Allow-Origin: *
Access-Control-Allow-Credentials: true

body, 2000 bytes: [39, c, 8c, 7d, 72, 47, 34, 2c, d8, 10] ... [fd, 18, b0, c3, a3, d5, d1, 4c, 99, c0]
body checksum: 0xa1

---
response headers: HTTP/1.1 200 OK
Date: Wed, 22 Mar 2023 00:52:32 GMT
Content-Type: application/octet-stream
Transfer-Encoding: chunked
Connection: keep-alive
Server: gunicorn/19.9.0
Access-Control-Allow-Origin: *
Access-Control-Allow-Credentials: true

parsed 512 body bytes
parsed 512 body bytes
parsed 512 body bytes
parsed 464 body bytes
body, 2000 bytes streaming decoded, chunked? true
body checksum: 0xa1