From c8199d8c5a2c22d96dcd3b449d170fcf4d79ca41 Mon Sep 17 00:00:00 2001 From: Mohammad Nejati Date: Thu, 9 Jan 2025 05:20:09 +0000 Subject: [PATCH] parser: two-phase parsing With the new changes, the parser returns immediately after the header is parsed and does not begin parsing the body until the next call to `parse()`. In the case of bodiless messages and head responses, it directly transitions to the `complete_in_place` state after the header is parsed, making a call to `parse()` unnecessary (but still valid). This two-phase parsing brings a few benefits with almost no complications on the usage side of the API: - It introduces an optimization opportunity for users who want to attach a body. If they do so immediately after the header is parsed (which seems to be the case most of the time), there's no need for `cb1_` for elastic bodies and a small `cb1_` for sink bodies (as it will be used temporarily). This means all the extra space can be utilized for `cb0_`. - Because parsing the body might complete with an error, returning after the header is parsed allows users to access the header and on the next call to parse encounter the error. - Setting the body limit in the middle of parsing the body or after it doesn't make much sense, so returning right after the header is parsed provides a window for setting such limits. - If users want to attach a body, they will almost always do so immediately after the header is parsed. By not continuing the parsing of the body, we avoid the need for an extra buffer copy operation (in case the user wants to attach a buffer). --- include/boost/http_proto/parser.hpp | 22 +- src/parser.cpp | 367 ++++++++++++++++------------ test/unit/parser.cpp | 354 +++++++++++++++------------ test/unit/zlib.cpp | 8 +- 4 files changed, 421 insertions(+), 330 deletions(-) diff --git a/include/boost/http_proto/parser.hpp b/include/boost/http_proto/parser.hpp index 724e7f1f..8a28c3b3 100644 --- a/include/boost/http_proto/parser.hpp +++ b/include/boost/http_proto/parser.hpp @@ -323,6 +323,18 @@ class BOOST_SYMBOL_VISIBLE Sink& set_body(Args&&... args); + /** Sets the maximum allowed size of the body for the current message. + + This overrides the default value specified by + @ref config_base::body_limit. + The limit automatically resets to the default + for the next message. + + @param n The new body size limit in bytes. + */ + void + set_body_limit(std::uint64_t n); + /** Return the available body data. The returned buffer span will be invalidated if any member @@ -369,9 +381,6 @@ class BOOST_SYMBOL_VISIBLE bool is_plain() const noexcept; - void - on_headers(system::error_code&); - BOOST_HTTP_PROTO_DECL void on_set_body() noexcept; @@ -382,6 +391,9 @@ class BOOST_SYMBOL_VISIBLE std::size_t, bool); + std::uint64_t + body_limit_remain() const noexcept; + static constexpr unsigned buffers_N = 8; enum class state @@ -389,6 +401,7 @@ class BOOST_SYMBOL_VISIBLE reset, start, header, + header_done, body, set_body, complete_in_place, @@ -407,10 +420,11 @@ class BOOST_SYMBOL_VISIBLE detail::workspace ws_; detail::header h_; - std::size_t body_avail_ = 0; + std::uint64_t body_limit_= 0; std::uint64_t body_total_ = 0; std::uint64_t payload_remain_ = 0; std::uint64_t chunk_remain_ = 0; + std::size_t body_avail_ = 0; std::size_t nprepare_ = 0; // used to store initial headers + any potential overread diff --git a/src/parser.cpp b/src/parser.cpp index f6b9c033..3ebfd9d0 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -488,7 +488,6 @@ parser:: reset() noexcept { ws_.clear(); - eb_ = nullptr; st_ = state::start; got_eof_ = false; } @@ -520,6 +519,7 @@ start_impl( } BOOST_FALLTHROUGH; + case state::header_done: case state::body: case state::set_body: // current message is incomplete @@ -599,6 +599,7 @@ start_impl( // begin with in_place mode how_ = how::in_place; st_ = state::header; + body_limit_ = svc_.cfg.body_limit; nprepare_ = 0; chunk_remain_ = 0; needs_chunk_close_ = false; @@ -606,6 +607,7 @@ start_impl( chunked_body_ended = false; filter_ = nullptr; sink_ = nullptr; + eb_ = nullptr; body_avail_ = 0; body_total_ = 0; } @@ -640,6 +642,10 @@ prepare() -> return mutable_buffers_type(&mbp_[0], 1); } + case state::header_done: + // forgot to call parse() + detail::throw_logic_error(); + case state::body: { if(got_eof_) @@ -683,9 +689,7 @@ prepare() -> { BOOST_ASSERT( h_.md.payload == payload::to_eof); - - n = clamp( - svc_.cfg.body_limit - body_total_ + 1, n); + n = clamp(body_limit_remain() + 1, n); } nprepare_ = n; @@ -710,10 +714,8 @@ prepare() -> { BOOST_ASSERT( h_.md.payload == payload::to_eof); - n = clamp( - svc_.cfg.body_limit - body_total_, n); - n = clamp( - n, eb_->max_size() - eb_->size()); + n = clamp(body_limit_remain() + 1, n); + n = clamp(n, eb_->max_size() - eb_->size()); // fill capacity first to avoid an allocation std::size_t avail = eb_->capacity() - eb_->size(); @@ -791,6 +793,12 @@ commit( break; } + case state::header_done: + { + // forgot to call parse() + detail::throw_logic_error(); + } + case state::body: { nprepare_ = 0; // invalidate @@ -798,14 +806,17 @@ commit( { if(eb_->max_size() == eb_->size()) { - // borrowed 1 byte from cb0_ + // borrowed 1 byte from + // cb0_ in prepare() BOOST_ASSERT(n <= 1); cb0_.commit(n); - break; } - eb_->commit(n); - payload_remain_ -= n; - body_total_ += n; + else + { + eb_->commit(n); + payload_remain_ -= n; + body_total_ += n; + } } else { @@ -850,6 +861,10 @@ commit_eof() got_eof_ = true; break; + case state::header_done: + // forgot to call parse() + detail::throw_logic_error(); + case state::body: got_eof_ = true; break; @@ -919,7 +934,7 @@ parse( error::incomplete); return; } - if(ec.failed()) + else if(ec.failed()) { // other error, // @@ -930,13 +945,100 @@ parse( return; } - // headers are complete - on_headers(ec); - if(ec.failed()) + // reserve headers + table + ws_.reserve_front(h_.size); + ws_.reserve_back(h_.table_space()); + + // no payload + if(h_.md.payload == payload::none || + head_response_) + { + // octets of the next message + auto overread = fb_.size() - h_.size; + cb0_ = { ws_.data(), overread, overread }; + ws_.reserve_front(overread); + st_ = state::complete_in_place; return; - if(st_ == state::complete_in_place) - break; + } + + st_ = state::header_done; + break; + } + + case state::header_done: + { + // metadata error + if(h_.md.payload == payload::error) + { + // VFALCO This needs looking at + ec = BOOST_HTTP_PROTO_ERR( + error::bad_payload); + st_ = state::reset; // unrecoverable + return; + } + + // overread currently includes any and all octets that + // extend beyond the current end of the header + // this can include associated body octets for the + // current message or octets of the next message in the + // stream, e.g. pipelining is being used + auto const overread = fb_.size() - h_.size; + BOOST_ASSERT(overread <= svc_.max_overread()); + + auto cap = fb_.capacity() + overread + + svc_.cfg.min_buffer; + + // reserve body buffers first, as the decoder + // must be installed after them. + auto const p = ws_.reserve_front(cap); + + if(svc_.cfg.apply_deflate_decoder && + h_.md.content_encoding.encoding == encoding::deflate) + { + filter_ = &ws_.emplace(ctx_, ws_, false); + } + else if(svc_.cfg.apply_gzip_decoder && + h_.md.content_encoding.encoding == encoding::gzip) + { + filter_ = &ws_.emplace(ctx_, ws_, true); + } + else + { + cap += svc_.max_codec; + ws_.reserve_front(svc_.max_codec); + } + + if(is_plain() || how_ == how::elastic) + { + cb0_ = { p, cap, overread }; + cb1_ = {}; + } + else + { + // buffered payload + std::size_t n0 = (overread > svc_.cfg.min_buffer) + ? overread + : svc_.cfg.min_buffer; + std::size_t n1 = svc_.cfg.min_buffer; + + cb0_ = { p , n0, overread }; + cb1_ = { p + n0 , n1 }; + } + + if(h_.md.payload == payload::size) + { + if(!filter_ && + body_limit_ < h_.md.payload_size) + { + ec = BOOST_HTTP_PROTO_ERR( + error::body_too_large); + st_ = state::reset; + return; + } + payload_remain_ = h_.md.payload_size; + } + st_ = state::body; BOOST_FALLTHROUGH; } @@ -1042,8 +1144,7 @@ parse( const auto chunk = buffers::prefix(cb0_.data(), chunk_avail); - if(svc_.cfg.body_limit - body_total_ - < chunk_avail) + if(body_limit_remain() < chunk_avail) { ec = BOOST_HTTP_PROTO_ERR( error::body_too_large); @@ -1150,6 +1251,19 @@ parse( } else { + // plain body + + if(h_.md.payload == payload::to_eof) + { + if(body_limit_remain() < payload_avail) + { + ec = BOOST_HTTP_PROTO_ERR( + error::body_too_large); + st_ = state::reset; + return; + } + } + switch(how_) { case how::in_place: @@ -1190,27 +1304,31 @@ parse( // payload_remain_ and body_total_ // are already updated in commit() - if(cb0_.size() != 0) + // cb0_ contains data + if(payload_avail != 0) { - // a non-empty cb0_ indicates that - // the elastic buffer ran out of space - ec = BOOST_HTTP_PROTO_ERR( - error::buffer_overflow); - st_ = state::reset; - return; + if(eb_->max_size() - eb_->size() + < payload_avail) + { + ec = BOOST_HTTP_PROTO_ERR( + error::buffer_overflow); + st_ = state::reset; + return; + } + // only happens when an elastic body + // is attached in header_done state + buffers::buffer_copy( + eb_->prepare(payload_avail), + cb0_.data()); + cb0_.consume(payload_avail); + eb_->commit(payload_avail); + payload_remain_ -= payload_avail; + body_total_ += payload_avail; } break; } } - if(body_total_ > svc_.cfg.body_limit) - { - ec = BOOST_HTTP_PROTO_ERR( - error::body_too_large); - st_ = state::reset; - return; - } - if(is_complete) { set_state_to_complete(); @@ -1239,9 +1357,6 @@ parse( { auto& body_buf = is_plain() ? cb0_ : cb1_; - BOOST_ASSERT(body_avail_ == body_buf.size()); - BOOST_ASSERT(body_total_ == body_avail_); - switch(how_) { case how::in_place: @@ -1307,6 +1422,8 @@ pull_body() -> { switch(st_) { + case state::header_done: + return {}; case state::body: case state::complete_in_place: cbp_ = buffers::prefix( @@ -1324,6 +1441,8 @@ consume_body(std::size_t n) { switch(st_) { + case state::header_done: + return; case state::body: case state::complete_in_place: n = clamp(n, body_avail_); @@ -1358,6 +1477,27 @@ release_buffered_data() noexcept return {}; } +void +parser:: +set_body_limit(std::uint64_t n) +{ + switch(st_) + { + case state::header: + case state::header_done: + body_limit_ = n; + break; + case state::complete_in_place: + // only allowed for empty bodies + if(body_total_ == 0) + break; + BOOST_FALLTHROUGH; + default: + // set body_limit before parsing the body + detail::throw_logic_error(); + } +} + //------------------------------------------------ // // Implementation @@ -1385,118 +1525,15 @@ is_plain() const noexcept h_.md.payload != payload::chunked; } -// Called immediately after complete headers are received -// to setup the circular buffers for subsequent operations. -// We leave fb_ as-is to indicate whether any data was -// received before eof. -// -void -parser:: -on_headers( - system::error_code& ec) -{ - // overread currently includes any and all octets that - // extend beyond the current end of the header - // this can include associated body octets for the - // current message or octets of the next message in the - // stream, e.g. pipelining is being used - auto const overread = fb_.size() - h_.size; - BOOST_ASSERT( - overread <= svc_.max_overread()); - - // metadata error - if(h_.md.payload == payload::error) - { - // VFALCO This needs looking at - ec = BOOST_HTTP_PROTO_ERR( - error::bad_payload); - st_ = state::reset; // unrecoverable - return; - } - - // reserve headers + table - ws_.reserve_front(h_.size); - ws_.reserve_back(h_.table_space()); - - // no payload - if( h_.md.payload == payload::none || - head_response_ ) - { - // set cb0_ to overread - cb0_ = { - ws_.data(), - overread + fb_.capacity(), - overread }; - st_ = state::complete_in_place; - return; - } - - auto cap = fb_.capacity() + overread + - svc_.cfg.min_buffer; - - // reserve body buffers first, as the decoder - // must be installed after them. - auto const p = ws_.reserve_front(cap); - - if( svc_.cfg.apply_deflate_decoder && - h_.md.content_encoding.encoding == encoding::deflate ) - { - filter_ = &ws_.emplace( - ctx_, ws_, false); - } - else if( svc_.cfg.apply_gzip_decoder && - h_.md.content_encoding.encoding == encoding::gzip ) - { - filter_ = &ws_.emplace( - ctx_, ws_, true); - } - else - { - cap += svc_.max_codec; - ws_.reserve_front(svc_.max_codec); - } - - if(h_.md.payload == payload::size ) - payload_remain_ = h_.md.payload_size; - - if(is_plain()) - { - cb0_ = { p, cap, overread }; - if(h_.md.payload == payload::size) - { - if(h_.md.payload_size > - svc_.cfg.body_limit) - { - ec = BOOST_HTTP_PROTO_ERR( - error::body_too_large); - st_ = state::reset; - return; - } - } - st_ = state::body; - return; - } - - // buffered payload - - const auto n0 = (overread > svc_.cfg.min_buffer) - ? overread - : svc_.cfg.min_buffer; - const auto n1 = svc_.cfg.min_buffer; - - cb0_ = { p , n0, overread }; - cb1_ = { p + n0 , n1 }; - st_ = state::body; -} - // Called at the end of set_body void parser:: on_set_body() noexcept { BOOST_ASSERT( - st_ == state::complete_in_place || - st_ == state::body); + st_ == state::header_done || + st_ == state::body || + st_ == state::complete_in_place); nprepare_ = 0; // invalidate @@ -1512,17 +1549,15 @@ apply_filter( bool more) { std::size_t p0 = payload_avail; - for(;;) { if(payload_avail == 0 && more) - return p0 - payload_avail; + break; auto f_rs = [&](){ if(how_ == how::elastic) { - std::size_t n = clamp( - svc_.cfg.body_limit - body_total_); + std::size_t n = clamp(body_limit_remain()); n = clamp(n, svc_.cfg.min_buffer); n = clamp(n, eb_->max_size() - eb_->size()); @@ -1542,8 +1577,7 @@ apply_filter( } else // in-place and sink { - std::size_t n = clamp( - svc_.cfg.body_limit - body_total_); + std::size_t n = clamp(body_limit_remain()); n = clamp(n, cb1_.capacity()); return filter_->process( @@ -1568,12 +1602,12 @@ apply_filter( { cb1_.commit(f_rs.out_bytes); body_avail_ += f_rs.out_bytes; - if( cb1_.capacity() == 0 && + if(cb1_.capacity() == 0 && needs_more_space) { ec = BOOST_HTTP_PROTO_ERR( error::in_place_overflow); - return p0 - payload_avail; + goto done; } break; } @@ -1587,20 +1621,20 @@ apply_filter( { ec = sink_rs.ec; st_ = state::reset; - return p0 - payload_avail; + goto done; } break; } case how::elastic: { eb_->commit(f_rs.out_bytes); - if( eb_->max_size() - eb_->size() == 0 && + if(eb_->max_size() - eb_->size() == 0 && needs_more_space) { ec = BOOST_HTTP_PROTO_ERR( error::buffer_overflow); st_ = state::reset; - return p0 - payload_avail; + goto done; } break; } @@ -1610,16 +1644,16 @@ apply_filter( { ec = f_rs.ec; st_ = state::reset; - return p0 - payload_avail; + break; } - if( svc_.cfg.body_limit == body_total_ && + if(body_limit_remain() == 0 && needs_more_space) { - ec = BOOST_HTTP_PROTO_ERR( + ec = BOOST_HTTP_PROTO_ERR( error::body_too_large); st_ = state::reset; - return p0 - payload_avail; + break; } if(f_rs.finished) @@ -1630,10 +1664,19 @@ apply_filter( ? state::complete_in_place : state::complete; } - - return p0 - payload_avail; + break; } } + +done: + return p0 - payload_avail; +} + +std::uint64_t +parser:: +body_limit_remain() const noexcept +{ + return body_limit_ - body_total_; } } // http_proto diff --git a/test/unit/parser.cpp b/test/unit/parser.cpp index 927a0454..628a17e4 100644 --- a/test/unit/parser.cpp +++ b/test/unit/parser.cpp @@ -232,6 +232,9 @@ struct parser_test pieces& in, system::error_code& ec) { + pr.parse(ec); + if(ec != condition::need_more_input) + return; if(! in.empty()) { core::string_view& s = in[0]; @@ -259,7 +262,7 @@ struct parser_test pieces& in, system::error_code& ec) { - while(! pr.got_header()) + do { read_some(pr, in, ec); if(ec == condition::need_more_input) @@ -267,7 +270,7 @@ struct parser_test if(ec.failed()) return; } - ec = {}; + while(! pr.got_header()); } static @@ -277,11 +280,6 @@ struct parser_test pieces& in, system::error_code& ec) { - if(pr.is_complete()) - { - pr.parse(ec); - return; - } do { read_some(pr, in, ec); @@ -506,6 +504,8 @@ struct parser_test ! BOOST_TEST(pr.got_header()) || ! BOOST_TEST(! pr.is_complete())) return; + pr.parse(ec); + BOOST_TEST_EQ(ec, error::need_data); parser::mutable_buffers_type dest; BOOST_TEST_NO_THROW( dest = pr.prepare()); @@ -581,7 +581,7 @@ struct parser_test &tmp, dynamic_max_size); pr->set_body(std::move(sb)); pr->parse(ec); - BOOST_TEST(ec == ex); + BOOST_TEST_EQ(ec, ex); if(! ec.failed()) { parser::mutable_buffers_type dest; @@ -807,6 +807,7 @@ struct parser_test "\r\n" }; read_header(pr, in, ec); BOOST_TEST(! ec.failed()); + pr.parse(ec); auto dest = pr.prepare(); BOOST_TEST_THROWS( pr.commit( @@ -917,7 +918,7 @@ struct parser_test "Content-Length: 3\r\n" "\r\n" "123" }; - read_header(pr, in, ec); + read(pr, in, ec); BOOST_TEST(! ec.failed()); BOOST_TEST(pr.is_complete()); std::unique_ptr ps(new std::string()); @@ -925,7 +926,7 @@ struct parser_test pr.set_body( buffers::string_buffer(&s)); pr.parse(ec); - BOOST_TEST(! ec); + BOOST_TEST(! ec.failed()); pr.reset(); } @@ -943,7 +944,7 @@ struct parser_test "Content-Length: 3\r\n" "\r\n" "123" }; - read_header(pr, in, ec); + read(pr, in, ec); BOOST_TEST(! ec.failed()); BOOST_TEST(pr.is_complete()); @@ -974,6 +975,9 @@ struct parser_test read_header(pr, in, ec); BOOST_TEST(! ec.failed()); BOOST_TEST(!pr.is_complete()); + pr.parse(ec); + BOOST_TEST_EQ( + ec, condition::need_more_input); auto dest = pr.prepare(); ignore_unused(dest); BOOST_TEST_NO_THROW(pr.commit(0)); @@ -992,6 +996,9 @@ struct parser_test read_header(pr, in, ec); BOOST_TEST(! ec.failed()); BOOST_TEST(!pr.is_complete()); + pr.parse(ec); + BOOST_TEST_EQ( + ec, condition::need_more_input); auto dest = pr.prepare(); ignore_unused(dest); BOOST_TEST_THROWS( @@ -1043,6 +1050,9 @@ struct parser_test read_header(pr, in, ec); BOOST_TEST(! ec.failed()); BOOST_TEST(! pr.is_complete()); + pr.parse(ec); + BOOST_TEST_EQ( + ec, condition::need_more_input); BOOST_TEST_NO_THROW( pr.commit_eof()); } @@ -1299,7 +1309,6 @@ struct parser_test } buffers::flat_buffer fb(buf, sizeof(buf)); pr_->set_body(std::ref(fb)); - pr_->parse(ec); BOOST_TEST(pr_->body().empty()); if(! pr_->is_complete()) { @@ -1337,7 +1346,6 @@ struct parser_test return; } auto& ts = pr_->set_body(); - pr_->parse(ec); BOOST_TEST(pr_->body().empty()); if(! pr_->is_complete()) { @@ -1618,28 +1626,16 @@ struct parser_test pr.reset(); pr.start(); - core::string_view res = + pieces in = { "HTTP/1.1 200 OK\r\n" "transfer-encoding: chunked\r\n" "\r\n" "d\r\nhello, world!\r\n" - "0\r\n\r\n"; - - auto bs = pr.prepare(); - auto n = - buffers::buffer_copy( - bs, - buffers::const_buffer( - res.data(), - res.size())); - - BOOST_TEST_EQ(n, res.size()); - pr.commit(n); + "0\r\n\r\n" }; system::error_code ec; - - pr.parse(ec); - BOOST_TEST(!ec); + read(pr, in, ec); + BOOST_TEST(! ec.failed()); BOOST_TEST(pr.is_complete()); auto str = pr.body(); @@ -1656,38 +1652,24 @@ struct parser_test pr.reset(); pr.start(); - core::string_view res = + pieces in1 = { "HTTP/1.1 200 OK\r\n" "transfer-encoding: chunked\r\n" "\r\n" - "00000000000000000000000000000000000d\r\n"; + "00000000000000000000000000000000000d\r\n" }; - core::string_view body1 = + pieces in2 = { "hello, world!\r\n" - "0\r\n\r\n"; - - pr.commit( - buffers::buffer_copy( - pr.prepare(), - buffers::const_buffer( - res.data(), res.size()))); + "0\r\n\r\n" }; system::error_code ec; - - pr.parse(ec); - BOOST_TEST(ec == condition::need_more_input); + read_header(pr, in1, ec); + BOOST_TEST(! ec.failed()); BOOST_TEST(pr.got_header()); - BOOST_TEST(!pr.is_complete()); - - pr.commit( - buffers::buffer_copy( - pr.prepare(), - buffers::const_buffer( - body1.data(), body1.size()))); + BOOST_TEST(! pr.is_complete()); - pr.parse(ec); - BOOST_TEST(!ec); - BOOST_TEST(pr.got_header()); + read(pr, in2, ec); + BOOST_TEST(! ec.failed()); BOOST_TEST(pr.is_complete()); BOOST_TEST_EQ(pr.body(), "hello, world!"); } @@ -1703,25 +1685,23 @@ struct parser_test // the chunk body is an OCTET stream, thus it // can contain anything - core::string_view res = + pieces in1 = { "HTTP/1.1 200 OK\r\n" "transfer-encoding: chunked\r\n" "\r\n" "1234\r\nhello, world!\r\n" - "0\r\n\r\n"; - - pr.commit( - buffers::buffer_copy( - pr.prepare(), - buffers::const_buffer( - res.data(), res.size()))); + "0\r\n\r\n" }; system::error_code ec; + read_header(pr, in1, ec); + BOOST_TEST(! ec.failed()); + BOOST_TEST(pr.got_header()); + BOOST_TEST(! pr.is_complete()); pr.parse(ec); - BOOST_TEST(ec); - BOOST_TEST_EQ(ec, condition::need_more_input); - BOOST_TEST(!pr.is_complete()); + BOOST_TEST_EQ( + ec, condition::need_more_input); + BOOST_TEST(! pr.is_complete()); } { @@ -1733,24 +1713,22 @@ struct parser_test // the chunk body is an OCTET stream, thus it // can contain anything - core::string_view res = + pieces in1 = { "HTTP/1.1 200 OK\r\n" "transfer-encoding: chunked\r\n" "\r\n" "03\r\nhello, world!\r\n" - "0\r\n\r\n"; - - pr.commit( - buffers::buffer_copy( - pr.prepare(), - buffers::const_buffer( - res.data(), res.size()))); + "0\r\n\r\n" }; system::error_code ec; + read_header(pr, in1, ec); + BOOST_TEST(! ec.failed()); + BOOST_TEST(pr.got_header()); + BOOST_TEST(! pr.is_complete()); pr.parse(ec); - BOOST_TEST(ec); - BOOST_TEST_EQ(ec, condition::invalid_payload); + BOOST_TEST_EQ( + ec, condition::invalid_payload); BOOST_TEST(!pr.is_complete()); } @@ -1759,7 +1737,7 @@ struct parser_test pr.reset(); pr.start(); - std::vector pieces = { + pieces in = { "HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n", "d\r\nhello, ", "world!", @@ -1772,23 +1750,7 @@ struct parser_test }; system::error_code ec; - for( auto piece : pieces ) - { - pr.commit( - buffers::buffer_copy( - pr.prepare(), - buffers::const_buffer( - piece.data(), piece.size()))); - - - pr.parse(ec); - if( ec) - { - BOOST_TEST_EQ( - ec, condition::need_more_input); - } - } - + read(pr, in, ec); BOOST_TEST(pr.is_complete()); BOOST_TEST_EQ( pr.body(), @@ -1804,7 +1766,7 @@ struct parser_test "transfer-encoding: chunked\r\n" "\r\n"; - std::vector pieces = { + pieces samples = { "xxxasdfasdfasd", // invalid chunk header "1\rzxcv", // invalid crlf on chunk-size close "1\r\nabcd", // invalid chunk-body close @@ -1816,27 +1778,23 @@ struct parser_test "fffffffffffffffff\r\n" }; - for( auto piece : pieces ) + for( core::string_view bad_chunk : samples ) { pr.reset(); pr.start(); - pr.commit( - buffers::buffer_copy( - pr.prepare(), - buffers::const_buffer( - headers.data(), - headers.size()))); - pr.commit( - buffers::buffer_copy( - pr.prepare(), - buffers::const_buffer( - piece.data(), - piece.size()))); + pieces in = { headers, bad_chunk }; system::error_code ec; + read_header(pr, in, ec); + BOOST_TEST(! ec.failed()); + BOOST_TEST(pr.got_header()); + pr.parse(ec); - BOOST_TEST(ec); + BOOST_TEST_EQ( + ec, condition::need_more_input); + + read_some(pr, in, ec); BOOST_TEST_EQ( ec, condition::invalid_payload); BOOST_TEST(!pr.is_complete()); @@ -1861,55 +1819,24 @@ struct parser_test for( std::size_t i = 0; i < body.size(); ++i ) { - auto s1 = body.substr(0, i); - auto s2 = body.substr(i); - pr.reset(); pr.start(); - pr.commit( - buffers::buffer_copy( - pr.prepare(), - buffers::const_buffer( - headers.data(), - headers.size()))); + pieces in = { + headers, + body.substr(0, i), + body.substr(i) }; system::error_code ec; - pr.parse(ec); - BOOST_TEST_EQ( - ec, condition::need_more_input); + read_header(pr, in, ec); + BOOST_TEST(! ec.failed()); BOOST_TEST(pr.got_header()); - pr.commit( - buffers::buffer_copy( - pr.prepare(), - buffers::const_buffer( - s1.data(), - s1.size()))); - - pr.parse(ec); - if( ec && - !BOOST_TEST_EQ( - ec, condition::need_more_input) ) - { - break; - } - - pr.commit( - buffers::buffer_copy( - pr.prepare(), - buffers::const_buffer( - s2.data(), - s2.size()))); - pr.parse(ec); - if( ec && - !BOOST_TEST_EQ( - ec, condition::need_more_input) ) - { - break; - } + BOOST_TEST_EQ( + ec, condition::need_more_input); + read(pr, in, ec); BOOST_TEST(pr.is_complete()); BOOST_TEST_EQ( pr.body(), @@ -1956,11 +1883,14 @@ struct parser_test pr.commit(buffers::buffer_copy( pr.prepare(), - buffers::const_buffer(octets.data(), octets.size()))); + buffers::const_buffer( + octets.data(), octets.size()))); pr.parse(ec); - BOOST_TEST_EQ(ec, condition::need_more_input); - BOOST_TEST(pr.got_header()); + if(! ec) + pr.parse(ec); + BOOST_TEST_EQ( + ec, condition::need_more_input); BOOST_TEST(!pr.is_complete()); pr.consume_body( @@ -1978,21 +1908,27 @@ struct parser_test pr.commit(buffers::buffer_copy( pr.prepare(), - buffers::const_buffer(octets.data(), octets.size()))); + buffers::const_buffer( + octets.data(), octets.size()))); // first message + if(! pr.got_header()) + { + pr.parse(ec); + BOOST_TEST(! ec.failed()); + BOOST_TEST(pr.got_header()); + } pr.parse(ec); - BOOST_TEST(!ec); - BOOST_TEST(pr.got_header()); + BOOST_TEST(! ec.failed()); BOOST_TEST(pr.is_complete()); // second message pr.start(); - BOOST_TEST(!pr.got_header()); - BOOST_TEST(!pr.is_complete()); pr.parse(ec); - BOOST_TEST(!ec); + BOOST_TEST(! ec.failed()); BOOST_TEST(pr.got_header()); + pr.parse(ec); + BOOST_TEST(! ec.failed()); BOOST_TEST(pr.is_complete()); } @@ -2051,11 +1987,14 @@ struct parser_test pr.commit(buffers::buffer_copy( pr.prepare(), - buffers::const_buffer(octets.data(), octets.size()))); + buffers::const_buffer( + octets.data(), octets.size()))); pr.parse(ec); - BOOST_TEST_EQ(ec, condition::need_more_input); - BOOST_TEST(pr.got_header()); + if(! ec) + pr.parse(ec); + BOOST_TEST_EQ( + ec, condition::need_more_input); BOOST_TEST(!pr.is_complete()); pr.consume_body( @@ -2074,27 +2013,115 @@ struct parser_test pr.commit(buffers::buffer_copy( pr.prepare(), - buffers::const_buffer(octets.data(), octets.size()))); + buffers::const_buffer( + octets.data(), octets.size()))); // first message pr.parse(ec); - BOOST_TEST(!ec); + BOOST_TEST(! ec.failed()); BOOST_TEST(pr.got_header()); + pr.parse(ec); + BOOST_TEST(! ec.failed()); BOOST_TEST(pr.is_complete()); // second message pr.start(); - BOOST_TEST(!pr.got_header()); - BOOST_TEST(!pr.is_complete()); pr.parse(ec); - BOOST_TEST(!ec); + BOOST_TEST(! ec.failed()); BOOST_TEST(pr.got_header()); + pr.parse(ec); + BOOST_TEST(! ec.failed()); BOOST_TEST(pr.is_complete()); } } //------------------------------------------- + void + testSetBodyLimit() + { + context ctx; + response_parser::config cfg; + cfg.body_limit = 7; + install_parser_service(ctx, cfg); + + response_parser pr(ctx); + + // before reset + BOOST_TEST_THROWS( + pr.set_body_limit(3), + std::logic_error) + + pr.reset(); + + // before start + BOOST_TEST_THROWS( + pr.set_body_limit(3), + std::logic_error) + + pr.start(); + pr.set_body_limit(3); + + { + pieces in = { + "HTTP/1.1 200 OK\r\n" + "content-length: 7\r\n" + "\r\n" + "1234567" }; + system::error_code ec; + read_header(pr, in, ec); + BOOST_TEST(! ec.failed()); + BOOST_TEST(pr.got_header()); + read(pr, in, ec); + BOOST_TEST_EQ( + ec, error::body_too_large); + } + + pr.reset(); + pr.start(); + + // body_limit reset to cfg value + // after start + { + pieces in = { + "HTTP/1.1 200 OK\r\n" + "content-length: 7\r\n" + "\r\n" + "1234567" }; + system::error_code ec; + read_header(pr, in, ec); + BOOST_TEST(! ec.failed()); + BOOST_TEST(pr.got_header()); + read(pr, in, ec); + BOOST_TEST(! ec.failed()); + BOOST_TEST(pr.is_complete()); + + } + + // after completion with body + BOOST_TEST(! pr.body().empty()); + BOOST_TEST_THROWS( + pr.set_body_limit(3), + std::logic_error) + + pr.start(); + + // after completion of a bodiless message + { + pieces in = { + "HTTP/1.1 200 OK\r\n" + "content-length: 0\r\n" + "\r\n" }; + system::error_code ec; + read_header(pr, in, ec); + BOOST_TEST(! ec.failed()); + BOOST_TEST(pr.got_header()); + BOOST_TEST(pr.is_complete()); + BOOST_TEST(pr.body().empty()); + pr.set_body_limit(3); + } + } + void run() { @@ -2110,6 +2137,7 @@ struct parser_test testChunkedInPlace(); testMultipleMessageInPlace(); testMultipleMessageInPlaceChunked(); + testSetBodyLimit(); #else // For profiling for(int i = 0; i < 10000; ++i ) diff --git a/test/unit/zlib.cpp b/test/unit/zlib.cpp index 0a511fa3..e2ede0c1 100644 --- a/test/unit/zlib.cpp +++ b/test/unit/zlib.cpp @@ -551,7 +551,9 @@ struct zlib_test boost::system::error_code ec; pr.parse(ec); - if( ec ) + if(!ec) + pr.parse(ec); + if(ec) BOOST_TEST(ec == error::in_place_overflow || ec == error::need_data); @@ -739,6 +741,7 @@ struct zlib_test } pr.start(); + pr.set_body_limit(body_size); auto rs = receiver( pr, @@ -799,6 +802,9 @@ struct zlib_test boost::system::error_code ec; pr.parse(ec); + BOOST_TEST(! ec.failed()); + BOOST_TEST(pr.got_header()); + pr.parse(ec); BOOST_TEST_EQ(ec, zlib::error::version_err); }