diff --git a/proxygen/lib/http/codec/HTTPBinaryCodec.cpp b/proxygen/lib/http/codec/HTTPBinaryCodec.cpp index 2af41c1d22..a36cc24a49 100644 --- a/proxygen/lib/http/codec/HTTPBinaryCodec.cpp +++ b/proxygen/lib/http/codec/HTTPBinaryCodec.cpp @@ -11,6 +11,8 @@ #include #include +#include + #define HANDLE_ERROR_OR_WAITING_PARSE_RESULT(parseResult) \ if ((parseResult).parseResultState_ == ParseResultState::ERROR) { \ parseError_ = (parseResult).error_; \ @@ -55,6 +57,13 @@ HTTPBinaryCodec::HTTPBinaryCodec(TransportDirection direction, HTTPBinaryCodec::~HTTPBinaryCodec() = default; +void HTTPBinaryCodec::setBodyStreamingEnabled(bool enabled) { + if (state_ != ParseState::FRAMING_INDICATOR || !bufferedIngress_.empty()) { + return; + } + bodyStreamingEnabled_ = enabled; +} + ParseResult HTTPBinaryCodec::parseFramingIndicator(folly::io::Cursor& cursor, bool& request, bool& knownLength) { @@ -86,6 +95,7 @@ ParseResult HTTPBinaryCodec::parseFramingIndicator(folly::io::Cursor& cursor, ParseResult HTTPBinaryCodec::parseKnownLengthString( folly::io::Cursor& cursor, size_t remaining, + uint64_t maxLength, folly::StringPiece stringName, std::string& stringValue) { size_t parsed = 0; @@ -97,6 +107,14 @@ ParseResult HTTPBinaryCodec::parseKnownLengthString( } // Increase parsed by the number of bytes read parsed += encodedStringLength->second; + if (encodedStringLength->first > maxLength) { + return ParseResult( + fmt::format("Failure to parse: {} of declared length {} exceeds the " + "remaining field section budget of {}", + stringName, + encodedStringLength->first, + maxLength)); + } // If this would cause us to go beyond "remaining", we need to wait for more // data if (encodedStringLength->first > remaining - parsed) { @@ -124,7 +142,8 @@ ParseResult HTTPBinaryCodec::parseRequestControlData(folly::io::Cursor& cursor, // Parse method std::string method; - auto methodRes = parseKnownLengthString(cursor, remaining, "method", method); + auto methodRes = parseKnownLengthString( + cursor, remaining, maxFieldSectionSize_, "method", method); if (methodRes.parseResultState_ == ParseResultState::ERROR || methodRes.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { return methodRes; @@ -135,7 +154,8 @@ ParseResult HTTPBinaryCodec::parseRequestControlData(folly::io::Cursor& cursor, // Parse scheme std::string scheme; - auto schemeRes = parseKnownLengthString(cursor, remaining, "scheme", scheme); + auto schemeRes = parseKnownLengthString( + cursor, remaining, maxFieldSectionSize_, "scheme", scheme); if (schemeRes.parseResultState_ == ParseResultState::ERROR || schemeRes.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { return schemeRes; @@ -153,8 +173,8 @@ ParseResult HTTPBinaryCodec::parseRequestControlData(folly::io::Cursor& cursor, // Parse authority std::string authority; - auto authorityRes = - parseKnownLengthString(cursor, remaining, "authority", authority); + auto authorityRes = parseKnownLengthString( + cursor, remaining, maxFieldSectionSize_, "authority", authority); if (authorityRes.parseResultState_ == ParseResultState::ERROR || authorityRes.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { @@ -165,7 +185,8 @@ ParseResult HTTPBinaryCodec::parseRequestControlData(folly::io::Cursor& cursor, // Parse path std::string path; - auto pathRes = parseKnownLengthString(cursor, remaining, "path", path); + auto pathRes = parseKnownLengthString( + cursor, remaining, maxFieldSectionSize_, "path", path); if (pathRes.parseResultState_ == ParseResultState::ERROR || pathRes.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { return pathRes; @@ -205,10 +226,11 @@ ParseResult HTTPBinaryCodec::parseSingleHeaderHelper( HeaderDecodeInfo& decodeInfo, size_t& parsed, size_t& remaining, + uint64_t maxLength, size_t& numHeaders) { std::string headerName; - auto headerNameRes = - parseKnownLengthString(cursor, remaining, "headerName", headerName); + auto headerNameRes = parseKnownLengthString( + cursor, remaining, maxLength, "headerName", headerName); if (headerNameRes.parseResultState_ == ParseResultState::ERROR || headerNameRes.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { @@ -216,10 +238,11 @@ ParseResult HTTPBinaryCodec::parseSingleHeaderHelper( } parsed += headerNameRes.bytesParsed_; remaining -= headerNameRes.bytesParsed_; + maxLength -= std::min(maxLength, headerNameRes.bytesParsed_); std::string headerValue; - auto headerValueRes = - parseKnownLengthString(cursor, remaining, "headerValue", headerValue); + auto headerValueRes = parseKnownLengthString( + cursor, remaining, maxLength, "headerValue", headerValue); if (headerValueRes.parseResultState_ == ParseResultState::ERROR || headerValueRes.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { @@ -257,14 +280,28 @@ ParseResult HTTPBinaryCodec::parseKnownLengthHeadersHelper( // Increase parsed and decrease remaining by the number of bytes read parsed += lengthOfHeaders->second; remaining -= lengthOfHeaders->second; + // Reject an oversized declaration before buffering anything for it, rather + // than waiting for the whole section to arrive + if (lengthOfHeaders->first > maxFieldSectionSize_) { + return ParseResult( + fmt::format("Declared {} section length {} exceeds the maximum field " + "section size of {}", + isTrailers ? "trailer" : "header", + lengthOfHeaders->first, + maxFieldSectionSize_)); + } if (remaining < lengthOfHeaders->first) { return ParseResult(ParseResultState::WAITING_FOR_MORE_DATA); } size_t numHeaders = 0; while (parsed < lengthOfHeaders->first) { - auto result = parseSingleHeaderHelper( - cursor, decodeInfo, parsed, remaining, numHeaders); + auto result = parseSingleHeaderHelper(cursor, + decodeInfo, + parsed, + remaining, + maxFieldSectionSize_, + numHeaders); if (result.parseResultState_ == ParseResultState::ERROR || result.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { return result; @@ -285,12 +322,26 @@ ParseResult HTTPBinaryCodec::parseIndeterminateLengthHeadersHelper( size_t numHeaders = 0; // Continue parsing headers until we reach the Content Terminator field (0) while (currentByte != nullptr && *currentByte != 0x00) { + // A field whose declared length cannot fit in what is left of the section + // has to be rejected on sight: waiting for it to arrive would retain the + // fields already parsed plus the whole declaration + const uint64_t sectionBudget = + maxFieldSectionSize_ - std::min(parsed, maxFieldSectionSize_); auto result = parseSingleHeaderHelper( - cursor, decodeInfo, parsed, remaining, numHeaders); + cursor, decodeInfo, parsed, remaining, sectionBudget, numHeaders); if (result.parseResultState_ == ParseResultState::ERROR || result.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { return result; } + // There is no declared length to check against, so bound the section by + // what has actually been seen + if (parsed > maxFieldSectionSize_) { + return ParseResult( + fmt::format("Unterminated {} section exceeds the maximum field " + "section size of {}", + isTrailers ? "trailer" : "header", + maxFieldSectionSize_)); + } // If we have reached the end of the cursor at this point, we must be // waiting for more data since we haven't seen a Content Terminator field // yet @@ -323,45 +374,74 @@ ParseResult HTTPBinaryCodec::parseContent(folly::io::Cursor& cursor, : parseIndeterminateLengthContentHelper(cursor, remaining); } -ParseResult HTTPBinaryCodec::parseSingleContentHelper(folly::io::Cursor& cursor, - size_t remaining) { - size_t parsed = 0; - - // Parse the contentLength and advance cursor +ParseResult HTTPBinaryCodec::parseBufferedContentHelper( + folly::io::Cursor& cursor, size_t remaining) { auto contentLength = quic::follyutils::decodeQuicInteger(cursor); - if (!contentLength) { - return ParseResult(ParseResultState::WAITING_FOR_MORE_DATA); - } - // Check that we had enough bytes to parse contentLength, otherwise the - // "remaining - parsed" below underflows - if (remaining < contentLength->second) { + if (!contentLength || remaining < contentLength->second) { return ParseResult(ParseResultState::WAITING_FOR_MORE_DATA); } - // Increase parsed by the number of bytes read - parsed += contentLength->second; + + const size_t parsed = contentLength->second; if (contentLength->first == 0) { return ParseResult(parsed); } - // Check that we have not gone beyond "remaining" if (contentLength->first > remaining - parsed) { return ParseResult(ParseResultState::WAITING_FOR_MORE_DATA); } - // Write the data to msgBody_ and then advance the cursor msgBody_ = std::make_unique(); - if (contentLength->first > 0 && msgBody_) { - cursor.cloneAtMost(*msgBody_.get(), contentLength->first); + cursor.cloneAtMost(*msgBody_, contentLength->first); + return ParseResult(parsed + contentLength->first); +} + +ParseResult HTTPBinaryCodec::parseSingleContentHelper(folly::io::Cursor& cursor, + size_t remaining) { + if (!bodyStreamingEnabled_) { + return parseBufferedContentHelper(cursor, remaining); } - // Increase parsed by the number of bytes read - parsed += contentLength->first; + size_t parsed = 0; + + // A chunk whose length prefix was consumed on an earlier call resumes here + if (!remainingContentLength_) { + // Parse the contentLength and advance cursor + auto contentLength = quic::follyutils::decodeQuicInteger(cursor); + if (!contentLength) { + return ParseResult(ParseResultState::WAITING_FOR_MORE_DATA); + } + // Check that we had enough bytes to parse contentLength + if (remaining < contentLength->second) { + return ParseResult(ParseResultState::WAITING_FOR_MORE_DATA); + } + // Increase parsed and decrease remaining by the number of bytes read + parsed += contentLength->second; + remaining -= contentLength->second; + if (contentLength->first == 0) { + return ParseResult(parsed); + } + remainingContentLength_ = contentLength->first; + } + + const size_t available = + std::min(*remainingContentLength_, remaining); + if (available > 0) { + msgBody_ = std::make_unique(); + cursor.cloneAtMost(*msgBody_, available); + parsed += available; + *remainingContentLength_ -= available; + } + + if (*remainingContentLength_ > 0) { + return ParseResult(parsed, ParseResultState::WAITING_FOR_MORE_DATA); + } + remainingContentLength_.reset(); return ParseResult(parsed); } ParseResult HTTPBinaryCodec::parseKnownLengthContentHelper( folly::io::Cursor& cursor, size_t remaining) { auto parseResult = parseSingleContentHelper(cursor, remaining); - if (parseResult.parseResultState_ == ParseResultState::DONE && msgBody_ && + if (parseResult.parseResultState_ != ParseResultState::ERROR && msgBody_ && callback_) { callback_->onBody(ingressTxnID_, std::move(msgBody_), 0); } @@ -381,17 +461,17 @@ ParseResult HTTPBinaryCodec::parseIndeterminateLengthContentHelper( if (parseResult.parseResultState_ == ParseResultState::ERROR) { return parseResult; } - // Report the bytes of the chunks already handed to the callback, otherwise - // they are not trimmed and are parsed and delivered a second time - if (parseResult.parseResultState_ == - ParseResultState::WAITING_FOR_MORE_DATA) { - return ParseResult(parsed, ParseResultState::WAITING_FOR_MORE_DATA); - } - // After successfully processing a body chunk, we call onBody + // After processing a body chunk, whole or partial, we call onBody parsed += parseResult.bytesParsed_; if (msgBody_ && callback_) { callback_->onBody(ingressTxnID_, std::move(msgBody_), 0); } + // Report the bytes already handed to the callback so that they are not + // parsed and delivered a second time once the chunk resumes + if (parseResult.parseResultState_ == + ParseResultState::WAITING_FOR_MORE_DATA) { + return ParseResult(parsed, ParseResultState::WAITING_FOR_MORE_DATA); + } // If we have reached the end of the cursor at this point, we must // be waiting for more data since we haven't seen a Content // Terminator field yet @@ -563,7 +643,8 @@ size_t HTTPBinaryCodec::onIngress(const folly::IOBuf& buf) { } void HTTPBinaryCodec::onIngressEOF() { - if (!parseError_ && !bufferedIngress_.empty()) { + if (!parseError_ && + (!bufferedIngress_.empty() || remainingContentLength_.value_or(0) > 0)) { // Case where the ingress EOF is received before the entire message is // parsed callback_->onError(ingressTxnID_, diff --git a/proxygen/lib/http/codec/HTTPBinaryCodec.h b/proxygen/lib/http/codec/HTTPBinaryCodec.h index 816da4b05f..582bc7fc18 100644 --- a/proxygen/lib/http/codec/HTTPBinaryCodec.h +++ b/proxygen/lib/http/codec/HTTPBinaryCodec.h @@ -9,6 +9,7 @@ #pragma once #include +#include #include #include #include @@ -53,6 +54,8 @@ class ParseResult { */ class HTTPBinaryCodec : public HTTPCodec { public: + static constexpr uint64_t kDefaultMaxFieldSectionSize = uint64_t{128} * 1024; + // Default strictValidation to false for now to match existing behavior explicit HTTPBinaryCodec(TransportDirection direction); HTTPBinaryCodec(TransportDirection direction, bool knownEgressLength); @@ -78,6 +81,18 @@ class HTTPBinaryCodec : public HTTPCodec { void setCallback(Callback* callback) override { callback_ = callback; } + void setMaxFieldSectionSize(uint64_t size) { + maxFieldSectionSize_ = + (size == 0) ? std::numeric_limits::max() : size; + } + uint64_t getMaxFieldSectionSize() const { + return maxFieldSectionSize_; + } + // Ignored while ingress parsing is in progress. + void setBodyStreamingEnabled(bool enabled); + bool getBodyStreamingEnabled() const { + return bodyStreamingEnabled_; + } bool isBusy() const override { return false; } @@ -169,6 +184,7 @@ class HTTPBinaryCodec : public HTTPCodec { bool& knownLength); ParseResult parseKnownLengthString(folly::io::Cursor& cursor, size_t remaining, + uint64_t maxLength, folly::StringPiece stringName, std::string& stringValue); ParseResult parseRequestControlData(folly::io::Cursor& cursor, @@ -182,6 +198,8 @@ class HTTPBinaryCodec : public HTTPCodec { HeaderDecodeInfo& decodeInfo, bool knownLength); ParseResult parseContent(folly::io::Cursor& cursor, size_t remaining); + ParseResult parseBufferedContentHelper(folly::io::Cursor& cursor, + size_t remaining); ParseResult parseSingleContentHelper(folly::io::Cursor& cursor, size_t remaining); ParseResult parseKnownLengthContentHelper(folly::io::Cursor& cursor, @@ -195,6 +213,7 @@ class HTTPBinaryCodec : public HTTPCodec { HeaderDecodeInfo& decodeInfo, size_t& parsed, size_t& remaining, + uint64_t maxLength, size_t& numHeaders); ParseResult parseKnownLengthHeadersHelper(folly::io::Cursor& cursor, size_t remaining, @@ -211,6 +230,8 @@ class HTTPBinaryCodec : public HTTPCodec { bool isRequest_{true}; bool knownEgressLength_{true}; bool knownIngressLength_{false}; + bool bodyStreamingEnabled_{true}; + uint64_t maxFieldSectionSize_{kDefaultMaxFieldSectionSize}; enum class ParseState : uint8_t { FRAMING_INDICATOR = 0, INFORMATIONAL_RESPONSE = 1, @@ -240,10 +261,13 @@ class HTTPBinaryCodec : public HTTPCodec { StreamID ingressTxnID_; folly::IOBufQueue bufferedIngress_{folly::IOBufQueue::cacheChainLength()}; - // We don't need to use an IOBufQueue for msgBody_ since we are writing a - // complete message to it and don't need to rely on the efficient size - // computation provided by IOBufQueue + // The piece of the current content chunk that is about to be handed to + // onBody. Content is delivered incrementally, so this holds one piece rather + // than the whole chunk. std::unique_ptr msgBody_; + // Bytes still outstanding for the content chunk being delivered, set while a + // chunk has only partially arrived so that later ingress resumes mid-chunk + folly::Optional remainingContentLength_; HeaderDecodeInfo decodeInfo_; std::unique_ptr msg_; diff --git a/proxygen/lib/http/codec/test/HTTPBinaryCodecTest.cpp b/proxygen/lib/http/codec/test/HTTPBinaryCodecTest.cpp index 907a3166ce..3dcefa8e86 100644 --- a/proxygen/lib/http/codec/test/HTTPBinaryCodecTest.cpp +++ b/proxygen/lib/http/codec/test/HTTPBinaryCodecTest.cpp @@ -57,6 +57,10 @@ class HTTPBinaryCodecForTest : public HTTPBinaryCodec { folly::IOBuf& getMsgBody() { return *msgBody_; } + + size_t getBufferedIngressSize() const { + return bufferedIngress_.chainLength(); + } }; namespace { @@ -85,6 +89,17 @@ void writeRequestPreamble(folly::io::QueueAppender& appender, writeVarintString(appender, "/"); } +std::unique_ptr makeFiller(size_t size) { + auto buf = folly::IOBuf::create(size); + buf->append(size); + memset(buf->writableData(), 'a', size); + return buf; +} + +constexpr size_t kFillerChunkSize = size_t{64} * 1024; +constexpr size_t kFillerChunkCount = 16; +constexpr uint64_t kAbsurdDeclaredLength = uint64_t(1) << 40; + } // namespace template @@ -1225,4 +1240,351 @@ TEST_F(HttpBinaryDownstreamCodecTest, testPartialChunkIsNotOverCounted) { EXPECT_EQ(callback.data_.move()->to(), first + second); } +TEST_F(HttpBinaryDownstreamCodecTest, + testSplitContentChunkIsDeliveredOnceWithStreamingDisabled) { + const std::string body(512, 'a'); + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 1024); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + writeVarint(appender, 0 /* empty field section */); + writeVarintString(appender, "first-"); + writeVarintString(appender, body); + + auto head = message.split(message.chainLength() - 256); + auto tail = message.move(); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setBodyStreamingEnabled(false); + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*head); + binaryCodecIndeterminateLength_->onIngress(*tail); + + EXPECT_EQ(callback.lastParseError, nullptr); + EXPECT_EQ(callback.bodyLength, body.size() + 6); + EXPECT_EQ(callback.data_.move()->to(), "first-" + body); +} + +TEST_F(HttpBinaryDownstreamCodecTest, + testPartialChunkIsNotOverCountedWithStreamingDisabled) { + const std::string first(200, 'a'); + const std::string second(200, 'b'); + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 1024); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + writeVarint(appender, 0 /* empty field section */); + writeVarintString(appender, first); + writeVarintString(appender, second); + + auto head = message.split(message.chainLength() - 100); + auto tail = message.move(); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setBodyStreamingEnabled(false); + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*head); + binaryCodecIndeterminateLength_->onIngress(*tail); + + EXPECT_EQ(callback.lastParseError, nullptr); + EXPECT_EQ(callback.bodyLength, first.size() + second.size()); + EXPECT_EQ(callback.data_.move()->to(), first + second); +} + +TEST_F(HttpBinaryDownstreamCodecTest, + testBodyStreamingSettingCannotChangeMidChunk) { + const std::string body(512, 'a'); + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 1024); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + writeVarint(appender, 0 /* empty field section */); + writeVarintString(appender, body); + + auto head = message.split(message.chainLength() - 256); + auto tail = message.move(); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*head); + binaryCodecIndeterminateLength_->setBodyStreamingEnabled(false); + + EXPECT_TRUE(binaryCodecIndeterminateLength_->getBodyStreamingEnabled()); + + binaryCodecIndeterminateLength_->onIngress(*tail); + + EXPECT_EQ(callback.lastParseError, nullptr); + EXPECT_EQ(callback.bodyLength, body.size()); + EXPECT_EQ(callback.data_.move()->to(), body); +} + +// Field section lengths are QUIC varints, so a peer can declare up to 2^62-1 +// bytes. The declaration must be rejected up front instead of buffering +// ingress until the section is complete. +TEST_F(HttpBinaryDownstreamCodecTest, testDeclaredFieldSectionSizeIsBounded) { + folly::IOBufQueue preamble{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&preamble, 128); + writeRequestPreamble(appender, 0 /* request, known length */); + writeVarint(appender, kAbsurdDeclaredLength); + + FakeHTTPCodecCallback callback; + binaryCodecKnownLength_->setCallback(&callback); + binaryCodecKnownLength_->onIngress(*preamble.front()); + + auto filler = makeFiller(kFillerChunkSize); + for (size_t i = 0; i < kFillerChunkCount && !callback.lastParseError; i++) { + binaryCodecKnownLength_->onIngress(*filler); + } + + EXPECT_NE(callback.lastParseError, nullptr); + EXPECT_LT(binaryCodecKnownLength_->getBufferedIngressSize(), + kFillerChunkSize * kFillerChunkCount); +} + +// The same bound applies to an individual field, whose length is also a varint +// and which is buffered whole before the name/value pair is validated. +TEST_F(HttpBinaryDownstreamCodecTest, testDeclaredFieldSizeIsBounded) { + folly::IOBufQueue preamble{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&preamble, 128); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + writeVarintString(appender, "x-pad"); + writeVarint(appender, kAbsurdDeclaredLength); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*preamble.front()); + + auto filler = makeFiller(kFillerChunkSize); + for (size_t i = 0; i < kFillerChunkCount && !callback.lastParseError; i++) { + binaryCodecIndeterminateLength_->onIngress(*filler); + } + + EXPECT_NE(callback.lastParseError, nullptr); + EXPECT_LT(binaryCodecIndeterminateLength_->getBufferedIngressSize(), + kFillerChunkSize * kFillerChunkCount); +} + +// An indeterminate-length field section is terminated by a zero byte rather +// than a declared length, so it is bounded by the bytes actually seen. +TEST_F(HttpBinaryDownstreamCodecTest, + testIndeterminateFieldSectionSizeIsBounded) { + folly::IOBufQueue preamble{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&preamble, 128); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*preamble.front()); + + // A stream of well-formed fields that never reaches the terminator + folly::IOBufQueue fields{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender fieldsAppender(&fields, 1024); + const std::string padValue(64, 'a'); + for (size_t i = 0; i < 512; i++) { + writeVarintString(fieldsAppender, "x-pad"); + writeVarintString(fieldsAppender, padValue); + } + auto fieldsBuf = fields.move(); + + for (size_t i = 0; i < kFillerChunkCount && !callback.lastParseError; i++) { + binaryCodecIndeterminateLength_->onIngress(*fieldsBuf); + } + + EXPECT_NE(callback.lastParseError, nullptr); +} + +TEST_F(HttpBinaryDownstreamCodecTest, testMaxFieldSectionSizeIsConfigurable) { + const std::string headerValue(256, 'a'); + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 512); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + writeVarintString(appender, "x-pad"); + writeVarintString(appender, headerValue); + writeVarint(appender, 0 /* field section terminator */); + auto messageBuf = message.move(); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setMaxFieldSectionSize(64); + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*messageBuf); + + EXPECT_NE(callback.lastParseError, nullptr); +} + +// A field is bounded by what is left of the section, not by the section +// maximum, so a declaration that only fits when the fields already parsed are +// ignored is rejected instead of being waited on. +TEST_F(HttpBinaryDownstreamCodecTest, testFieldIsBoundedByRemainingSection) { + constexpr size_t kMaxFieldSectionSize = 4096; + constexpr size_t kPadFieldCount = 3; + + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 1024); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + const std::string padValue(1024, 'a'); + for (size_t i = 0; i < kPadFieldCount; i++) { + writeVarintString(appender, "x-pad"); + writeVarintString(appender, padValue); + } + // Comfortably under the maximum on its own, but well over the ~1000 bytes + // the fields above have left of it. None of the declared bytes follow. + writeVarintString(appender, "x-last"); + writeVarint(appender, kMaxFieldSectionSize / 2); + auto messageBuf = message.move(); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setMaxFieldSectionSize(kMaxFieldSectionSize); + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*messageBuf); + + EXPECT_NE(callback.lastParseError, nullptr); +} + +// Setting the maximum to 0 turns enforcement off, so a declaration that would +// otherwise be rejected is accepted again. This is the runtime escape hatch. +TEST_F(HttpBinaryDownstreamCodecTest, testMaxFieldSectionSizeZeroDisables) { + folly::IOBufQueue preamble{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&preamble, 128); + writeRequestPreamble(appender, 0 /* request, known length */); + writeVarint(appender, kAbsurdDeclaredLength); + + FakeHTTPCodecCallback callback; + binaryCodecKnownLength_->setMaxFieldSectionSize(0); + EXPECT_EQ(binaryCodecKnownLength_->getMaxFieldSectionSize(), + std::numeric_limits::max()); + binaryCodecKnownLength_->setCallback(&callback); + binaryCodecKnownLength_->onIngress(*preamble.front()); + + // Deliberately well under the buffered-ingress ceiling added later in this + // stack: disabling the field section limit does not disable that one, and + // this test is about the field section limit alone + auto filler = makeFiller(kFillerChunkSize); + for (size_t i = 0; i < kFillerChunkCount / 4; i++) { + binaryCodecKnownLength_->onIngress(*filler); + } + + // Enforcement is off, so the codec waits for the declared section instead of + // rejecting it -- the pre-ceiling behaviour + EXPECT_EQ(callback.lastParseError, nullptr); +} + +// A declared content length is also a varint. The codec must hand body bytes +// to the callback as they arrive rather than buffering the whole chunk. +TEST_F(HttpBinaryDownstreamCodecTest, testLargeKnownLengthContentIsStreamed) { + folly::IOBufQueue preamble{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&preamble, 128); + writeRequestPreamble(appender, 0 /* request, known length */); + writeVarint(appender, 0 /* empty field section */); + writeVarint(appender, kAbsurdDeclaredLength); + + FakeHTTPCodecCallback callback; + binaryCodecKnownLength_->setCallback(&callback); + binaryCodecKnownLength_->onIngress(*preamble.front()); + + auto filler = makeFiller(kFillerChunkSize); + for (size_t i = 0; i < kFillerChunkCount; i++) { + binaryCodecKnownLength_->onIngress(*filler); + } + + EXPECT_EQ(callback.lastParseError, nullptr); + EXPECT_EQ(callback.bodyLength, kFillerChunkSize * kFillerChunkCount); + EXPECT_LT(binaryCodecKnownLength_->getBufferedIngressSize(), + kFillerChunkSize); +} + +TEST_F(HttpBinaryDownstreamCodecTest, + testLargeIndeterminateLengthContentIsStreamed) { + folly::IOBufQueue preamble{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&preamble, 128); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + writeVarint(appender, 0 /* empty field section */); + writeVarint(appender, kAbsurdDeclaredLength); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*preamble.front()); + + auto filler = makeFiller(kFillerChunkSize); + for (size_t i = 0; i < kFillerChunkCount; i++) { + binaryCodecIndeterminateLength_->onIngress(*filler); + } + + EXPECT_EQ(callback.lastParseError, nullptr); + EXPECT_EQ(callback.bodyLength, kFillerChunkSize * kFillerChunkCount); + EXPECT_LT(binaryCodecIndeterminateLength_->getBufferedIngressSize(), + kFillerChunkSize); +} + +TEST_F(HttpBinaryDownstreamCodecTest, + testKnownLengthContentStreamingCanBeDisabled) { + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 128); + writeRequestPreamble(appender, 0 /* request, known length */); + writeVarint(appender, 0 /* empty field section */); + writeVarint(appender, 64 /* declared content length */); + const std::string firstHalf(32, 'a'); + appender.pushAtMost(reinterpret_cast(firstHalf.data()), + firstHalf.size()); + + FakeHTTPCodecCallback callback; + binaryCodecKnownLength_->setBodyStreamingEnabled(false); + binaryCodecKnownLength_->setCallback(&callback); + binaryCodecKnownLength_->onIngress(*message.front()); + + EXPECT_EQ(callback.bodyLength, 0); + + const std::string secondHalf(32, 'b'); + auto tail = folly::IOBuf::copyBuffer(secondHalf); + binaryCodecKnownLength_->onIngress(*tail); + + EXPECT_EQ(callback.bodyLength, 64); + EXPECT_EQ(callback.data_.move()->to(), firstHalf + secondHalf); +} + +TEST_F(HttpBinaryDownstreamCodecTest, + testIndeterminateLengthContentStreamingCanBeDisabled) { + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 128); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + writeVarint(appender, 0 /* empty field section */); + writeVarint(appender, 64 /* declared content length */); + const std::string firstHalf(32, 'a'); + appender.pushAtMost(reinterpret_cast(firstHalf.data()), + firstHalf.size()); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setBodyStreamingEnabled(false); + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*message.front()); + + EXPECT_EQ(callback.bodyLength, 0); + + const std::string secondHalf(32, 'b'); + auto tail = folly::IOBuf::copyBuffer(secondHalf); + binaryCodecIndeterminateLength_->onIngress(*tail); + + EXPECT_EQ(callback.bodyLength, 64); + EXPECT_EQ(callback.data_.move()->to(), firstHalf + secondHalf); +} + +// Content is trimmed from bufferedIngress_ as it is delivered, so a truncated +// chunk must still be reported as an incomplete message at EOF +TEST_F(HttpBinaryDownstreamCodecTest, testTruncatedContentIsIncompleteAtEOF) { + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 128); + writeRequestPreamble(appender, 0 /* request, known length */); + writeVarint(appender, 0 /* empty field section */); + writeVarint(appender, 64 /* declared content length */); + const std::string partialContent(32, 'a'); + appender.pushAtMost(reinterpret_cast(partialContent.data()), + partialContent.size()); + auto messageBuf = message.move(); + + FakeHTTPCodecCallback callback; + binaryCodecKnownLength_->setCallback(&callback); + binaryCodecKnownLength_->onIngress(*messageBuf); + binaryCodecKnownLength_->onIngressEOF(); + + ASSERT_NE(callback.lastParseError, nullptr); + EXPECT_EQ(std::string(callback.lastParseError->what()), + "Incomplete message received"); +} + } // namespace proxygen::test