From 655b6e0e348bc04d0de06e9a0bdf21f4645e0268 Mon Sep 17 00:00:00 2001 From: Facebook GitHub Bot Date: Thu, 3 Sep 2026 16:29:42 -0700 Subject: [PATCH] Re-sync with internal repository The internal and external repositories are out of sync. This Pull Request attempts to brings them back in sync by patching the GitHub repository. Please carefully review this patch. You must disable ShipIt for your project in order to merge this pull request. DO NOT IMPORT this pull request. Instead, merge it directly on GitHub using the MERGE BUTTON. Re-enable ShipIt after merging. fbshipit-source-id: 21b26a072fd57b8b3085079c223707b7010cea94 --- proxygen/lib/http/codec/HTTPBinaryCodec.cpp | 161 ++++++-- proxygen/lib/http/codec/HTTPBinaryCodec.h | 30 +- .../http/codec/test/HTTPBinaryCodecTest.cpp | 362 ++++++++++++++++++ 3 files changed, 510 insertions(+), 43 deletions(-) diff --git a/proxygen/lib/http/codec/HTTPBinaryCodec.cpp b/proxygen/lib/http/codec/HTTPBinaryCodec.cpp index 2af41c1d22..a36cc24a49 100644 --- a/proxygen/lib/http/codec/HTTPBinaryCodec.cpp +++ b/proxygen/lib/http/codec/HTTPBinaryCodec.cpp @@ -11,6 +11,8 @@ #include #include +#include + #define HANDLE_ERROR_OR_WAITING_PARSE_RESULT(parseResult) \ if ((parseResult).parseResultState_ == ParseResultState::ERROR) { \ parseError_ = (parseResult).error_; \ @@ -55,6 +57,13 @@ HTTPBinaryCodec::HTTPBinaryCodec(TransportDirection direction, HTTPBinaryCodec::~HTTPBinaryCodec() = default; +void HTTPBinaryCodec::setBodyStreamingEnabled(bool enabled) { + if (state_ != ParseState::FRAMING_INDICATOR || !bufferedIngress_.empty()) { + return; + } + bodyStreamingEnabled_ = enabled; +} + ParseResult HTTPBinaryCodec::parseFramingIndicator(folly::io::Cursor& cursor, bool& request, bool& knownLength) { @@ -86,6 +95,7 @@ ParseResult HTTPBinaryCodec::parseFramingIndicator(folly::io::Cursor& cursor, ParseResult HTTPBinaryCodec::parseKnownLengthString( folly::io::Cursor& cursor, size_t remaining, + uint64_t maxLength, folly::StringPiece stringName, std::string& stringValue) { size_t parsed = 0; @@ -97,6 +107,14 @@ ParseResult HTTPBinaryCodec::parseKnownLengthString( } // Increase parsed by the number of bytes read parsed += encodedStringLength->second; + if (encodedStringLength->first > maxLength) { + return ParseResult( + fmt::format("Failure to parse: {} of declared length {} exceeds the " + "remaining field section budget of {}", + stringName, + encodedStringLength->first, + maxLength)); + } // If this would cause us to go beyond "remaining", we need to wait for more // data if (encodedStringLength->first > remaining - parsed) { @@ -124,7 +142,8 @@ ParseResult HTTPBinaryCodec::parseRequestControlData(folly::io::Cursor& cursor, // Parse method std::string method; - auto methodRes = parseKnownLengthString(cursor, remaining, "method", method); + auto methodRes = parseKnownLengthString( + cursor, remaining, maxFieldSectionSize_, "method", method); if (methodRes.parseResultState_ == ParseResultState::ERROR || methodRes.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { return methodRes; @@ -135,7 +154,8 @@ ParseResult HTTPBinaryCodec::parseRequestControlData(folly::io::Cursor& cursor, // Parse scheme std::string scheme; - auto schemeRes = parseKnownLengthString(cursor, remaining, "scheme", scheme); + auto schemeRes = parseKnownLengthString( + cursor, remaining, maxFieldSectionSize_, "scheme", scheme); if (schemeRes.parseResultState_ == ParseResultState::ERROR || schemeRes.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { return schemeRes; @@ -153,8 +173,8 @@ ParseResult HTTPBinaryCodec::parseRequestControlData(folly::io::Cursor& cursor, // Parse authority std::string authority; - auto authorityRes = - parseKnownLengthString(cursor, remaining, "authority", authority); + auto authorityRes = parseKnownLengthString( + cursor, remaining, maxFieldSectionSize_, "authority", authority); if (authorityRes.parseResultState_ == ParseResultState::ERROR || authorityRes.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { @@ -165,7 +185,8 @@ ParseResult HTTPBinaryCodec::parseRequestControlData(folly::io::Cursor& cursor, // Parse path std::string path; - auto pathRes = parseKnownLengthString(cursor, remaining, "path", path); + auto pathRes = parseKnownLengthString( + cursor, remaining, maxFieldSectionSize_, "path", path); if (pathRes.parseResultState_ == ParseResultState::ERROR || pathRes.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { return pathRes; @@ -205,10 +226,11 @@ ParseResult HTTPBinaryCodec::parseSingleHeaderHelper( HeaderDecodeInfo& decodeInfo, size_t& parsed, size_t& remaining, + uint64_t maxLength, size_t& numHeaders) { std::string headerName; - auto headerNameRes = - parseKnownLengthString(cursor, remaining, "headerName", headerName); + auto headerNameRes = parseKnownLengthString( + cursor, remaining, maxLength, "headerName", headerName); if (headerNameRes.parseResultState_ == ParseResultState::ERROR || headerNameRes.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { @@ -216,10 +238,11 @@ ParseResult HTTPBinaryCodec::parseSingleHeaderHelper( } parsed += headerNameRes.bytesParsed_; remaining -= headerNameRes.bytesParsed_; + maxLength -= std::min(maxLength, headerNameRes.bytesParsed_); std::string headerValue; - auto headerValueRes = - parseKnownLengthString(cursor, remaining, "headerValue", headerValue); + auto headerValueRes = parseKnownLengthString( + cursor, remaining, maxLength, "headerValue", headerValue); if (headerValueRes.parseResultState_ == ParseResultState::ERROR || headerValueRes.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { @@ -257,14 +280,28 @@ ParseResult HTTPBinaryCodec::parseKnownLengthHeadersHelper( // Increase parsed and decrease remaining by the number of bytes read parsed += lengthOfHeaders->second; remaining -= lengthOfHeaders->second; + // Reject an oversized declaration before buffering anything for it, rather + // than waiting for the whole section to arrive + if (lengthOfHeaders->first > maxFieldSectionSize_) { + return ParseResult( + fmt::format("Declared {} section length {} exceeds the maximum field " + "section size of {}", + isTrailers ? "trailer" : "header", + lengthOfHeaders->first, + maxFieldSectionSize_)); + } if (remaining < lengthOfHeaders->first) { return ParseResult(ParseResultState::WAITING_FOR_MORE_DATA); } size_t numHeaders = 0; while (parsed < lengthOfHeaders->first) { - auto result = parseSingleHeaderHelper( - cursor, decodeInfo, parsed, remaining, numHeaders); + auto result = parseSingleHeaderHelper(cursor, + decodeInfo, + parsed, + remaining, + maxFieldSectionSize_, + numHeaders); if (result.parseResultState_ == ParseResultState::ERROR || result.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { return result; @@ -285,12 +322,26 @@ ParseResult HTTPBinaryCodec::parseIndeterminateLengthHeadersHelper( size_t numHeaders = 0; // Continue parsing headers until we reach the Content Terminator field (0) while (currentByte != nullptr && *currentByte != 0x00) { + // A field whose declared length cannot fit in what is left of the section + // has to be rejected on sight: waiting for it to arrive would retain the + // fields already parsed plus the whole declaration + const uint64_t sectionBudget = + maxFieldSectionSize_ - std::min(parsed, maxFieldSectionSize_); auto result = parseSingleHeaderHelper( - cursor, decodeInfo, parsed, remaining, numHeaders); + cursor, decodeInfo, parsed, remaining, sectionBudget, numHeaders); if (result.parseResultState_ == ParseResultState::ERROR || result.parseResultState_ == ParseResultState::WAITING_FOR_MORE_DATA) { return result; } + // There is no declared length to check against, so bound the section by + // what has actually been seen + if (parsed > maxFieldSectionSize_) { + return ParseResult( + fmt::format("Unterminated {} section exceeds the maximum field " + "section size of {}", + isTrailers ? "trailer" : "header", + maxFieldSectionSize_)); + } // If we have reached the end of the cursor at this point, we must be // waiting for more data since we haven't seen a Content Terminator field // yet @@ -323,45 +374,74 @@ ParseResult HTTPBinaryCodec::parseContent(folly::io::Cursor& cursor, : parseIndeterminateLengthContentHelper(cursor, remaining); } -ParseResult HTTPBinaryCodec::parseSingleContentHelper(folly::io::Cursor& cursor, - size_t remaining) { - size_t parsed = 0; - - // Parse the contentLength and advance cursor +ParseResult HTTPBinaryCodec::parseBufferedContentHelper( + folly::io::Cursor& cursor, size_t remaining) { auto contentLength = quic::follyutils::decodeQuicInteger(cursor); - if (!contentLength) { - return ParseResult(ParseResultState::WAITING_FOR_MORE_DATA); - } - // Check that we had enough bytes to parse contentLength, otherwise the - // "remaining - parsed" below underflows - if (remaining < contentLength->second) { + if (!contentLength || remaining < contentLength->second) { return ParseResult(ParseResultState::WAITING_FOR_MORE_DATA); } - // Increase parsed by the number of bytes read - parsed += contentLength->second; + + const size_t parsed = contentLength->second; if (contentLength->first == 0) { return ParseResult(parsed); } - // Check that we have not gone beyond "remaining" if (contentLength->first > remaining - parsed) { return ParseResult(ParseResultState::WAITING_FOR_MORE_DATA); } - // Write the data to msgBody_ and then advance the cursor msgBody_ = std::make_unique(); - if (contentLength->first > 0 && msgBody_) { - cursor.cloneAtMost(*msgBody_.get(), contentLength->first); + cursor.cloneAtMost(*msgBody_, contentLength->first); + return ParseResult(parsed + contentLength->first); +} + +ParseResult HTTPBinaryCodec::parseSingleContentHelper(folly::io::Cursor& cursor, + size_t remaining) { + if (!bodyStreamingEnabled_) { + return parseBufferedContentHelper(cursor, remaining); } - // Increase parsed by the number of bytes read - parsed += contentLength->first; + size_t parsed = 0; + + // A chunk whose length prefix was consumed on an earlier call resumes here + if (!remainingContentLength_) { + // Parse the contentLength and advance cursor + auto contentLength = quic::follyutils::decodeQuicInteger(cursor); + if (!contentLength) { + return ParseResult(ParseResultState::WAITING_FOR_MORE_DATA); + } + // Check that we had enough bytes to parse contentLength + if (remaining < contentLength->second) { + return ParseResult(ParseResultState::WAITING_FOR_MORE_DATA); + } + // Increase parsed and decrease remaining by the number of bytes read + parsed += contentLength->second; + remaining -= contentLength->second; + if (contentLength->first == 0) { + return ParseResult(parsed); + } + remainingContentLength_ = contentLength->first; + } + + const size_t available = + std::min(*remainingContentLength_, remaining); + if (available > 0) { + msgBody_ = std::make_unique(); + cursor.cloneAtMost(*msgBody_, available); + parsed += available; + *remainingContentLength_ -= available; + } + + if (*remainingContentLength_ > 0) { + return ParseResult(parsed, ParseResultState::WAITING_FOR_MORE_DATA); + } + remainingContentLength_.reset(); return ParseResult(parsed); } ParseResult HTTPBinaryCodec::parseKnownLengthContentHelper( folly::io::Cursor& cursor, size_t remaining) { auto parseResult = parseSingleContentHelper(cursor, remaining); - if (parseResult.parseResultState_ == ParseResultState::DONE && msgBody_ && + if (parseResult.parseResultState_ != ParseResultState::ERROR && msgBody_ && callback_) { callback_->onBody(ingressTxnID_, std::move(msgBody_), 0); } @@ -381,17 +461,17 @@ ParseResult HTTPBinaryCodec::parseIndeterminateLengthContentHelper( if (parseResult.parseResultState_ == ParseResultState::ERROR) { return parseResult; } - // Report the bytes of the chunks already handed to the callback, otherwise - // they are not trimmed and are parsed and delivered a second time - if (parseResult.parseResultState_ == - ParseResultState::WAITING_FOR_MORE_DATA) { - return ParseResult(parsed, ParseResultState::WAITING_FOR_MORE_DATA); - } - // After successfully processing a body chunk, we call onBody + // After processing a body chunk, whole or partial, we call onBody parsed += parseResult.bytesParsed_; if (msgBody_ && callback_) { callback_->onBody(ingressTxnID_, std::move(msgBody_), 0); } + // Report the bytes already handed to the callback so that they are not + // parsed and delivered a second time once the chunk resumes + if (parseResult.parseResultState_ == + ParseResultState::WAITING_FOR_MORE_DATA) { + return ParseResult(parsed, ParseResultState::WAITING_FOR_MORE_DATA); + } // If we have reached the end of the cursor at this point, we must // be waiting for more data since we haven't seen a Content // Terminator field yet @@ -563,7 +643,8 @@ size_t HTTPBinaryCodec::onIngress(const folly::IOBuf& buf) { } void HTTPBinaryCodec::onIngressEOF() { - if (!parseError_ && !bufferedIngress_.empty()) { + if (!parseError_ && + (!bufferedIngress_.empty() || remainingContentLength_.value_or(0) > 0)) { // Case where the ingress EOF is received before the entire message is // parsed callback_->onError(ingressTxnID_, diff --git a/proxygen/lib/http/codec/HTTPBinaryCodec.h b/proxygen/lib/http/codec/HTTPBinaryCodec.h index 816da4b05f..582bc7fc18 100644 --- a/proxygen/lib/http/codec/HTTPBinaryCodec.h +++ b/proxygen/lib/http/codec/HTTPBinaryCodec.h @@ -9,6 +9,7 @@ #pragma once #include +#include #include #include #include @@ -53,6 +54,8 @@ class ParseResult { */ class HTTPBinaryCodec : public HTTPCodec { public: + static constexpr uint64_t kDefaultMaxFieldSectionSize = uint64_t{128} * 1024; + // Default strictValidation to false for now to match existing behavior explicit HTTPBinaryCodec(TransportDirection direction); HTTPBinaryCodec(TransportDirection direction, bool knownEgressLength); @@ -78,6 +81,18 @@ class HTTPBinaryCodec : public HTTPCodec { void setCallback(Callback* callback) override { callback_ = callback; } + void setMaxFieldSectionSize(uint64_t size) { + maxFieldSectionSize_ = + (size == 0) ? std::numeric_limits::max() : size; + } + uint64_t getMaxFieldSectionSize() const { + return maxFieldSectionSize_; + } + // Ignored while ingress parsing is in progress. + void setBodyStreamingEnabled(bool enabled); + bool getBodyStreamingEnabled() const { + return bodyStreamingEnabled_; + } bool isBusy() const override { return false; } @@ -169,6 +184,7 @@ class HTTPBinaryCodec : public HTTPCodec { bool& knownLength); ParseResult parseKnownLengthString(folly::io::Cursor& cursor, size_t remaining, + uint64_t maxLength, folly::StringPiece stringName, std::string& stringValue); ParseResult parseRequestControlData(folly::io::Cursor& cursor, @@ -182,6 +198,8 @@ class HTTPBinaryCodec : public HTTPCodec { HeaderDecodeInfo& decodeInfo, bool knownLength); ParseResult parseContent(folly::io::Cursor& cursor, size_t remaining); + ParseResult parseBufferedContentHelper(folly::io::Cursor& cursor, + size_t remaining); ParseResult parseSingleContentHelper(folly::io::Cursor& cursor, size_t remaining); ParseResult parseKnownLengthContentHelper(folly::io::Cursor& cursor, @@ -195,6 +213,7 @@ class HTTPBinaryCodec : public HTTPCodec { HeaderDecodeInfo& decodeInfo, size_t& parsed, size_t& remaining, + uint64_t maxLength, size_t& numHeaders); ParseResult parseKnownLengthHeadersHelper(folly::io::Cursor& cursor, size_t remaining, @@ -211,6 +230,8 @@ class HTTPBinaryCodec : public HTTPCodec { bool isRequest_{true}; bool knownEgressLength_{true}; bool knownIngressLength_{false}; + bool bodyStreamingEnabled_{true}; + uint64_t maxFieldSectionSize_{kDefaultMaxFieldSectionSize}; enum class ParseState : uint8_t { FRAMING_INDICATOR = 0, INFORMATIONAL_RESPONSE = 1, @@ -240,10 +261,13 @@ class HTTPBinaryCodec : public HTTPCodec { StreamID ingressTxnID_; folly::IOBufQueue bufferedIngress_{folly::IOBufQueue::cacheChainLength()}; - // We don't need to use an IOBufQueue for msgBody_ since we are writing a - // complete message to it and don't need to rely on the efficient size - // computation provided by IOBufQueue + // The piece of the current content chunk that is about to be handed to + // onBody. Content is delivered incrementally, so this holds one piece rather + // than the whole chunk. std::unique_ptr msgBody_; + // Bytes still outstanding for the content chunk being delivered, set while a + // chunk has only partially arrived so that later ingress resumes mid-chunk + folly::Optional remainingContentLength_; HeaderDecodeInfo decodeInfo_; std::unique_ptr msg_; diff --git a/proxygen/lib/http/codec/test/HTTPBinaryCodecTest.cpp b/proxygen/lib/http/codec/test/HTTPBinaryCodecTest.cpp index 907a3166ce..3dcefa8e86 100644 --- a/proxygen/lib/http/codec/test/HTTPBinaryCodecTest.cpp +++ b/proxygen/lib/http/codec/test/HTTPBinaryCodecTest.cpp @@ -57,6 +57,10 @@ class HTTPBinaryCodecForTest : public HTTPBinaryCodec { folly::IOBuf& getMsgBody() { return *msgBody_; } + + size_t getBufferedIngressSize() const { + return bufferedIngress_.chainLength(); + } }; namespace { @@ -85,6 +89,17 @@ void writeRequestPreamble(folly::io::QueueAppender& appender, writeVarintString(appender, "/"); } +std::unique_ptr makeFiller(size_t size) { + auto buf = folly::IOBuf::create(size); + buf->append(size); + memset(buf->writableData(), 'a', size); + return buf; +} + +constexpr size_t kFillerChunkSize = size_t{64} * 1024; +constexpr size_t kFillerChunkCount = 16; +constexpr uint64_t kAbsurdDeclaredLength = uint64_t(1) << 40; + } // namespace template @@ -1225,4 +1240,351 @@ TEST_F(HttpBinaryDownstreamCodecTest, testPartialChunkIsNotOverCounted) { EXPECT_EQ(callback.data_.move()->to(), first + second); } +TEST_F(HttpBinaryDownstreamCodecTest, + testSplitContentChunkIsDeliveredOnceWithStreamingDisabled) { + const std::string body(512, 'a'); + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 1024); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + writeVarint(appender, 0 /* empty field section */); + writeVarintString(appender, "first-"); + writeVarintString(appender, body); + + auto head = message.split(message.chainLength() - 256); + auto tail = message.move(); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setBodyStreamingEnabled(false); + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*head); + binaryCodecIndeterminateLength_->onIngress(*tail); + + EXPECT_EQ(callback.lastParseError, nullptr); + EXPECT_EQ(callback.bodyLength, body.size() + 6); + EXPECT_EQ(callback.data_.move()->to(), "first-" + body); +} + +TEST_F(HttpBinaryDownstreamCodecTest, + testPartialChunkIsNotOverCountedWithStreamingDisabled) { + const std::string first(200, 'a'); + const std::string second(200, 'b'); + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 1024); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + writeVarint(appender, 0 /* empty field section */); + writeVarintString(appender, first); + writeVarintString(appender, second); + + auto head = message.split(message.chainLength() - 100); + auto tail = message.move(); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setBodyStreamingEnabled(false); + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*head); + binaryCodecIndeterminateLength_->onIngress(*tail); + + EXPECT_EQ(callback.lastParseError, nullptr); + EXPECT_EQ(callback.bodyLength, first.size() + second.size()); + EXPECT_EQ(callback.data_.move()->to(), first + second); +} + +TEST_F(HttpBinaryDownstreamCodecTest, + testBodyStreamingSettingCannotChangeMidChunk) { + const std::string body(512, 'a'); + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 1024); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + writeVarint(appender, 0 /* empty field section */); + writeVarintString(appender, body); + + auto head = message.split(message.chainLength() - 256); + auto tail = message.move(); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*head); + binaryCodecIndeterminateLength_->setBodyStreamingEnabled(false); + + EXPECT_TRUE(binaryCodecIndeterminateLength_->getBodyStreamingEnabled()); + + binaryCodecIndeterminateLength_->onIngress(*tail); + + EXPECT_EQ(callback.lastParseError, nullptr); + EXPECT_EQ(callback.bodyLength, body.size()); + EXPECT_EQ(callback.data_.move()->to(), body); +} + +// Field section lengths are QUIC varints, so a peer can declare up to 2^62-1 +// bytes. The declaration must be rejected up front instead of buffering +// ingress until the section is complete. +TEST_F(HttpBinaryDownstreamCodecTest, testDeclaredFieldSectionSizeIsBounded) { + folly::IOBufQueue preamble{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&preamble, 128); + writeRequestPreamble(appender, 0 /* request, known length */); + writeVarint(appender, kAbsurdDeclaredLength); + + FakeHTTPCodecCallback callback; + binaryCodecKnownLength_->setCallback(&callback); + binaryCodecKnownLength_->onIngress(*preamble.front()); + + auto filler = makeFiller(kFillerChunkSize); + for (size_t i = 0; i < kFillerChunkCount && !callback.lastParseError; i++) { + binaryCodecKnownLength_->onIngress(*filler); + } + + EXPECT_NE(callback.lastParseError, nullptr); + EXPECT_LT(binaryCodecKnownLength_->getBufferedIngressSize(), + kFillerChunkSize * kFillerChunkCount); +} + +// The same bound applies to an individual field, whose length is also a varint +// and which is buffered whole before the name/value pair is validated. +TEST_F(HttpBinaryDownstreamCodecTest, testDeclaredFieldSizeIsBounded) { + folly::IOBufQueue preamble{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&preamble, 128); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + writeVarintString(appender, "x-pad"); + writeVarint(appender, kAbsurdDeclaredLength); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*preamble.front()); + + auto filler = makeFiller(kFillerChunkSize); + for (size_t i = 0; i < kFillerChunkCount && !callback.lastParseError; i++) { + binaryCodecIndeterminateLength_->onIngress(*filler); + } + + EXPECT_NE(callback.lastParseError, nullptr); + EXPECT_LT(binaryCodecIndeterminateLength_->getBufferedIngressSize(), + kFillerChunkSize * kFillerChunkCount); +} + +// An indeterminate-length field section is terminated by a zero byte rather +// than a declared length, so it is bounded by the bytes actually seen. +TEST_F(HttpBinaryDownstreamCodecTest, + testIndeterminateFieldSectionSizeIsBounded) { + folly::IOBufQueue preamble{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&preamble, 128); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*preamble.front()); + + // A stream of well-formed fields that never reaches the terminator + folly::IOBufQueue fields{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender fieldsAppender(&fields, 1024); + const std::string padValue(64, 'a'); + for (size_t i = 0; i < 512; i++) { + writeVarintString(fieldsAppender, "x-pad"); + writeVarintString(fieldsAppender, padValue); + } + auto fieldsBuf = fields.move(); + + for (size_t i = 0; i < kFillerChunkCount && !callback.lastParseError; i++) { + binaryCodecIndeterminateLength_->onIngress(*fieldsBuf); + } + + EXPECT_NE(callback.lastParseError, nullptr); +} + +TEST_F(HttpBinaryDownstreamCodecTest, testMaxFieldSectionSizeIsConfigurable) { + const std::string headerValue(256, 'a'); + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 512); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + writeVarintString(appender, "x-pad"); + writeVarintString(appender, headerValue); + writeVarint(appender, 0 /* field section terminator */); + auto messageBuf = message.move(); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setMaxFieldSectionSize(64); + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*messageBuf); + + EXPECT_NE(callback.lastParseError, nullptr); +} + +// A field is bounded by what is left of the section, not by the section +// maximum, so a declaration that only fits when the fields already parsed are +// ignored is rejected instead of being waited on. +TEST_F(HttpBinaryDownstreamCodecTest, testFieldIsBoundedByRemainingSection) { + constexpr size_t kMaxFieldSectionSize = 4096; + constexpr size_t kPadFieldCount = 3; + + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 1024); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + const std::string padValue(1024, 'a'); + for (size_t i = 0; i < kPadFieldCount; i++) { + writeVarintString(appender, "x-pad"); + writeVarintString(appender, padValue); + } + // Comfortably under the maximum on its own, but well over the ~1000 bytes + // the fields above have left of it. None of the declared bytes follow. + writeVarintString(appender, "x-last"); + writeVarint(appender, kMaxFieldSectionSize / 2); + auto messageBuf = message.move(); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setMaxFieldSectionSize(kMaxFieldSectionSize); + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*messageBuf); + + EXPECT_NE(callback.lastParseError, nullptr); +} + +// Setting the maximum to 0 turns enforcement off, so a declaration that would +// otherwise be rejected is accepted again. This is the runtime escape hatch. +TEST_F(HttpBinaryDownstreamCodecTest, testMaxFieldSectionSizeZeroDisables) { + folly::IOBufQueue preamble{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&preamble, 128); + writeRequestPreamble(appender, 0 /* request, known length */); + writeVarint(appender, kAbsurdDeclaredLength); + + FakeHTTPCodecCallback callback; + binaryCodecKnownLength_->setMaxFieldSectionSize(0); + EXPECT_EQ(binaryCodecKnownLength_->getMaxFieldSectionSize(), + std::numeric_limits::max()); + binaryCodecKnownLength_->setCallback(&callback); + binaryCodecKnownLength_->onIngress(*preamble.front()); + + // Deliberately well under the buffered-ingress ceiling added later in this + // stack: disabling the field section limit does not disable that one, and + // this test is about the field section limit alone + auto filler = makeFiller(kFillerChunkSize); + for (size_t i = 0; i < kFillerChunkCount / 4; i++) { + binaryCodecKnownLength_->onIngress(*filler); + } + + // Enforcement is off, so the codec waits for the declared section instead of + // rejecting it -- the pre-ceiling behaviour + EXPECT_EQ(callback.lastParseError, nullptr); +} + +// A declared content length is also a varint. The codec must hand body bytes +// to the callback as they arrive rather than buffering the whole chunk. +TEST_F(HttpBinaryDownstreamCodecTest, testLargeKnownLengthContentIsStreamed) { + folly::IOBufQueue preamble{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&preamble, 128); + writeRequestPreamble(appender, 0 /* request, known length */); + writeVarint(appender, 0 /* empty field section */); + writeVarint(appender, kAbsurdDeclaredLength); + + FakeHTTPCodecCallback callback; + binaryCodecKnownLength_->setCallback(&callback); + binaryCodecKnownLength_->onIngress(*preamble.front()); + + auto filler = makeFiller(kFillerChunkSize); + for (size_t i = 0; i < kFillerChunkCount; i++) { + binaryCodecKnownLength_->onIngress(*filler); + } + + EXPECT_EQ(callback.lastParseError, nullptr); + EXPECT_EQ(callback.bodyLength, kFillerChunkSize * kFillerChunkCount); + EXPECT_LT(binaryCodecKnownLength_->getBufferedIngressSize(), + kFillerChunkSize); +} + +TEST_F(HttpBinaryDownstreamCodecTest, + testLargeIndeterminateLengthContentIsStreamed) { + folly::IOBufQueue preamble{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&preamble, 128); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + writeVarint(appender, 0 /* empty field section */); + writeVarint(appender, kAbsurdDeclaredLength); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*preamble.front()); + + auto filler = makeFiller(kFillerChunkSize); + for (size_t i = 0; i < kFillerChunkCount; i++) { + binaryCodecIndeterminateLength_->onIngress(*filler); + } + + EXPECT_EQ(callback.lastParseError, nullptr); + EXPECT_EQ(callback.bodyLength, kFillerChunkSize * kFillerChunkCount); + EXPECT_LT(binaryCodecIndeterminateLength_->getBufferedIngressSize(), + kFillerChunkSize); +} + +TEST_F(HttpBinaryDownstreamCodecTest, + testKnownLengthContentStreamingCanBeDisabled) { + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 128); + writeRequestPreamble(appender, 0 /* request, known length */); + writeVarint(appender, 0 /* empty field section */); + writeVarint(appender, 64 /* declared content length */); + const std::string firstHalf(32, 'a'); + appender.pushAtMost(reinterpret_cast(firstHalf.data()), + firstHalf.size()); + + FakeHTTPCodecCallback callback; + binaryCodecKnownLength_->setBodyStreamingEnabled(false); + binaryCodecKnownLength_->setCallback(&callback); + binaryCodecKnownLength_->onIngress(*message.front()); + + EXPECT_EQ(callback.bodyLength, 0); + + const std::string secondHalf(32, 'b'); + auto tail = folly::IOBuf::copyBuffer(secondHalf); + binaryCodecKnownLength_->onIngress(*tail); + + EXPECT_EQ(callback.bodyLength, 64); + EXPECT_EQ(callback.data_.move()->to(), firstHalf + secondHalf); +} + +TEST_F(HttpBinaryDownstreamCodecTest, + testIndeterminateLengthContentStreamingCanBeDisabled) { + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 128); + writeRequestPreamble(appender, 2 /* request, indeterminate length */); + writeVarint(appender, 0 /* empty field section */); + writeVarint(appender, 64 /* declared content length */); + const std::string firstHalf(32, 'a'); + appender.pushAtMost(reinterpret_cast(firstHalf.data()), + firstHalf.size()); + + FakeHTTPCodecCallback callback; + binaryCodecIndeterminateLength_->setBodyStreamingEnabled(false); + binaryCodecIndeterminateLength_->setCallback(&callback); + binaryCodecIndeterminateLength_->onIngress(*message.front()); + + EXPECT_EQ(callback.bodyLength, 0); + + const std::string secondHalf(32, 'b'); + auto tail = folly::IOBuf::copyBuffer(secondHalf); + binaryCodecIndeterminateLength_->onIngress(*tail); + + EXPECT_EQ(callback.bodyLength, 64); + EXPECT_EQ(callback.data_.move()->to(), firstHalf + secondHalf); +} + +// Content is trimmed from bufferedIngress_ as it is delivered, so a truncated +// chunk must still be reported as an incomplete message at EOF +TEST_F(HttpBinaryDownstreamCodecTest, testTruncatedContentIsIncompleteAtEOF) { + folly::IOBufQueue message{folly::IOBufQueue::cacheChainLength()}; + folly::io::QueueAppender appender(&message, 128); + writeRequestPreamble(appender, 0 /* request, known length */); + writeVarint(appender, 0 /* empty field section */); + writeVarint(appender, 64 /* declared content length */); + const std::string partialContent(32, 'a'); + appender.pushAtMost(reinterpret_cast(partialContent.data()), + partialContent.size()); + auto messageBuf = message.move(); + + FakeHTTPCodecCallback callback; + binaryCodecKnownLength_->setCallback(&callback); + binaryCodecKnownLength_->onIngress(*messageBuf); + binaryCodecKnownLength_->onIngressEOF(); + + ASSERT_NE(callback.lastParseError, nullptr); + EXPECT_EQ(std::string(callback.lastParseError->what()), + "Incomplete message received"); +} + } // namespace proxygen::test