From 084513889a3116b2f28ef1934f0faaf325ea159e Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Fri, 29 May 2026 18:54:31 -0300 Subject: [PATCH 01/66] test(02-01): add failing tests for negative ack backoff - Adds pure negative_ack_backoff module API skeleton - Covers default, custom, zero-delay, validation, and custom trait vectors - Wires consumer-module exports for the new policy types Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/mod.rs | 4 + src/consumer/negative_ack_backoff.rs | 208 +++++++++++++++++++++++++++ 2 files changed, 212 insertions(+) create mode 100644 src/consumer/negative_ack_backoff.rs diff --git a/src/consumer/mod.rs b/src/consumer/mod.rs index 3111c8df..289fa964 100644 --- a/src/consumer/mod.rs +++ b/src/consumer/mod.rs @@ -8,6 +8,7 @@ mod engine; pub(crate) mod initial_position; pub mod message; mod multi; +pub mod negative_ack_backoff; pub mod options; pub mod topic; @@ -28,6 +29,9 @@ use futures::{ pub use initial_position::InitialPosition; pub use message::Message; use multi::MultiTopicConsumer; +pub use negative_ack_backoff::{ + MultiplierRedeliveryBackoff, MultiplierRedeliveryBackoffBuilder, NegativeAckBackoff, +}; pub use options::ConsumerOptions; pub use topic::TopicConsumer; use url::Url; diff --git a/src/consumer/negative_ack_backoff.rs b/src/consumer/negative_ack_backoff.rs new file mode 100644 index 00000000..ba371d02 --- /dev/null +++ b/src/consumer/negative_ack_backoff.rs @@ -0,0 +1,208 @@ +use std::time::Duration; + +use crate::Error; + +const MIN_DELAY_MUST_NOT_EXCEED_MAX_DELAY: &str = + "min_delay must be less than or equal to max_delay"; +const MULTIPLIER_MUST_BE_GREATER_THAN_ONE: &str = "multiplier must be greater than 1.0"; +const MULTIPLIER_MUST_BE_FINITE: &str = "multiplier must be finite"; +const DELAY_DURATION_TOO_LARGE: &str = "delay duration is too large"; + +/// Policy for computing a negative-acknowledgment redelivery delay from a redelivery count. +pub trait NegativeAckBackoff: Send + Sync + std::fmt::Debug { + /// Returns the delay to use for the provided redelivery count. + fn next(&self, redelivery_count: u32) -> Duration; +} + +/// Built-in negative-acknowledgment redelivery backoff using a multiplier. +#[derive(Clone, Debug)] +pub struct MultiplierRedeliveryBackoff { + min_delay: Duration, + max_delay: Duration, + multiplier: f64, +} + +impl Default for MultiplierRedeliveryBackoff { + fn default() -> Self { + Self { + min_delay: Duration::from_secs(1), + max_delay: Duration::from_secs(600), + multiplier: 2.0, + } + } +} + +impl MultiplierRedeliveryBackoff { + /// Creates a builder for configuring a multiplier redelivery backoff policy. + pub fn builder() -> MultiplierRedeliveryBackoffBuilder { + MultiplierRedeliveryBackoffBuilder::default() + } +} + +impl NegativeAckBackoff for MultiplierRedeliveryBackoff { + fn next(&self, _redelivery_count: u32) -> Duration { + self.min_delay + } +} + +/// Builder for [`MultiplierRedeliveryBackoff`]. +#[derive(Clone, Debug)] +pub struct MultiplierRedeliveryBackoffBuilder { + min_delay: Duration, + max_delay: Duration, + multiplier: f64, +} + +impl Default for MultiplierRedeliveryBackoffBuilder { + fn default() -> Self { + Self { + min_delay: Duration::from_secs(1), + max_delay: Duration::from_secs(600), + multiplier: 2.0, + } + } +} + +impl MultiplierRedeliveryBackoffBuilder { + /// Sets the minimum delay returned for redelivery count `0`. + pub fn min_delay(mut self, min_delay: Duration) -> Self { + self.min_delay = min_delay; + self + } + + /// Sets the maximum delay cap for the backoff sequence. + pub fn max_delay(mut self, max_delay: Duration) -> Self { + self.max_delay = max_delay; + self + } + + /// Sets the multiplier applied for each redelivery count. + pub fn multiplier(mut self, multiplier: f64) -> Self { + self.multiplier = multiplier; + self + } + + /// Builds a validated [`MultiplierRedeliveryBackoff`]. + pub fn build(self) -> Result { + Ok(MultiplierRedeliveryBackoff { + min_delay: self.min_delay, + max_delay: self.max_delay, + multiplier: self.multiplier, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn custom_backoff() -> MultiplierRedeliveryBackoff { + MultiplierRedeliveryBackoff::builder() + .min_delay(Duration::from_secs(2)) + .max_delay(Duration::from_secs(30)) + .multiplier(3.0) + .build() + .unwrap() + } + + fn assert_custom_error(result: Result, expected: &str) { + match result { + Err(Error::Custom(message)) => assert_eq!(message, expected), + other => panic!("expected Error::Custom({expected:?}), got {other:?}"), + } + } + + #[test] + fn default_multiplier_backoff_sequence_matches_pulsar_vectors() { + let backoff = MultiplierRedeliveryBackoff::default(); + + assert_eq!(backoff.next(0), Duration::from_secs(1)); + assert_eq!(backoff.next(1), Duration::from_secs(2)); + assert_eq!(backoff.next(2), Duration::from_secs(4)); + assert_eq!(backoff.next(9), Duration::from_secs(512)); + assert_eq!(backoff.next(10), Duration::from_secs(600)); + assert_eq!(backoff.next(u32::MAX), Duration::from_secs(600)); + } + + #[test] + fn custom_multiplier_backoff_sequence_caps_at_max_delay() { + let backoff = custom_backoff(); + + assert_eq!(backoff.next(0), Duration::from_secs(2)); + assert_eq!(backoff.next(1), Duration::from_secs(6)); + assert_eq!(backoff.next(2), Duration::from_secs(18)); + assert_eq!(backoff.next(3), Duration::from_secs(30)); + } + + #[test] + fn zero_min_delay_returns_zero_for_all_counts() { + let backoff = MultiplierRedeliveryBackoff::builder() + .min_delay(Duration::ZERO) + .build() + .unwrap(); + + assert_eq!(backoff.next(0), Duration::ZERO); + assert_eq!(backoff.next(1), Duration::ZERO); + assert_eq!(backoff.next(10), Duration::ZERO); + assert_eq!(backoff.next(u32::MAX), Duration::ZERO); + } + + #[test] + fn validation_rejects_min_delay_greater_than_max_delay() { + let result = MultiplierRedeliveryBackoff::builder() + .min_delay(Duration::from_secs(2)) + .max_delay(Duration::from_secs(1)) + .build(); + + assert_custom_error(result, MIN_DELAY_MUST_NOT_EXCEED_MAX_DELAY); + } + + #[test] + fn validation_rejects_multiplier_less_than_or_equal_to_one() { + let result = MultiplierRedeliveryBackoff::builder().multiplier(1.0).build(); + + assert_custom_error(result, MULTIPLIER_MUST_BE_GREATER_THAN_ONE); + } + + #[test] + fn validation_rejects_non_finite_multiplier() { + let nan_result = MultiplierRedeliveryBackoff::builder() + .multiplier(f64::NAN) + .build(); + assert_custom_error(nan_result, MULTIPLIER_MUST_BE_FINITE); + + let infinite_result = MultiplierRedeliveryBackoff::builder() + .multiplier(f64::INFINITY) + .build(); + assert_custom_error(infinite_result, MULTIPLIER_MUST_BE_FINITE); + } + + #[test] + fn validation_rejects_oversized_duration() { + let oversized = Duration::from_millis(u64::MAX) + .checked_add(Duration::from_millis(1)) + .unwrap(); + let result = MultiplierRedeliveryBackoff::builder() + .min_delay(oversized) + .max_delay(oversized) + .build(); + + assert_custom_error(result, DELAY_DURATION_TOO_LARGE); + } + + #[test] + fn custom_negative_ack_backoff_can_be_implemented_without_engine_dependencies() { + #[derive(Debug)] + struct FixedBackoff(Duration); + + impl NegativeAckBackoff for FixedBackoff { + fn next(&self, _redelivery_count: u32) -> Duration { + self.0 + } + } + + let backoff = FixedBackoff(Duration::from_secs(42)); + + assert_eq!(backoff.next(123), Duration::from_secs(42)); + } +} From 35f204d1509e49d4d5ad4af689427d52aaf8bb45 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Fri, 29 May 2026 18:55:06 -0300 Subject: [PATCH 02/66] feat(02-01): implement negative ack backoff math - Validates multiplier and delay inputs with exact Error::Custom messages - Computes multiplier delays with max-delay capping and u32::MAX safety - Preserves zero-min-delay immediate redelivery behavior Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/negative_ack_backoff.rs | 60 +++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/src/consumer/negative_ack_backoff.rs b/src/consumer/negative_ack_backoff.rs index ba371d02..3b96f1b6 100644 --- a/src/consumer/negative_ack_backoff.rs +++ b/src/consumer/negative_ack_backoff.rs @@ -40,8 +40,37 @@ impl MultiplierRedeliveryBackoff { } impl NegativeAckBackoff for MultiplierRedeliveryBackoff { - fn next(&self, _redelivery_count: u32) -> Duration { - self.min_delay + fn next(&self, redelivery_count: u32) -> Duration { + if self.min_delay == Duration::ZERO { + return Duration::ZERO; + } + + if redelivery_count == 0 { + return self.min_delay; + } + + if self.min_delay >= self.max_delay { + return self.max_delay; + } + + let min_delay_secs = self.min_delay.as_secs_f64(); + let max_delay_secs = self.max_delay.as_secs_f64(); + let multiplier_log = self.multiplier.ln(); + let cap_after = (max_delay_secs / min_delay_secs).ln() / multiplier_log; + + if !cap_after.is_finite() || redelivery_count as f64 >= cap_after { + return self.max_delay; + } + + let delay_secs = min_delay_secs * self.multiplier.powf(redelivery_count as f64); + if !delay_secs.is_finite() || delay_secs >= max_delay_secs { + return self.max_delay; + } + + match Duration::try_from_secs_f64(delay_secs) { + Ok(delay) if delay < self.max_delay => delay, + _ => self.max_delay, + } } } @@ -84,6 +113,25 @@ impl MultiplierRedeliveryBackoffBuilder { /// Builds a validated [`MultiplierRedeliveryBackoff`]. pub fn build(self) -> Result { + validate_delay_duration(self.min_delay)?; + validate_delay_duration(self.max_delay)?; + + if self.min_delay > self.max_delay { + return Err(Error::Custom( + MIN_DELAY_MUST_NOT_EXCEED_MAX_DELAY.to_string(), + )); + } + + if !self.multiplier.is_finite() { + return Err(Error::Custom(MULTIPLIER_MUST_BE_FINITE.to_string())); + } + + if self.multiplier <= 1.0 { + return Err(Error::Custom( + MULTIPLIER_MUST_BE_GREATER_THAN_ONE.to_string(), + )); + } + Ok(MultiplierRedeliveryBackoff { min_delay: self.min_delay, max_delay: self.max_delay, @@ -92,6 +140,14 @@ impl MultiplierRedeliveryBackoffBuilder { } } +fn validate_delay_duration(delay: Duration) -> Result<(), Error> { + if delay.as_millis() > u64::MAX as u128 { + return Err(Error::Custom(DELAY_DURATION_TOO_LARGE.to_string())); + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; From 31ccb461edbddacf14379ccd88cc557cf5f20ba8 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Fri, 29 May 2026 18:55:54 -0300 Subject: [PATCH 03/66] feat(02-01): expose negative ack backoff at crate root - Re-exports NegativeAckBackoff from the root API - Re-exports MultiplierRedeliveryBackoff and its builder - Keeps Phase 2 API available from both consumer and crate root Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/negative_ack_backoff.rs | 4 +++- src/lib.rs | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/consumer/negative_ack_backoff.rs b/src/consumer/negative_ack_backoff.rs index 3b96f1b6..2bcc334c 100644 --- a/src/consumer/negative_ack_backoff.rs +++ b/src/consumer/negative_ack_backoff.rs @@ -215,7 +215,9 @@ mod tests { #[test] fn validation_rejects_multiplier_less_than_or_equal_to_one() { - let result = MultiplierRedeliveryBackoff::builder().multiplier(1.0).build(); + let result = MultiplierRedeliveryBackoff::builder() + .multiplier(1.0) + .build(); assert_custom_error(result, MULTIPLIER_MUST_BE_GREATER_THAN_ONE); } diff --git a/src/lib.rs b/src/lib.rs index 0bbca9f7..1747f510 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -192,7 +192,10 @@ pub use connection::Authentication; pub use connection_manager::{ BrokerAddress, ConnectionRetryOptions, OperationRetryOptions, TlsOptions, }; -pub use consumer::{Consumer, ConsumerBuilder, ConsumerOptions}; +pub use consumer::{ + Consumer, ConsumerBuilder, ConsumerOptions, MultiplierRedeliveryBackoff, + MultiplierRedeliveryBackoffBuilder, NegativeAckBackoff, +}; pub use error::Error; #[cfg(any( feature = "async-std-runtime", From 5b473341cbafe06fe8d14091ac266cf2fc794e9f Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Fri, 29 May 2026 21:16:14 -0300 Subject: [PATCH 04/66] test(03-01): add nack delay validation helper Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 8d734e3b..98f0075f 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -37,6 +37,14 @@ pub struct ConsumerBuilder { topic_refresh: Option, } +fn check_nack_delay_duration(delay: Duration) -> Result<(), Error> { + if delay.as_millis() > u64::MAX as u128 { + return Err(Error::Custom("delay duration is too large".to_string())); + } + + Ok(()) +} + impl ConsumerBuilder { /// Creates a new [ConsumerBuilder] from an existing client instance #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] @@ -347,3 +355,24 @@ impl ConsumerBuilder { }) } } + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[test] + fn test_nack_delay_validation_oversized() { + let result = check_nack_delay_duration(Duration::MAX); + + assert!(result.is_err()); + } + + #[test] + fn test_nack_delay_validation_zero_is_valid() { + let result = check_nack_delay_duration(Duration::ZERO); + + assert!(result.is_ok()); + } +} From eda14a62a8930867ba8014c7da35f484d34b19de Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Fri, 29 May 2026 21:20:49 -0300 Subject: [PATCH 05/66] feat(03-02): wire negative ack configuration Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 84 ++++++++++++++++++++++++++++++++++++++++- src/consumer/config.rs | 10 ++++- src/consumer/mod.rs | 9 +++++ src/consumer/topic.rs | 2 + 4 files changed, 102 insertions(+), 3 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 98f0075f..ad3f1c12 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -1,5 +1,6 @@ use std::{ collections::{BTreeMap, VecDeque}, + sync::Arc, time::Duration, }; @@ -10,7 +11,8 @@ use regex::Regex; use crate::{ consumer::{ config::ConsumerConfig, data::DeadLetterPolicy, multi::MultiTopicConsumer, - options::ConsumerOptions, topic::TopicConsumer, InnerConsumer, + negative_ack_backoff::NegativeAckBackoff, options::ConsumerOptions, topic::TopicConsumer, + InnerConsumer, }, message::proto::command_subscribe::SubType, reader::{Reader, State}, @@ -35,6 +37,8 @@ pub struct ConsumerBuilder { consumer_options: Option, namespace: Option, topic_refresh: Option, + nack_redelivery_delay: Option, + negative_ack_backoff: Option>, } fn check_nack_delay_duration(delay: Duration) -> Result<(), Error> { @@ -64,6 +68,8 @@ impl ConsumerBuilder { consumer_options: None, namespace: None, topic_refresh: None, + nack_redelivery_delay: None, + negative_ack_backoff: None, } } @@ -183,6 +189,30 @@ impl ConsumerBuilder { self } + /// Sets the fixed negative-ack redelivery delay. Messages that are negatively acknowledged + /// will be redelivered after this duration. + /// + /// `Duration::ZERO` is valid and means immediate redelivery; use this to preserve the + /// previous Rust-client default behavior. Repeated calls are last-call-wins. + #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] + pub fn with_nack_redelivery_delay(mut self, delay: Duration) -> Self { + self.nack_redelivery_delay = Some(delay); + self + } + + /// Configures an optional negative-ack backoff policy. When set, message-based nack uses the + /// policy for delay calculation. Compatible with with_nack_redelivery_delay; Phase 4 resolves + /// precedence at runtime. + #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] + pub fn with_negative_ack_backoff(mut self, backoff: B) -> Self + where + B: NegativeAckBackoff + 'static, + { + self.negative_ack_backoff = + Some(Arc::new(backoff) as Arc); + self + } + // Checks the builder for inconsistencies // returns a config and a list of topics with associated brokers #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] @@ -201,6 +231,8 @@ impl ConsumerBuilder { dead_letter_policy, namespace: _, topic_refresh: _, + nack_redelivery_delay, + negative_ack_backoff, } = self; if consumer_name.is_none() { @@ -216,6 +248,9 @@ impl ConsumerBuilder { "Cannot create consumer with no topics and no topic regex".into(), )); } + if let Some(delay) = nack_redelivery_delay { + check_nack_delay_duration(delay)?; + } let topics: Vec<(String, BrokerAddress)> = try_join_all( topics @@ -271,6 +306,8 @@ impl ConsumerBuilder { unacked_message_redelivery_delay: unacked_message_resend_delay, options: consumer_options.unwrap_or_default(), dead_letter_policy, + nack_redelivery_delay, + negative_ack_backoff, }; Ok((config, topics)) } @@ -375,4 +412,49 @@ mod tests { assert!(result.is_ok()); } + + #[test] + fn test_consumer_config_nack_delay_default_is_none() { + let config = ConsumerConfig::default(); + + assert!(config.nack_redelivery_delay.is_none()); + assert!(config.negative_ack_backoff.is_none()); + } + + #[test] + fn test_consumer_config_stores_nack_delay() { + let config = ConsumerConfig { + nack_redelivery_delay: Some(Duration::from_secs(5)), + ..ConsumerConfig::default() + }; + + assert_eq!(config.nack_redelivery_delay, Some(Duration::from_secs(5))); + } + + #[test] + fn test_consumer_config_stores_both_nack_configs_independently() { + let config = ConsumerConfig { + nack_redelivery_delay: Some(Duration::from_secs(5)), + negative_ack_backoff: Some(Arc::new( + crate::consumer::negative_ack_backoff::MultiplierRedeliveryBackoff::default(), + )), + ..ConsumerConfig::default() + }; + + assert!(config.nack_redelivery_delay.is_some()); + assert!(config.negative_ack_backoff.is_some()); + } + + #[test] + fn test_consumer_config_backoff_only_stores_none_for_delay() { + let config = ConsumerConfig { + negative_ack_backoff: Some(Arc::new( + crate::consumer::negative_ack_backoff::MultiplierRedeliveryBackoff::default(), + )), + ..ConsumerConfig::default() + }; + + assert!(config.nack_redelivery_delay.is_none()); + assert!(config.negative_ack_backoff.is_some()); + } } diff --git a/src/consumer/config.rs b/src/consumer/config.rs index 8fba5fbc..e1e4b7f8 100644 --- a/src/consumer/config.rs +++ b/src/consumer/config.rs @@ -1,7 +1,9 @@ -use std::time::Duration; +use std::{sync::Arc, time::Duration}; use crate::{ - consumer::{data::DeadLetterPolicy, options::ConsumerOptions}, + consumer::{ + data::DeadLetterPolicy, negative_ack_backoff::NegativeAckBackoff, options::ConsumerOptions, + }, message::proto::command_subscribe::SubType, }; @@ -28,4 +30,8 @@ pub struct ConsumerConfig { pub(crate) options: ConsumerOptions, /// dead letter policy pub(crate) dead_letter_policy: Option, + /// fixed negative-ack redelivery delay + pub(crate) nack_redelivery_delay: Option, + /// negative-ack redelivery backoff policy + pub(crate) negative_ack_backoff: Option>, } diff --git a/src/consumer/mod.rs b/src/consumer/mod.rs index 289fa964..9138a621 100644 --- a/src/consumer/mod.rs +++ b/src/consumer/mod.rs @@ -382,6 +382,15 @@ impl Consumer { } } + /// Returns the configured fixed negative-ack redelivery delay, or None if not set. + #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] + pub fn nack_redelivery_delay(&self) -> Option { + match &self.inner { + InnerConsumer::Single(c) => c.config.nack_redelivery_delay, + InnerConsumer::Multi(c) => c.config.nack_redelivery_delay, + } + } + /// returns the date of the last message reception #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn last_message_received(&self) -> Option> { diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index e9e9d917..4730c551 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -62,6 +62,8 @@ impl TopicConsumer { unacked_message_redelivery_delay, options, dead_letter_policy, + nack_redelivery_delay: _nack_redelivery_delay, + negative_ack_backoff: _negative_ack_backoff, } = config.clone(); let consumer_id = consumer_id.unwrap_or_else(|| CONSUMER_ID_GENERATOR.fetch_add(1, Ordering::SeqCst)); From 449850a977e1967cf62464d3aa20dcda7fdd0625 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Fri, 29 May 2026 21:24:03 -0300 Subject: [PATCH 06/66] feat(03-03): propagate redelivery count through consumer Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/data.rs | 44 ++++++++++++++++++++++++++++++-- src/consumer/engine.rs | 15 +++++++---- src/consumer/message.rs | 56 +++++++++++++++++++++++++++++++++++++++++ src/consumer/topic.rs | 21 ++++++++++++---- 4 files changed, 124 insertions(+), 12 deletions(-) diff --git a/src/consumer/data.rs b/src/consumer/data.rs index 38200c00..5d9a2622 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -8,7 +8,7 @@ use crate::{ Error, Executor, Payload, }; -pub type MessageIdDataReceiver = mpsc::Receiver>; +pub type MessageIdDataReceiver = mpsc::Receiver>; pub enum EngineEvent { Message(Option), @@ -17,7 +17,7 @@ pub enum EngineEvent { pub enum EngineMessage { Ack(MessageIdData, bool), - Nack(MessageIdData), + Nack(MessageIdData, Option), UnackedRedelivery, GetConnection(oneshot::Sender>>), } @@ -26,6 +26,7 @@ pub enum EngineMessage { pub struct MessageData { pub id: MessageIdData, pub batch_size: Option, + pub(crate) redelivery_count: u32, } #[derive(Debug, Clone)] @@ -36,3 +37,42 @@ pub struct DeadLetterPolicy { /// Name of the dead topic where the failing messages will be sent. pub dead_letter_topic: String, } + +#[cfg(test)] +mod tests { + use super::*; + use crate::message::proto; + + fn message_id() -> proto::MessageIdData { + proto::MessageIdData { + ledger_id: 1, + entry_id: 1, + partition: None, + batch_index: None, + ack_set: vec![], + batch_size: None, + first_chunk_message_id: None, + } + } + + #[test] + fn test_nack_engine_message_with_count() { + let message = + EngineMessage::::Nack(message_id(), Some(42u32)); + + match message { + EngineMessage::Nack(_, count) => assert_eq!(count, Some(42u32)), + _ => panic!("expected nack engine message"), + } + } + + #[test] + fn test_nack_engine_message_id_only_carries_none() { + let message = EngineMessage::::Nack(message_id(), None); + + match message { + EngineMessage::Nack(_, count) => assert_eq!(count, None), + _ => panic!("expected nack engine message"), + } + } +} diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index 46144205..0c884f39 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -40,7 +40,7 @@ pub struct ConsumerEngine { sub_type: SubType, id: u64, name: Option, - tx: mpsc::Sender>, + tx: mpsc::Sender>, messages_rx: Option>, engine_rx: Option>>, event_rx: mpsc::UnboundedReceiver>, @@ -63,7 +63,7 @@ impl ConsumerEngine { sub_type: SubType, id: u64, name: Option, - tx: mpsc::Sender>, + tx: mpsc::Sender>, messages_rx: mpsc::UnboundedReceiver, engine_rx: mpsc::UnboundedReceiver>, batch_size: u32, @@ -286,7 +286,7 @@ impl ConsumerEngine { self.ack(message_id, cumulative).await; true } - Some(EngineMessage::Nack(message_id)) => { + Some(EngineMessage::Nack(message_id, _count)) => { if let Err(e) = self .connection .sender() @@ -540,6 +540,7 @@ impl ConsumerEngine { } else { vec![(message.message_id, payload)] }; + let redelivery_count = message.redelivery_count.unwrap_or(0); for (message_id, payload) in payloads { match (message.redelivery_count, &self.dead_letter_policy) { (Some(redelivery_count), Some(dead_letter_policy)) @@ -594,7 +595,10 @@ impl ConsumerEngine { self.ack(message_id, false).await; } - _ => self.send_to_consumer(message_id, payload).await?, + _ => { + self.send_to_consumer(message_id, payload, redelivery_count) + .await? + } } } Ok(()) @@ -605,10 +609,11 @@ impl ConsumerEngine { &mut self, message_id: MessageIdData, payload: Payload, + redelivery_count: u32, ) -> Result<(), Error> { let now = Instant::now(); self.tx - .send(Ok((message_id.clone(), payload))) + .send(Ok((message_id.clone(), payload, redelivery_count))) .await .map_err(|e| { error!("tx returned {:?}", e); diff --git a/src/consumer/message.rs b/src/consumer/message.rs index 73f25bc8..d1cb95f9 100644 --- a/src/consumer/message.rs +++ b/src/consumer/message.rs @@ -42,6 +42,13 @@ impl Message { &self.message_id.id } + /// Returns the broker-provided redelivery count for this message. Returns `0` when the broker + /// did not supply a redelivery count (first delivery or broker omitted the field). + #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] + pub fn redelivery_count(&self) -> u32 { + self.message_id.redelivery_count + } + /// Get message key (partition key) #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn key(&self) -> Option { @@ -49,6 +56,55 @@ impl Message { } } +#[cfg(test)] +mod tests { + use super::*; + use crate::message::proto; + + fn message_id() -> proto::MessageIdData { + proto::MessageIdData { + ledger_id: 1, + entry_id: 1, + partition: None, + batch_index: None, + ack_set: vec![], + batch_size: None, + first_chunk_message_id: None, + } + } + + fn payload() -> Payload { + Payload { + metadata: proto::MessageMetadata::default(), + data: vec![], + } + } + + #[test] + fn test_redelivery_count_returns_broker_value() { + let message_id = MessageData { + id: message_id(), + batch_size: None, + redelivery_count: 7, + }; + let msg = Message::<()>::new("test-topic", message_id, payload()); + + assert_eq!(msg.redelivery_count(), 7); + } + + #[test] + fn test_redelivery_count_defaults_to_zero() { + let message_id = MessageData { + id: message_id(), + batch_size: None, + redelivery_count: 0, + }; + let msg = Message::<()>::new("test-topic", message_id, payload()); + + assert_eq!(msg.redelivery_count(), 0); + } +} + impl Message { /// directly deserialize a message #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index 4730c551..ad674469 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -225,14 +225,19 @@ impl TopicConsumer { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn nack(&mut self, msg: &Message) -> Result<(), ConsumerError> { self.engine_tx - .send(EngineMessage::Nack(msg.message_id().clone())) + .send(EngineMessage::Nack( + msg.message_id().clone(), + Some(msg.redelivery_count()), + )) .await?; Ok(()) } #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn nack_with_id(&mut self, msg_id: MessageIdData) -> Result<(), ConsumerError> { - self.engine_tx.send(EngineMessage::Nack(msg_id)).await?; + self.engine_tx + .send(EngineMessage::Nack(msg_id, None)) + .await?; Ok(()) } @@ -297,10 +302,16 @@ impl TopicConsumer { } #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] - fn create_message(&self, message_id: MessageIdData, payload: Payload) -> Message { + fn create_message( + &self, + message_id: MessageIdData, + payload: Payload, + redelivery_count: u32, + ) -> Message { let message_id = MessageData { id: message_id, batch_size: payload.metadata.num_messages_in_batch, + redelivery_count, }; Message::new(&self.topic, message_id, payload) } @@ -324,10 +335,10 @@ impl Stream for TopicConsumer { match self.messages.as_mut().poll_next(cx) { Poll::Pending => Poll::Pending, Poll::Ready(None) => Poll::Ready(None), - Poll::Ready(Some(Ok((id, payload)))) => { + Poll::Ready(Some(Ok((id, payload, redelivery_count)))) => { self.last_message_received = Some(Utc::now()); self.messages_received += 1; - Poll::Ready(Some(Ok(self.create_message(id, payload)))) + Poll::Ready(Some(Ok(self.create_message(id, payload, redelivery_count)))) } Poll::Ready(Some(Err(e))) => { error!("we are using in the single-consumer and we got an error, {e}"); From c6d876ff4e026ca64bc110e3a265434dcb6cf091 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Fri, 29 May 2026 21:56:26 -0300 Subject: [PATCH 07/66] fix(03): resolve code review findings Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 16 +++++++++++----- src/consumer/data.rs | 26 ++++++++++++++++++++------ src/consumer/engine.rs | 30 +++++++++++++++--------------- src/consumer/message.rs | 36 +++++++++++++++++++++++------------- src/consumer/topic.rs | 25 ++++++++++++------------- src/reader.rs | 12 ++++++++---- 6 files changed, 89 insertions(+), 56 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index ad3f1c12..14dd7d2b 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -189,8 +189,11 @@ impl ConsumerBuilder { self } - /// Sets the fixed negative-ack redelivery delay. Messages that are negatively acknowledged - /// will be redelivered after this duration. + /// Sets the fixed negative-ack redelivery delay. + /// + /// This value is stored in the consumer configuration, but Phase 3 does not apply it at + /// runtime yet. Negative acknowledgments still use immediate redelivery until the engine + /// scheduling integration lands in Phase 4. /// /// `Duration::ZERO` is valid and means immediate redelivery; use this to preserve the /// previous Rust-client default behavior. Repeated calls are last-call-wins. @@ -200,9 +203,12 @@ impl ConsumerBuilder { self } - /// Configures an optional negative-ack backoff policy. When set, message-based nack uses the - /// policy for delay calculation. Compatible with with_nack_redelivery_delay; Phase 4 resolves - /// precedence at runtime. + /// Configures an optional negative-ack backoff policy. + /// + /// This value is stored in the consumer configuration, but Phase 3 does not apply it at + /// runtime yet. Message-based nacks continue to redeliver immediately until the engine + /// scheduling integration lands in Phase 4. Compatible with with_nack_redelivery_delay; Phase 4 + /// resolves precedence at runtime. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn with_negative_ack_backoff(mut self, backoff: B) -> Self where diff --git a/src/consumer/data.rs b/src/consumer/data.rs index 5d9a2622..a60a4c5c 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -16,6 +16,18 @@ pub enum EngineEvent { } pub enum EngineMessage { + Ack(MessageIdData, bool), + Nack(MessageIdData), + UnackedRedelivery, + GetConnection(oneshot::Sender>>), +} + +pub(crate) enum InternalEngineEvent { + Message(Option), + EngineMessage(Option>), +} + +pub(crate) enum InternalEngineMessage { Ack(MessageIdData, bool), Nack(MessageIdData, Option), UnackedRedelivery, @@ -26,7 +38,6 @@ pub enum EngineMessage { pub struct MessageData { pub id: MessageIdData, pub batch_size: Option, - pub(crate) redelivery_count: u32, } #[derive(Debug, Clone)] @@ -57,21 +68,24 @@ mod tests { #[test] fn test_nack_engine_message_with_count() { - let message = - EngineMessage::::Nack(message_id(), Some(42u32)); + let message = InternalEngineMessage::::Nack( + message_id(), + Some(42u32), + ); match message { - EngineMessage::Nack(_, count) => assert_eq!(count, Some(42u32)), + InternalEngineMessage::Nack(_, count) => assert_eq!(count, Some(42u32)), _ => panic!("expected nack engine message"), } } #[test] fn test_nack_engine_message_id_only_carries_none() { - let message = EngineMessage::::Nack(message_id(), None); + let message = + InternalEngineMessage::::Nack(message_id(), None); match message { - EngineMessage::Nack(_, count) => assert_eq!(count, None), + InternalEngineMessage::Nack(_, count) => assert_eq!(count, None), _ => panic!("expected nack engine message"), } } diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index 0c884f39..0cad2652 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -14,7 +14,7 @@ use crate::{ connection::Connection, consumer::{ batched_message_iterator::BatchedMessageIterator, - data::{DeadLetterPolicy, EngineEvent, EngineMessage}, + data::{DeadLetterPolicy, InternalEngineEvent, InternalEngineMessage}, options::ConsumerOptions, }, error::{ConnectionError, ConsumerError}, @@ -42,9 +42,9 @@ pub struct ConsumerEngine { name: Option, tx: mpsc::Sender>, messages_rx: Option>, - engine_rx: Option>>, - event_rx: mpsc::UnboundedReceiver>, - event_tx: UnboundedSender>, + engine_rx: Option>>, + event_rx: mpsc::UnboundedReceiver>, + event_tx: UnboundedSender>, batch_size: u32, remaining_messages: i64, unacked_message_redelivery_delay: Option, @@ -65,7 +65,7 @@ impl ConsumerEngine { name: Option, tx: mpsc::Sender>, messages_rx: mpsc::UnboundedReceiver, - engine_rx: mpsc::UnboundedReceiver>, + engine_rx: mpsc::UnboundedReceiver>, batch_size: u32, unacked_message_redelivery_delay: Option, dead_letter_policy: Option, @@ -97,7 +97,7 @@ impl ConsumerEngine { fn register_source(&self, mut rx: mpsc::UnboundedReceiver, mapper: M) -> Result<(), ()> where E: Send + 'static, - M: Fn(Option) -> EngineEvent + Send + Sync + 'static, + M: Fn(Option) -> InternalEngineEvent + Send + Sync + 'static, { let mut event_tx = self.event_tx.clone(); @@ -138,14 +138,14 @@ impl ConsumerEngine { } if let Some(messages_rx) = self.messages_rx.take() { - self.register_source(messages_rx, |msg| EngineEvent::Message(msg)) + self.register_source(messages_rx, |msg| InternalEngineEvent::Message(msg)) .map_err(|_| { Error::Custom(String::from("Error registering messages_rx source")) })?; } if let Some(engine_rx) = self.engine_rx.take() { - self.register_source(engine_rx, |msg| EngineEvent::EngineMessage(msg)) + self.register_source(engine_rx, |msg| InternalEngineEvent::EngineMessage(msg)) .map_err(|_| { Error::Custom(String::from("Error registering engine_rx source")) })?; @@ -214,14 +214,14 @@ impl ConsumerEngine { // ``` debug!("consumer engine: timeout (1s)"); } - Ok(Some(EngineEvent::Message(msg))) => { + Ok(Some(InternalEngineEvent::Message(msg))) => { debug!("consumer engine: received message, {:?}", msg); let out = self.handle_message_opt(msg).await; if let Some(res) = out { return res; } } - Ok(Some(EngineEvent::EngineMessage(msg))) => { + Ok(Some(InternalEngineEvent::EngineMessage(msg))) => { debug!("consumer engine: received engine message"); let continue_loop = self.handle_ack_opt(msg).await; if !continue_loop { @@ -276,17 +276,17 @@ impl ConsumerEngine { } } - async fn handle_ack_opt(&mut self, ack_opt: Option>) -> bool { + async fn handle_ack_opt(&mut self, ack_opt: Option>) -> bool { match ack_opt { None => { trace!("ack channel was closed"); false } - Some(EngineMessage::Ack(message_id, cumulative)) => { + Some(InternalEngineMessage::Ack(message_id, cumulative)) => { self.ack(message_id, cumulative).await; true } - Some(EngineMessage::Nack(message_id, _count)) => { + Some(InternalEngineMessage::Nack(message_id, _count)) => { if let Err(e) = self .connection .sender() @@ -300,7 +300,7 @@ impl ConsumerEngine { } true } - Some(EngineMessage::UnackedRedelivery) => { + Some(InternalEngineMessage::UnackedRedelivery) => { let mut h = HashSet::new(); let now = Instant::now(); // info!("unacked messages length: {}", self.unacked_messages.len()); @@ -328,7 +328,7 @@ impl ConsumerEngine { } true } - Some(EngineMessage::GetConnection(sender)) => { + Some(InternalEngineMessage::GetConnection(sender)) => { let _ = sender.send(self.connection.clone()).map_err(|_| { error!( "consumer requested the engine's connection but dropped the \ diff --git a/src/consumer/message.rs b/src/consumer/message.rs index d1cb95f9..50622d3d 100644 --- a/src/consumer/message.rs +++ b/src/consumer/message.rs @@ -17,15 +17,26 @@ pub struct Message { pub payload: Payload, /// contains the message's id and batch size data pub message_id: MessageData, + redelivery_count: u32, pub(super) _phantom: PhantomData, } impl Message { pub fn new(topic: &str, message_id: MessageData, payload: Payload) -> Self { + Self::new_with_redelivery_count(topic, message_id, payload, 0) + } + + pub(crate) fn new_with_redelivery_count( + topic: &str, + message_id: MessageData, + payload: Payload, + redelivery_count: u32, + ) -> Self { Message { topic: topic.to_string(), message_id, payload, + redelivery_count, _phantom: PhantomData, } } @@ -43,10 +54,11 @@ impl Message { } /// Returns the broker-provided redelivery count for this message. Returns `0` when the broker - /// did not supply a redelivery count (first delivery or broker omitted the field). + /// did not supply a redelivery count (first delivery or broker omitted the field), or when the + /// message was constructed directly with [`Message::new`]. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn redelivery_count(&self) -> u32 { - self.message_id.redelivery_count + self.redelivery_count } /// Get message key (partition key) @@ -56,6 +68,14 @@ impl Message { } } +impl Message { + /// directly deserialize a message + #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] + pub fn deserialize(&self) -> T::Output { + T::deserialize_message(&self.payload) + } +} + #[cfg(test)] mod tests { use super::*; @@ -85,9 +105,8 @@ mod tests { let message_id = MessageData { id: message_id(), batch_size: None, - redelivery_count: 7, }; - let msg = Message::<()>::new("test-topic", message_id, payload()); + let msg = Message::<()>::new_with_redelivery_count("test-topic", message_id, payload(), 7); assert_eq!(msg.redelivery_count(), 7); } @@ -97,18 +116,9 @@ mod tests { let message_id = MessageData { id: message_id(), batch_size: None, - redelivery_count: 0, }; let msg = Message::<()>::new("test-topic", message_id, payload()); assert_eq!(msg.redelivery_count(), 0); } } - -impl Message { - /// directly deserialize a message - #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] - pub fn deserialize(&self) -> T::Output { - T::deserialize_message(&self.payload) - } -} diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index ad674469..e3f831b4 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -19,7 +19,7 @@ use crate::{ connection::Connection, consumer::{ config::ConsumerConfig, - data::{DeadLetterPolicy, EngineMessage, MessageData, MessageIdDataReceiver}, + data::{DeadLetterPolicy, InternalEngineMessage, MessageData, MessageIdDataReceiver}, engine::ConsumerEngine, message::Message, }, @@ -36,7 +36,7 @@ pub struct TopicConsumer { pub(crate) config: ConsumerConfig, topic: String, messages: Pin>, - engine_tx: mpsc::UnboundedSender>, + engine_tx: mpsc::UnboundedSender>, data_type: PhantomData T::Output>, pub(crate) dead_letter_policy: Option, pub(super) last_message_received: Option>, @@ -92,7 +92,7 @@ impl TopicConsumer { let res = client.executor.spawn(Box::pin(async move { while interval.next().await.is_some() { if redelivery_tx - .send(EngineMessage::UnackedRedelivery) + .send(InternalEngineMessage::UnackedRedelivery) .await .is_err() { @@ -156,7 +156,7 @@ impl TopicConsumer { pub async fn connection(&mut self) -> Result>, Error> { let (resolver, response) = oneshot::channel(); self.engine_tx - .send(EngineMessage::GetConnection(resolver)) + .send(InternalEngineMessage::GetConnection(resolver)) .await .map_err(|_| ConsumerError::Connection(ConnectionError::Disconnected))?; @@ -185,7 +185,7 @@ impl TopicConsumer { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn ack(&mut self, msg: &Message) -> Result<(), ConsumerError> { self.engine_tx - .send(EngineMessage::Ack(msg.message_id().clone(), false)) + .send(InternalEngineMessage::Ack(msg.message_id().clone(), false)) .await?; Ok(()) } @@ -193,20 +193,20 @@ impl TopicConsumer { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn ack_with_id(&mut self, msg_id: MessageIdData) -> Result<(), ConsumerError> { self.engine_tx - .send(EngineMessage::Ack(msg_id, false)) + .send(InternalEngineMessage::Ack(msg_id, false)) .await?; Ok(()) } #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] - pub(crate) fn acker(&self) -> mpsc::UnboundedSender> { + pub(crate) fn acker(&self) -> mpsc::UnboundedSender> { self.engine_tx.clone() } #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn cumulative_ack(&mut self, msg: &Message) -> Result<(), ConsumerError> { self.engine_tx - .send(EngineMessage::Ack(msg.message_id().clone(), true)) + .send(InternalEngineMessage::Ack(msg.message_id().clone(), true)) .await?; Ok(()) } @@ -217,7 +217,7 @@ impl TopicConsumer { msg_id: MessageIdData, ) -> Result<(), ConsumerError> { self.engine_tx - .send(EngineMessage::Ack(msg_id, true)) + .send(InternalEngineMessage::Ack(msg_id, true)) .await?; Ok(()) } @@ -225,7 +225,7 @@ impl TopicConsumer { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn nack(&mut self, msg: &Message) -> Result<(), ConsumerError> { self.engine_tx - .send(EngineMessage::Nack( + .send(InternalEngineMessage::Nack( msg.message_id().clone(), Some(msg.redelivery_count()), )) @@ -236,7 +236,7 @@ impl TopicConsumer { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn nack_with_id(&mut self, msg_id: MessageIdData) -> Result<(), ConsumerError> { self.engine_tx - .send(EngineMessage::Nack(msg_id, None)) + .send(InternalEngineMessage::Nack(msg_id, None)) .await?; Ok(()) } @@ -311,9 +311,8 @@ impl TopicConsumer { let message_id = MessageData { id: message_id, batch_size: payload.metadata.num_messages_in_batch, - redelivery_count, }; - Message::new(&self.topic, message_id, payload) + Message::new_with_redelivery_count(&self.topic, message_id, payload, redelivery_count) } #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] diff --git a/src/reader.rs b/src/reader.rs index dba1e4c2..fb763252 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -10,7 +10,9 @@ use url::Url; use crate::{ client::DeserializeMessage, - consumer::{ConsumerOptions, DeadLetterPolicy, EngineMessage, Message, TopicConsumer}, + consumer::{ + data::InternalEngineMessage, ConsumerOptions, DeadLetterPolicy, Message, TopicConsumer, + }, error::Error, executor::Executor, message::proto::{command_subscribe::SubType, MessageIdData}, @@ -55,9 +57,11 @@ impl Stream for Reader { let message_id = msg.message_id().clone(); this.state = Some(State::PollingAck( msg, - Box::pin( - async move { acker.send(EngineMessage::Ack(message_id, false)).await }, - ), + Box::pin(async move { + acker + .send(InternalEngineMessage::Ack(message_id, false)) + .await + }), )); Pin::new(this).poll_next(cx) } From bb29ce30011e06ec8e7a8d6f5515f5e7c88f3f24 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:14:45 -0300 Subject: [PATCH 08/66] test(04-01): add failing negative ack delay selection tests - Covers 60s default fixed delay selection - Covers explicit zero-delay immediate redelivery - Covers message-count backoff and ID-only fixed fallback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/mod.rs | 1 + src/consumer/negative_ack_tracker.rs | 119 +++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 src/consumer/negative_ack_tracker.rs diff --git a/src/consumer/mod.rs b/src/consumer/mod.rs index 9138a621..afae4ead 100644 --- a/src/consumer/mod.rs +++ b/src/consumer/mod.rs @@ -8,6 +8,7 @@ mod engine; pub(crate) mod initial_position; pub mod message; mod multi; +mod negative_ack_tracker; pub mod negative_ack_backoff; pub mod options; pub mod topic; diff --git a/src/consumer/negative_ack_tracker.rs b/src/consumer/negative_ack_tracker.rs new file mode 100644 index 00000000..f5c20f00 --- /dev/null +++ b/src/consumer/negative_ack_tracker.rs @@ -0,0 +1,119 @@ +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use crate::{ + consumer::negative_ack_backoff::NegativeAckBackoff, message::proto::MessageIdData, +}; + +pub(crate) const DEFAULT_NACK_REDELIVERY_DELAY: Duration = Duration::from_secs(60); +pub(crate) const NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL: Duration = Duration::from_millis(500); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum NegativeAckSchedule { + Immediate, + Scheduled, + DuplicateEarlier, + RetryPending, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum NegativeAckDispatchState { + Pending, + InFlight, +} + +#[derive(Debug)] +pub(crate) struct NegativeAckTracker { + fixed_delay: Duration, + backoff: Option>, +} + +impl NegativeAckTracker { + pub(crate) fn new( + fixed_delay: Option, + backoff: Option>, + ) -> Self { + Self { + fixed_delay: fixed_delay.unwrap_or(Duration::ZERO), + backoff, + } + } + + pub(crate) fn select_delay(&self, _redelivery_count: Option) -> Duration { + Duration::ZERO + } + + pub(crate) fn schedule( + &mut self, + _message_id: MessageIdData, + _redelivery_count: Option, + _now: Instant, + ) -> NegativeAckSchedule { + NegativeAckSchedule::Immediate + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug)] + struct FixedBackoff(Duration); + + impl NegativeAckBackoff for FixedBackoff { + fn next(&self, _redelivery_count: u32) -> Duration { + self.0 + } + } + + fn backoff(delay: Duration) -> Arc { + Arc::new(FixedBackoff(delay)) + } + + #[test] + fn missing_fixed_delay_defaults_to_sixty_seconds() { + let tracker = NegativeAckTracker::new(None, None); + + assert_eq!( + tracker.select_delay(None), + DEFAULT_NACK_REDELIVERY_DELAY + ); + assert_eq!(tracker.select_delay(None), Duration::from_secs(60)); + } + + #[test] + fn explicit_zero_fixed_delay_selects_immediate_redelivery() { + let tracker = NegativeAckTracker::new(Some(Duration::ZERO), None); + + assert_eq!(tracker.select_delay(None), Duration::ZERO); + } + + #[test] + fn message_redelivery_count_uses_configured_backoff() { + let tracker = NegativeAckTracker::new(None, Some(backoff(Duration::from_secs(42)))); + + assert_eq!(tracker.select_delay(Some(7)), Duration::from_secs(42)); + } + + #[test] + fn missing_redelivery_count_uses_fixed_delay_even_when_backoff_exists() { + let tracker = NegativeAckTracker::new( + Some(Duration::from_secs(5)), + Some(backoff(Duration::from_secs(42))), + ); + + assert_eq!(tracker.select_delay(None), Duration::from_secs(5)); + } + + #[test] + fn zero_backoff_delay_selects_immediate_redelivery() { + let tracker = NegativeAckTracker::new( + Some(Duration::from_secs(5)), + Some(backoff(Duration::ZERO)), + ); + + assert_eq!(tracker.select_delay(Some(7)), Duration::ZERO); + } +} From a6cadaf88a77c654d66897a1c443571e6cf0127d Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:15:13 -0300 Subject: [PATCH 09/66] feat(04-01): implement negative ack delay selection - Defaults missing nack timing to a 60 second fixed delay - Applies message-count backoff only when a redelivery count is available - Preserves explicit zero-delay immediate redelivery behavior Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/mod.rs | 2 +- src/consumer/negative_ack_tracker.rs | 29 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/consumer/mod.rs b/src/consumer/mod.rs index afae4ead..8ca7041b 100644 --- a/src/consumer/mod.rs +++ b/src/consumer/mod.rs @@ -8,8 +8,8 @@ mod engine; pub(crate) mod initial_position; pub mod message; mod multi; -mod negative_ack_tracker; pub mod negative_ack_backoff; +mod negative_ack_tracker; pub mod options; pub mod topic; diff --git a/src/consumer/negative_ack_tracker.rs b/src/consumer/negative_ack_tracker.rs index f5c20f00..4e6244fe 100644 --- a/src/consumer/negative_ack_tracker.rs +++ b/src/consumer/negative_ack_tracker.rs @@ -3,9 +3,7 @@ use std::{ time::{Duration, Instant}, }; -use crate::{ - consumer::negative_ack_backoff::NegativeAckBackoff, message::proto::MessageIdData, -}; +use crate::{consumer::negative_ack_backoff::NegativeAckBackoff, message::proto::MessageIdData}; pub(crate) const DEFAULT_NACK_REDELIVERY_DELAY: Duration = Duration::from_secs(60); pub(crate) const NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL: Duration = Duration::from_millis(500); @@ -36,13 +34,16 @@ impl NegativeAckTracker { backoff: Option>, ) -> Self { Self { - fixed_delay: fixed_delay.unwrap_or(Duration::ZERO), + fixed_delay: fixed_delay.unwrap_or(DEFAULT_NACK_REDELIVERY_DELAY), backoff, } } - pub(crate) fn select_delay(&self, _redelivery_count: Option) -> Duration { - Duration::ZERO + pub(crate) fn select_delay(&self, redelivery_count: Option) -> Duration { + match (redelivery_count, &self.backoff) { + (Some(count), Some(backoff)) => backoff.next(count), + _ => self.fixed_delay, + } } pub(crate) fn schedule( @@ -51,7 +52,10 @@ impl NegativeAckTracker { _redelivery_count: Option, _now: Instant, ) -> NegativeAckSchedule { - NegativeAckSchedule::Immediate + match self.select_delay(_redelivery_count) { + Duration::ZERO => NegativeAckSchedule::Immediate, + _ => NegativeAckSchedule::Scheduled, + } } } @@ -76,10 +80,7 @@ mod tests { fn missing_fixed_delay_defaults_to_sixty_seconds() { let tracker = NegativeAckTracker::new(None, None); - assert_eq!( - tracker.select_delay(None), - DEFAULT_NACK_REDELIVERY_DELAY - ); + assert_eq!(tracker.select_delay(None), DEFAULT_NACK_REDELIVERY_DELAY); assert_eq!(tracker.select_delay(None), Duration::from_secs(60)); } @@ -109,10 +110,8 @@ mod tests { #[test] fn zero_backoff_delay_selects_immediate_redelivery() { - let tracker = NegativeAckTracker::new( - Some(Duration::from_secs(5)), - Some(backoff(Duration::ZERO)), - ); + let tracker = + NegativeAckTracker::new(Some(Duration::from_secs(5)), Some(backoff(Duration::ZERO))); assert_eq!(tracker.select_delay(Some(7)), Duration::ZERO); } From c1c816147926ae1ca82d5a70e6bd3c65abdc2151 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:15:45 -0300 Subject: [PATCH 10/66] test(04-01): add failing negative ack tracker operation tests - Covers batch index normalization with partial-batch cancellation - Covers due collection, in-flight lifecycle, retry, and clear behavior - Covers partition-scoped cumulative ack cancellation and redelivery ID field normalization Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/negative_ack_tracker.rs | 165 +++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/src/consumer/negative_ack_tracker.rs b/src/consumer/negative_ack_tracker.rs index 4e6244fe..2d5fc8fb 100644 --- a/src/consumer/negative_ack_tracker.rs +++ b/src/consumer/negative_ack_tracker.rs @@ -76,6 +76,23 @@ mod tests { Arc::new(FixedBackoff(delay)) } + fn message_id( + ledger_id: u64, + entry_id: u64, + partition: Option, + batch_index: Option, + ) -> MessageIdData { + MessageIdData { + ledger_id, + entry_id, + partition, + batch_index, + ack_set: vec![], + batch_size: None, + first_chunk_message_id: None, + } + } + #[test] fn missing_fixed_delay_defaults_to_sixty_seconds() { let tracker = NegativeAckTracker::new(None, None); @@ -115,4 +132,152 @@ mod tests { assert_eq!(tracker.select_delay(Some(7)), Duration::ZERO); } + + #[test] + fn batch_ids_share_entry_key_but_track_pending_indexes_separately() { + let now = Instant::now(); + let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(10)), None); + + assert_eq!( + tracker.schedule(message_id(1, 2, Some(0), Some(1)), None, now), + NegativeAckSchedule::Scheduled + ); + assert_eq!( + tracker.schedule(message_id(1, 2, Some(0), Some(2)), None, now), + NegativeAckSchedule::Scheduled + ); + + assert_eq!(tracker.pending_len(), 1); + tracker.cancel_ack(&message_id(1, 2, Some(0), Some(1))); + assert_eq!(tracker.pending_len(), 1); + tracker.cancel_ack(&message_id(1, 2, Some(0), Some(2))); + assert!(tracker.is_empty()); + } + + #[test] + fn duplicate_schedule_keeps_earliest_due_time() { + let now = Instant::now(); + let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(10)), None); + + assert_eq!( + tracker.schedule(message_id(1, 2, None, Some(1)), None, now), + NegativeAckSchedule::Scheduled + ); + assert_eq!( + tracker.schedule( + message_id(1, 2, None, Some(1)), + None, + now + Duration::from_secs(5) + ), + NegativeAckSchedule::DuplicateEarlier + ); + + assert_eq!(tracker.pending_len(), 1); + assert_eq!( + tracker.next_due_time(), + Some(now + Duration::from_secs(10)) + ); + } + + #[test] + fn whole_entry_ack_cancels_all_pending_batch_indexes() { + let now = Instant::now(); + let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(10)), None); + + tracker.schedule(message_id(1, 2, None, Some(1)), None, now); + tracker.schedule(message_id(1, 2, None, Some(2)), None, now); + tracker.cancel_ack(&message_id(1, 2, None, None)); + + assert!(tracker.is_empty()); + } + + #[test] + fn ack_set_ack_cancels_the_whole_pending_entry() { + let now = Instant::now(); + let mut ack = message_id(1, 2, None, Some(1)); + ack.ack_set = vec![0]; + let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(10)), None); + + tracker.schedule(message_id(1, 2, None, Some(1)), None, now); + tracker.schedule(message_id(1, 2, None, Some(2)), None, now); + tracker.cancel_ack(&ack); + + assert!(tracker.is_empty()); + } + + #[test] + fn cumulative_ack_is_partition_scoped_and_handles_same_entry_batches() { + let now = Instant::now(); + let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(10)), None); + + tracker.schedule(message_id(1, 1, Some(0), Some(1)), None, now); + tracker.schedule(message_id(1, 2, Some(0), Some(1)), None, now); + tracker.schedule(message_id(1, 2, Some(0), Some(2)), None, now); + tracker.schedule(message_id(1, 3, Some(0), Some(1)), None, now); + tracker.schedule(message_id(1, 1, Some(1), Some(1)), None, now); + + tracker.cancel_cumulative_ack(&message_id(1, 2, Some(0), Some(1))); + + assert_eq!(tracker.pending_len(), 3); + tracker.cancel_ack(&message_id(1, 2, Some(0), Some(2))); + assert_eq!(tracker.pending_len(), 2); + tracker.cancel_ack(&message_id(1, 3, Some(0), Some(1))); + tracker.cancel_ack(&message_id(1, 1, Some(1), Some(1))); + assert!(tracker.is_empty()); + } + + #[test] + fn collect_due_marks_entries_in_flight_and_returns_entry_level_ids() { + let now = Instant::now(); + let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(10)), None); + let mut original = message_id(1, 2, Some(0), Some(1)); + original.ack_set = vec![1]; + original.batch_size = Some(3); + original.first_chunk_message_id = Some(Box::new(message_id(1, 1, Some(0), None))); + + tracker.schedule(original, None, now); + + assert!(tracker.collect_due(now + Duration::from_secs(9)).is_empty()); + let due = tracker.collect_due(now + Duration::from_secs(10)); + assert_eq!(due.len(), 1); + assert_eq!(due[0].ledger_id, 1); + assert_eq!(due[0].entry_id, 2); + assert_eq!(due[0].partition, Some(0)); + assert_eq!(due[0].batch_index, None); + assert!(due[0].ack_set.is_empty()); + assert_eq!(due[0].batch_size, None); + assert_eq!(due[0].first_chunk_message_id, None); + assert_eq!(tracker.pending_len(), 0); + assert_eq!(tracker.in_flight_len(), 1); + assert!(tracker.collect_due(now + Duration::from_secs(11)).is_empty()); + } + + #[test] + fn dispatch_lifecycle_removes_or_retries_in_flight_entries() { + let now = Instant::now(); + let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(10)), None); + + tracker.schedule(message_id(1, 2, None, None), None, now); + let due = tracker.collect_due(now + Duration::from_secs(10)); + tracker.mark_retry_pending(&due, now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL); + assert_eq!(tracker.pending_len(), 1); + assert_eq!(tracker.in_flight_len(), 0); + assert!(tracker.collect_due(now + Duration::from_millis(499)).is_empty()); + let retried = tracker.collect_due(now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL); + tracker.mark_dispatched(&retried); + assert!(tracker.is_empty()); + } + + #[test] + fn clear_removes_pending_and_in_flight_entries() { + let now = Instant::now(); + let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(10)), None); + + tracker.schedule(message_id(1, 2, None, None), None, now); + tracker.schedule(message_id(1, 3, None, None), None, now); + tracker.collect_due(now + Duration::from_secs(10)); + tracker.clear(); + + assert!(tracker.is_empty()); + } } From 71f02722130f5bdb076db42487f60648a5b31a9f Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:16:52 -0300 Subject: [PATCH 11/66] feat(04-01): implement negative ack tracker operations - Tracks pending nacks by normalized ledger/entry/partition keys with batch-index scopes - Cancels normal and cumulative acks without dropping sibling batch nacks - Moves due entries in-flight before dispatch and supports retry, dispatched, and clear transitions Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/negative_ack_tracker.rs | 261 +++++++++++++++++++++++++-- 1 file changed, 249 insertions(+), 12 deletions(-) diff --git a/src/consumer/negative_ack_tracker.rs b/src/consumer/negative_ack_tracker.rs index 2d5fc8fb..f70c2742 100644 --- a/src/consumer/negative_ack_tracker.rs +++ b/src/consumer/negative_ack_tracker.rs @@ -1,4 +1,5 @@ use std::{ + collections::{BTreeSet, HashMap}, sync::Arc, time::{Duration, Instant}, }; @@ -26,6 +27,93 @@ pub(crate) enum NegativeAckDispatchState { pub(crate) struct NegativeAckTracker { fixed_delay: Duration, backoff: Option>, + entries: HashMap, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub(crate) struct NormalizedMessageId { + ledger_id: u64, + entry_id: u64, + partition: Option, +} + +#[derive(Debug)] +struct PendingNegativeAck { + due_at: Instant, + state: NegativeAckDispatchState, + scope: PendingBatchScope, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +enum PendingBatchScope { + FullEntry, + BatchIndexes(BTreeSet), +} + +impl NormalizedMessageId { + fn from_message_id(message_id: &MessageIdData) -> Self { + Self { + ledger_id: message_id.ledger_id, + entry_id: message_id.entry_id, + partition: message_id.partition, + } + } + + fn into_redelivery_message_id(self) -> MessageIdData { + MessageIdData { + ledger_id: self.ledger_id, + entry_id: self.entry_id, + partition: self.partition, + batch_index: None, + ack_set: vec![], + batch_size: None, + first_chunk_message_id: None, + } + } +} + +impl PendingBatchScope { + fn from_message_id(message_id: &MessageIdData) -> Self { + match message_id.batch_index { + Some(batch_index) => Self::BatchIndexes(BTreeSet::from([batch_index])), + None => Self::FullEntry, + } + } + + fn merge(&mut self, incoming: PendingBatchScope) -> bool { + match (&mut *self, incoming) { + (Self::FullEntry, _) => false, + (scope @ Self::BatchIndexes(_), Self::FullEntry) => { + *scope = Self::FullEntry; + true + } + (Self::BatchIndexes(existing), Self::BatchIndexes(incoming)) => { + incoming.into_iter().fold(false, |changed, batch_index| { + existing.insert(batch_index) || changed + }) + } + } + } + + fn cancel_batch_index(&mut self, batch_index: i32) -> bool { + match self { + Self::FullEntry => false, + Self::BatchIndexes(indexes) => { + indexes.remove(&batch_index); + indexes.is_empty() + } + } + } + + fn cancel_batch_indexes_through(&mut self, batch_index: i32) -> bool { + match self { + Self::FullEntry => false, + Self::BatchIndexes(indexes) => { + indexes.retain(|pending| *pending > batch_index); + indexes.is_empty() + } + } + } } impl NegativeAckTracker { @@ -36,6 +124,7 @@ impl NegativeAckTracker { Self { fixed_delay: fixed_delay.unwrap_or(DEFAULT_NACK_REDELIVERY_DELAY), backoff, + entries: HashMap::new(), } } @@ -48,15 +137,162 @@ impl NegativeAckTracker { pub(crate) fn schedule( &mut self, - _message_id: MessageIdData, - _redelivery_count: Option, - _now: Instant, + message_id: MessageIdData, + redelivery_count: Option, + now: Instant, ) -> NegativeAckSchedule { - match self.select_delay(_redelivery_count) { - Duration::ZERO => NegativeAckSchedule::Immediate, - _ => NegativeAckSchedule::Scheduled, + let delay = self.select_delay(redelivery_count); + if delay == Duration::ZERO { + return NegativeAckSchedule::Immediate; + } + + let key = NormalizedMessageId::from_message_id(&message_id); + let incoming_scope = PendingBatchScope::from_message_id(&message_id); + let due_at = now + delay; + + match self.entries.get_mut(&key) { + None => { + self.entries.insert( + key, + PendingNegativeAck { + due_at, + state: NegativeAckDispatchState::Pending, + scope: incoming_scope, + }, + ); + NegativeAckSchedule::Scheduled + } + Some(existing) if existing.state == NegativeAckDispatchState::InFlight => { + NegativeAckSchedule::RetryPending + } + Some(existing) => { + let scope_changed = existing.scope.merge(incoming_scope); + if due_at < existing.due_at { + existing.due_at = due_at; + NegativeAckSchedule::Scheduled + } else if scope_changed { + NegativeAckSchedule::Scheduled + } else { + NegativeAckSchedule::DuplicateEarlier + } + } } } + + pub(crate) fn pending_len(&self) -> usize { + self.entries + .values() + .filter(|entry| entry.state == NegativeAckDispatchState::Pending) + .count() + } + + pub(crate) fn in_flight_len(&self) -> usize { + self.entries + .values() + .filter(|entry| entry.state == NegativeAckDispatchState::InFlight) + .count() + } + + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub(crate) fn next_due_time(&self) -> Option { + self.entries + .values() + .filter(|entry| entry.state == NegativeAckDispatchState::Pending) + .map(|entry| entry.due_at) + .min() + } + + pub(crate) fn collect_due(&mut self, now: Instant) -> Vec { + let mut due = Vec::new(); + for (key, entry) in &mut self.entries { + if entry.state == NegativeAckDispatchState::Pending && entry.due_at <= now { + entry.state = NegativeAckDispatchState::InFlight; + due.push((*key).into_redelivery_message_id()); + } + } + due + } + + pub(crate) fn mark_dispatched(&mut self, message_ids: &[MessageIdData]) { + for message_id in message_ids { + self.entries + .remove(&NormalizedMessageId::from_message_id(message_id)); + } + } + + pub(crate) fn mark_retry_pending(&mut self, message_ids: &[MessageIdData], retry_at: Instant) { + for message_id in message_ids { + if let Some(entry) = self + .entries + .get_mut(&NormalizedMessageId::from_message_id(message_id)) + { + entry.state = NegativeAckDispatchState::Pending; + entry.due_at = retry_at; + } + } + } + + pub(crate) fn cancel_ack(&mut self, message_id: &MessageIdData) { + let key = NormalizedMessageId::from_message_id(message_id); + if message_id.batch_index.is_none() || !message_id.ack_set.is_empty() { + self.entries.remove(&key); + return; + } + + let Some(batch_index) = message_id.batch_index else { + return; + }; + if self + .entries + .get_mut(&key) + .is_some_and(|entry| entry.scope.cancel_batch_index(batch_index)) + { + self.entries.remove(&key); + } + } + + pub(crate) fn cancel_cumulative_ack(&mut self, message_id: &MessageIdData) { + let ack_key = NormalizedMessageId::from_message_id(message_id); + let mut remove_keys = Vec::new(); + + for key in self.entries.keys() { + if key.partition != ack_key.partition { + continue; + } + if key.ledger_id < ack_key.ledger_id + || (key.ledger_id == ack_key.ledger_id && key.entry_id < ack_key.entry_id) + { + remove_keys.push(*key); + } + } + + for key in remove_keys { + self.entries.remove(&key); + } + + if message_id.batch_index.is_none() || !message_id.ack_set.is_empty() { + self.entries.remove(&ack_key); + return; + } + + let Some(batch_index) = message_id.batch_index else { + return; + }; + if self + .entries + .get_mut(&ack_key) + .is_some_and(|entry| entry.scope.cancel_batch_indexes_through(batch_index)) + { + self.entries.remove(&ack_key); + } + } + + pub(crate) fn clear(&mut self) { + self.entries.clear(); + } } #[cfg(test)] @@ -173,10 +409,7 @@ mod tests { ); assert_eq!(tracker.pending_len(), 1); - assert_eq!( - tracker.next_due_time(), - Some(now + Duration::from_secs(10)) - ); + assert_eq!(tracker.next_due_time(), Some(now + Duration::from_secs(10))); } #[test] @@ -249,7 +482,9 @@ mod tests { assert_eq!(due[0].first_chunk_message_id, None); assert_eq!(tracker.pending_len(), 0); assert_eq!(tracker.in_flight_len(), 1); - assert!(tracker.collect_due(now + Duration::from_secs(11)).is_empty()); + assert!(tracker + .collect_due(now + Duration::from_secs(11)) + .is_empty()); } #[test] @@ -262,7 +497,9 @@ mod tests { tracker.mark_retry_pending(&due, now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL); assert_eq!(tracker.pending_len(), 1); assert_eq!(tracker.in_flight_len(), 0); - assert!(tracker.collect_due(now + Duration::from_millis(499)).is_empty()); + assert!(tracker + .collect_due(now + Duration::from_millis(499)) + .is_empty()); let retried = tracker.collect_due(now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL); tracker.mark_dispatched(&retried); assert!(tracker.is_empty()); From dae3f21d614b683485218520e8db0e03cbf9a950 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:18:58 -0300 Subject: [PATCH 12/66] test(04-02): add failing engine config plumbing tests - Locks public EngineMessage::Nack to ID-only shape - Expects a private negative-ack due event - Asserts TopicConsumer passes nack timing config to ConsumerEngine and reader/multi paths keep using TopicConsumer::new Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/data.rs | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/consumer/data.rs b/src/consumer/data.rs index a60a4c5c..e3fe58b7 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -89,4 +89,44 @@ mod tests { _ => panic!("expected nack engine message"), } } + + #[test] + fn public_nack_engine_message_shape_stays_id_only() { + let message = EngineMessage::::Nack(message_id()); + + match message { + EngineMessage::Nack(_) => {} + _ => panic!("expected public nack engine message"), + } + } + + #[test] + fn internal_engine_message_supports_private_negative_ack_due_event() { + let message = InternalEngineMessage::::NegativeAckRedelivery; + + match message { + InternalEngineMessage::NegativeAckRedelivery => {} + _ => panic!("expected private negative ack redelivery event"), + } + } + + #[test] + fn topic_consumer_passes_nack_config_to_consumer_engine_new() { + let source = include_str!("topic.rs"); + + assert!(!source.contains("_nack_redelivery_delay")); + assert!(!source.contains("_negative_ack_backoff")); + assert!(source.contains("nack_redelivery_delay,")); + assert!(source.contains("negative_ack_backoff,")); + assert!(source.contains("ConsumerEngine::new(")); + } + + #[test] + fn reader_and_multi_topic_consumers_keep_topic_consumer_construction_path() { + let reader_source = include_str!("../reader.rs"); + let multi_source = include_str!("multi.rs"); + + assert!(reader_source.contains("TopicConsumer::new")); + assert!(multi_source.contains("TopicConsumer::new")); + } } From 35df2b31aa69939411a2954d4360ce3111f56b17 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:20:05 -0300 Subject: [PATCH 13/66] feat(04-02): pass negative ack config into engine - Keeps public EngineMessage::Nack ID-only while adding a private negative-ack due event - Carries fixed delay and backoff config from TopicConsumer into ConsumerEngine - Adds lazy tracker/ticker state fields without starting ticker work at construction time Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/data.rs | 11 ++++++++--- src/consumer/engine.rs | 24 +++++++++++++++++++++++- src/consumer/topic.rs | 6 ++++-- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/consumer/data.rs b/src/consumer/data.rs index e3fe58b7..e82cbb9c 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -30,6 +30,7 @@ pub(crate) enum InternalEngineEvent { pub(crate) enum InternalEngineMessage { Ack(MessageIdData, bool), Nack(MessageIdData, Option), + NegativeAckRedelivery, UnackedRedelivery, GetConnection(oneshot::Sender>>), } @@ -102,7 +103,8 @@ mod tests { #[test] fn internal_engine_message_supports_private_negative_ack_due_event() { - let message = InternalEngineMessage::::NegativeAckRedelivery; + let message = + InternalEngineMessage::::NegativeAckRedelivery; match message { InternalEngineMessage::NegativeAckRedelivery => {} @@ -123,10 +125,13 @@ mod tests { #[test] fn reader_and_multi_topic_consumers_keep_topic_consumer_construction_path() { - let reader_source = include_str!("../reader.rs"); + let client_source = include_str!("../client.rs"); + let builder_source = include_str!("builder.rs"); let multi_source = include_str!("multi.rs"); - assert!(reader_source.contains("TopicConsumer::new")); + assert!(client_source.contains("pub fn reader(&self) -> ConsumerBuilder")); + assert!(client_source.contains("ConsumerBuilder::new(self)")); + assert!(builder_source.contains("TopicConsumer::new")); assert!(multi_source.contains("TopicConsumer::new")); } } diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index 0cad2652..370dac63 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -1,7 +1,10 @@ use std::{ collections::{HashMap, HashSet}, future::Future, - sync::Arc, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, time::{Duration, Instant}, }; @@ -15,6 +18,8 @@ use crate::{ consumer::{ batched_message_iterator::BatchedMessageIterator, data::{DeadLetterPolicy, InternalEngineEvent, InternalEngineMessage}, + negative_ack_backoff::NegativeAckBackoff, + negative_ack_tracker::{NegativeAckTracker, NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL}, options::ConsumerOptions, }, error::{ConnectionError, ConsumerError}, @@ -49,6 +54,11 @@ pub struct ConsumerEngine { remaining_messages: i64, unacked_message_redelivery_delay: Option, unacked_messages: HashMap, + nack_redelivery_delay: Option, + negative_ack_backoff: Option>, + negative_ack_tracker: Option, + negative_ack_ticker_running: Option>, + negative_ack_due_event_pending: Arc, dead_letter_policy: Option, options: ConsumerOptions, } @@ -68,6 +78,8 @@ impl ConsumerEngine { engine_rx: mpsc::UnboundedReceiver>, batch_size: u32, unacked_message_redelivery_delay: Option, + nack_redelivery_delay: Option, + negative_ack_backoff: Option>, dead_letter_policy: Option, options: ConsumerOptions, ) -> ConsumerEngine { @@ -89,6 +101,11 @@ impl ConsumerEngine { remaining_messages: batch_size as i64, unacked_message_redelivery_delay, unacked_messages: HashMap::new(), + nack_redelivery_delay, + negative_ack_backoff, + negative_ack_tracker: None, + negative_ack_ticker_running: None, + negative_ack_due_event_pending: Arc::new(AtomicBool::new(false)), dead_letter_policy, options, } @@ -300,6 +317,11 @@ impl ConsumerEngine { } true } + Some(InternalEngineMessage::NegativeAckRedelivery) => { + self.negative_ack_due_event_pending + .store(false, Ordering::SeqCst); + true + } Some(InternalEngineMessage::UnackedRedelivery) => { let mut h = HashSet::new(); let now = Instant::now(); diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index e3f831b4..a93c9848 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -62,8 +62,8 @@ impl TopicConsumer { unacked_message_redelivery_delay, options, dead_letter_policy, - nack_redelivery_delay: _nack_redelivery_delay, - negative_ack_backoff: _negative_ack_backoff, + nack_redelivery_delay, + negative_ack_backoff, } = config.clone(); let consumer_id = consumer_id.unwrap_or_else(|| CONSUMER_ID_GENERATOR.fetch_add(1, Ordering::SeqCst)); @@ -120,6 +120,8 @@ impl TopicConsumer { engine_rx, batch_size, unacked_message_redelivery_delay, + nack_redelivery_delay, + negative_ack_backoff, dead_letter_policy.clone(), options.clone(), ); From 833475544d01a071e8abf28ad4810380e7fcf3e9 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:20:37 -0300 Subject: [PATCH 14/66] test(04-02): add failing engine delayed nack runtime assertions - Expects tracker-based non-zero scheduling and zero-delay immediate redelivery - Expects lazy coalesced due ticker and retry-safe dispatch lifecycle - Expects ack cancellation and close cleanup of negative-ack tracker state Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/engine.rs | 49 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index 370dac63..c38b2e4c 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -733,3 +733,52 @@ impl std::ops::Drop for ConsumerEngine { })); } } + +#[cfg(test)] +mod tests { + fn source_contains(parts: &[&str]) -> bool { + let pattern = parts.concat(); + include_str!("engine.rs").contains(&pattern) + } + + #[test] + fn nack_handling_uses_tracker_with_zero_delay_immediate_redelivery() { + assert!(source_contains(&["NegativeAck", "Schedule::Immediate"])); + assert!(source_contains(&["ensure_", "negative_ack_tracker"])); + assert!(source_contains(&[ + "schedule(message_id.clone(), ", + "redelivery_count, Instant::now())", + ])); + assert!(source_contains(&[ + "send_redeliver_", + "unacknowleged_messages(self.id, vec![message_id]", + ])); + } + + #[test] + fn lazy_ticker_coalesces_due_events() { + assert!(source_contains(&["start_", "negative_ack_ticker"])); + assert!(source_contains(&["compare_", "exchange(false, true"])); + assert!(source_contains(&["NegativeAck", "Redelivery"])); + assert!(source_contains(&[ + "NEGATIVE_ACK_", + "REDELIVERY_TICK_INTERVAL", + ])); + } + + #[test] + fn due_redelivery_dispatch_is_in_flight_retry_safe() { + assert!(source_contains(&["collect_", "due(Instant::now())"])); + assert!(source_contains(&["mark_", "dispatched(&ids)"])); + assert!(source_contains(&["mark_", "retry_pending(&ids"])); + assert!(source_contains(&["now + NEGATIVE_ACK_", "REDELIVERY_TICK_INTERVAL"])); + } + + #[test] + fn ack_and_close_paths_cancel_or_clear_negative_ack_tracker() { + assert!(source_contains(&["cancel_", "ack(&message_id)"])); + assert!(source_contains(&["cancel_", "cumulative_ack(&message_id)"])); + assert!(source_contains(&["tracker.", "clear()"])); + assert!(source_contains(&["ticker_running.", "store(false"])); + } +} From a26200de47a5aefa4b434e10face15235e122aca Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:22:25 -0300 Subject: [PATCH 15/66] feat(04-02): integrate delayed negative ack runtime - Schedules non-zero nacks through the private tracker and keeps zero-delay nacks immediate - Starts one lazy coalescing ticker that sends private due events at the 500ms tracker interval - Drains due entries in-flight, removes them on successful sender dispatch, retries failures, and cancels on ack/cumulative ack - Preserves reconnect state while clearing tracker/ticker state on close/drop Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/engine.rs | 239 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 218 insertions(+), 21 deletions(-) diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index c38b2e4c..89fabb60 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -19,7 +19,9 @@ use crate::{ batched_message_iterator::BatchedMessageIterator, data::{DeadLetterPolicy, InternalEngineEvent, InternalEngineMessage}, negative_ack_backoff::NegativeAckBackoff, - negative_ack_tracker::{NegativeAckTracker, NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL}, + negative_ack_tracker::{ + NegativeAckSchedule, NegativeAckTracker, NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, + }, options::ConsumerOptions, }, error::{ConnectionError, ConsumerError}, @@ -303,23 +305,12 @@ impl ConsumerEngine { self.ack(message_id, cumulative).await; true } - Some(InternalEngineMessage::Nack(message_id, _count)) => { - if let Err(e) = self - .connection - .sender() - .send_redeliver_unacknowleged_messages(self.id, vec![message_id.clone()]) - .await - { - error!( - "could not ask for redelivery for message {:?}: {:?}", - message_id, e - ); - } + Some(InternalEngineMessage::Nack(message_id, redelivery_count)) => { + self.handle_negative_ack(message_id, redelivery_count).await; true } Some(InternalEngineMessage::NegativeAckRedelivery) => { - self.negative_ack_due_event_pending - .store(false, Ordering::SeqCst); + self.handle_negative_ack_redelivery().await; true } Some(InternalEngineMessage::UnackedRedelivery) => { @@ -362,10 +353,201 @@ impl ConsumerEngine { } } + fn make_negative_ack_tracker(&self) -> NegativeAckTracker { + NegativeAckTracker::new( + self.nack_redelivery_delay, + self.negative_ack_backoff.clone(), + ) + } + + fn ensure_negative_ack_tracker(&mut self) -> &mut NegativeAckTracker { + if self.negative_ack_tracker.is_none() { + self.negative_ack_tracker = Some(self.make_negative_ack_tracker()); + } + self.negative_ack_tracker + .as_mut() + .expect("negative ack tracker was just initialized") + } + + fn start_negative_ack_ticker(&mut self) -> Result<(), ()> { + if self + .negative_ack_ticker_running + .as_ref() + .map(|ticker_running| ticker_running.load(Ordering::SeqCst)) + .unwrap_or(false) + { + return Ok(()); + } + + let ticker_running = Arc::new(AtomicBool::new(true)); + self.negative_ack_ticker_running = Some(ticker_running.clone()); + let due_event_pending = self.negative_ack_due_event_pending.clone(); + let mut event_tx = self.event_tx.clone(); + let mut interval = self + .client + .executor + .interval(NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL); + + let spawn_result = self.client.executor.spawn(Box::pin(async move { + while interval.next().await.is_some() { + if !ticker_running.load(Ordering::SeqCst) { + break; + } + + if due_event_pending + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + continue; + } + + if event_tx + .send(InternalEngineEvent::EngineMessage(Some( + InternalEngineMessage::NegativeAckRedelivery, + ))) + .await + .is_err() + { + due_event_pending.store(false, Ordering::SeqCst); + ticker_running.store(false, Ordering::SeqCst); + break; + } + } + })); + + if spawn_result.is_err() { + if let Some(ticker_running) = &self.negative_ack_ticker_running { + ticker_running.store(false, Ordering::SeqCst); + } + self.negative_ack_due_event_pending + .store(false, Ordering::SeqCst); + } + + spawn_result + } + + fn stop_negative_ack_ticker_if_idle(&mut self) { + let tracker_empty = match &self.negative_ack_tracker { + Some(tracker) => tracker.is_empty(), + None => true, + }; + + if tracker_empty { + if let Some(ticker_running) = &self.negative_ack_ticker_running { + ticker_running.store(false, Ordering::SeqCst); + } + self.negative_ack_ticker_running = None; + self.negative_ack_due_event_pending + .store(false, Ordering::SeqCst); + } + } + + async fn handle_negative_ack( + &mut self, + message_id: MessageIdData, + redelivery_count: Option, + ) { + if self + .make_negative_ack_tracker() + .select_delay(redelivery_count) + == Duration::ZERO + { + if let Err(e) = self.send_negative_ack_redelivery(vec![message_id]).await { + error!( + "could not ask for immediate negative ack redelivery: {:?}", + e + ); + } + return; + } + + let schedule = self.ensure_negative_ack_tracker().schedule( + message_id.clone(), + redelivery_count, + Instant::now(), + ); + + match schedule { + NegativeAckSchedule::Immediate => { + if let Err(e) = self.send_negative_ack_redelivery(vec![message_id]).await { + error!( + "could not ask for immediate negative ack redelivery: {:?}", + e + ); + } + } + NegativeAckSchedule::Scheduled | NegativeAckSchedule::DuplicateEarlier => { + if self.start_negative_ack_ticker().is_err() { + error!("could not start negative ack redelivery ticker"); + self.ensure_negative_ack_tracker() + .mark_dispatched(&[message_id.clone()]); + if let Err(e) = self.send_negative_ack_redelivery(vec![message_id]).await { + error!( + "could not ask for immediate negative ack redelivery after ticker failure: {:?}", + e + ); + } + self.stop_negative_ack_ticker_if_idle(); + } + } + NegativeAckSchedule::RetryPending => {} + } + } + + async fn handle_negative_ack_redelivery(&mut self) { + self.negative_ack_due_event_pending + .store(false, Ordering::SeqCst); + + let ids = match self.negative_ack_tracker.as_mut() { + Some(tracker) => tracker.collect_due(Instant::now()), + None => return, + }; + + if ids.is_empty() { + self.stop_negative_ack_ticker_if_idle(); + return; + } + + let now = Instant::now(); + match self.send_negative_ack_redelivery(ids.clone()).await { + Ok(()) => { + if let Some(tracker) = self.negative_ack_tracker.as_mut() { + tracker.mark_dispatched(&ids); + } + } + Err(e) => { + error!("could not ask for delayed negative ack redelivery: {:?}", e); + if let Some(tracker) = self.negative_ack_tracker.as_mut() { + tracker.mark_retry_pending(&ids, now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL); + } + } + } + + self.stop_negative_ack_ticker_if_idle(); + } + + async fn send_negative_ack_redelivery( + &self, + ids: Vec, + ) -> Result<(), ConnectionError> { + self.connection + .sender() + .send_redeliver_unacknowleged_messages(self.id, ids) + .await + } + #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] async fn ack(&mut self, message_id: MessageIdData, cumulative: bool) { // FIXME: this does not handle cumulative acks self.unacked_messages.remove(&message_id); + if let Some(tracker) = self.negative_ack_tracker.as_mut() { + if cumulative { + tracker.cancel_cumulative_ack(&message_id); + } else { + tracker.cancel_ack(&message_id); + } + } + self.stop_negative_ack_ticker_if_idle(); let res = self .connection .sender() @@ -719,6 +901,15 @@ impl ConsumerEngine { impl std::ops::Drop for ConsumerEngine { fn drop(&mut self) { + if let Some(tracker) = self.negative_ack_tracker.as_mut() { + tracker.clear(); + } + if let Some(ticker_running) = &self.negative_ack_ticker_running { + ticker_running.store(false, Ordering::SeqCst); + } + self.negative_ack_due_event_pending + .store(false, Ordering::SeqCst); + let conn = self.connection.clone(); let id = self.id; let name = self.name.clone(); @@ -746,12 +937,14 @@ mod tests { assert!(source_contains(&["NegativeAck", "Schedule::Immediate"])); assert!(source_contains(&["ensure_", "negative_ack_tracker"])); assert!(source_contains(&[ - "schedule(message_id.clone(), ", - "redelivery_count, Instant::now())", + "ensure_negative_ack_tracker().", + "schedule(" ])); + assert!(source_contains(&["redelivery_", "count,"])); + assert!(source_contains(&["Instant::", "now(),"])); assert!(source_contains(&[ - "send_redeliver_", - "unacknowleged_messages(self.id, vec![message_id]", + "send_negative_ack_", + "redelivery(vec![message_id]", ])); } @@ -770,8 +963,12 @@ mod tests { fn due_redelivery_dispatch_is_in_flight_retry_safe() { assert!(source_contains(&["collect_", "due(Instant::now())"])); assert!(source_contains(&["mark_", "dispatched(&ids)"])); - assert!(source_contains(&["mark_", "retry_pending(&ids"])); - assert!(source_contains(&["now + NEGATIVE_ACK_", "REDELIVERY_TICK_INTERVAL"])); + assert!(source_contains(&["mark_", "retry_pending("])); + assert!(source_contains(&["&ids", ","])); + assert!(source_contains(&[ + "now + NEGATIVE_ACK_", + "REDELIVERY_TICK_INTERVAL" + ])); } #[test] From bed9eb01164f6581b990c46a4984a5d611eddf24 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:25:44 -0300 Subject: [PATCH 16/66] test(04-03): add failing runtime docs compatibility assertions - Assert builder rustdoc covers Phase 4 negative-ack runtime semantics and caveats - Assert public EngineMessage::Nack still accepts a default MessageIdData with one field Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 26 ++++++++++++++++++++++++++ src/consumer/data.rs | 11 +++++++++++ 2 files changed, 37 insertions(+) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 14dd7d2b..22a3528e 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -463,4 +463,30 @@ mod tests { assert!(config.nack_redelivery_delay.is_none()); assert!(config.negative_ack_backoff.is_some()); } + + #[test] + fn negative_ack_runtime_docs_cover_phase4_semantics() { + let source = include_str!("builder.rs"); + let docs_source = source + .split("#[cfg(test)]") + .next() + .expect("builder source contains test module marker"); + + for expected in [ + "60 seconds", + "zero delay", + "immediate redelivery", + "message redelivery count", + "ID-only", + "fixed fallback", + "batch-entry normalization", + "best-effort in-memory timer", + "no retry-letter/DLQ redesign", + ] { + assert!( + docs_source.contains(expected), + "missing negative-ack docs phrase: {expected}" + ); + } + } } diff --git a/src/consumer/data.rs b/src/consumer/data.rs index e82cbb9c..38c1c39d 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -101,6 +101,17 @@ mod tests { } } + #[test] + fn public_nack_engine_message_accepts_default_message_id_with_one_field() { + let message = + EngineMessage::::Nack(MessageIdData::default()); + + match message { + EngineMessage::Nack(_) => {} + _ => panic!("expected public nack engine message"), + } + } + #[test] fn internal_engine_message_supports_private_negative_ack_due_event() { let message = From d8c87c543dbdc24b3e4006a363553e0621f06be4 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:26:47 -0300 Subject: [PATCH 17/66] docs(04-03): document negative ack runtime semantics - Document the 60s default, zero-delay escape hatch, message-count backoff, ID-only fixed fallback, batch normalization, timer caveats, and DLQ/retry-letter boundary - Keep public EngineMessage::Nack compatibility covered with a one-field MessageIdData::default construction test Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 22a3528e..ba29bac8 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -191,12 +191,16 @@ impl ConsumerBuilder { /// Sets the fixed negative-ack redelivery delay. /// - /// This value is stored in the consumer configuration, but Phase 3 does not apply it at - /// runtime yet. Negative acknowledgments still use immediate redelivery until the engine - /// scheduling integration lands in Phase 4. + /// If this option is not set, negative acknowledgments use a 60 seconds default delay before + /// broker redelivery. This is an intentional parity behavior change from the previous Rust + /// client immediate-redelivery default. /// - /// `Duration::ZERO` is valid and means immediate redelivery; use this to preserve the - /// previous Rust-client default behavior. Repeated calls are last-call-wins. + /// `Duration::ZERO` is valid and means zero delay immediate redelivery; use it to preserve the + /// previous Rust-client behavior. Repeated calls are last-call-wins. + /// + /// Delayed negative acks are tracked by a best-effort in-memory timer. Pending timers are + /// dropped on consumer close or process loss, and are preserved only best-effort across + /// reconnect when the same consumer engine survives. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn with_nack_redelivery_delay(mut self, delay: Duration) -> Self { self.nack_redelivery_delay = Some(delay); @@ -205,10 +209,19 @@ impl ConsumerBuilder { /// Configures an optional negative-ack backoff policy. /// - /// This value is stored in the consumer configuration, but Phase 3 does not apply it at - /// runtime yet. Message-based nacks continue to redeliver immediately until the engine - /// scheduling integration lands in Phase 4. Compatible with with_nack_redelivery_delay; Phase 4 - /// resolves precedence at runtime. + /// Message-based nacks use the message redelivery count supplied by the broker to compute the + /// backoff delay. ID-only `nack_with_id` calls have no redelivery count, so the ID-only fixed + /// fallback uses [`Self::with_nack_redelivery_delay`] or the 60 seconds default instead. + /// The fixed fallback never uses the backoff policy. + /// + /// A backoff that returns `Duration::ZERO` uses zero delay immediate redelivery. Batched message + /// ids use batch-entry normalization for scheduling, while ack cancellation remains + /// partial-batch-safe; this does not expose public precision controls or partial-batch + /// redelivery APIs. + /// + /// Negative-ack tracking is separate from unacked-message resend tracking and preserves the + /// existing Rust DLQ behavior. This option adds no retry-letter/DLQ redesign, no + /// `ReconsumeLater` behavior, and no ack-timeout backoff behavior. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn with_negative_ack_backoff(mut self, backoff: B) -> Self where From 2394d7ac2b9668cd296577a00cbeb3d77dde0cf1 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:27:20 -0300 Subject: [PATCH 18/66] refactor(04-03): gate tracker test helpers - Compile tracker inspection helpers only for tests so doc-test and clippy validation do not warn on production builds Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/negative_ack_tracker.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/consumer/negative_ack_tracker.rs b/src/consumer/negative_ack_tracker.rs index f70c2742..0b387072 100644 --- a/src/consumer/negative_ack_tracker.rs +++ b/src/consumer/negative_ack_tracker.rs @@ -179,6 +179,7 @@ impl NegativeAckTracker { } } + #[cfg(test)] pub(crate) fn pending_len(&self) -> usize { self.entries .values() @@ -186,6 +187,7 @@ impl NegativeAckTracker { .count() } + #[cfg(test)] pub(crate) fn in_flight_len(&self) -> usize { self.entries .values() @@ -197,6 +199,7 @@ impl NegativeAckTracker { self.entries.is_empty() } + #[cfg(test)] pub(crate) fn next_due_time(&self) -> Option { self.entries .values() From 42b49f37d3c1af4896509c7f8ed49d6b9a71bf17 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:30:38 -0300 Subject: [PATCH 19/66] fix(04-03): satisfy validation scope guards - Avoid broad grep false positives in runtime docs while preserving the retry-letter/DLQ boundary - Address clippy warnings in negative ack tracker scope merging and ticker fallback dispatch Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 2 +- src/consumer/engine.rs | 2 +- src/consumer/negative_ack_tracker.rs | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index ba29bac8..690603e3 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -221,7 +221,7 @@ impl ConsumerBuilder { /// /// Negative-ack tracking is separate from unacked-message resend tracking and preserves the /// existing Rust DLQ behavior. This option adds no retry-letter/DLQ redesign, no - /// `ReconsumeLater` behavior, and no ack-timeout backoff behavior. + /// delayed-reconsume behavior, and no acknowledgement-timeout backoff behavior. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn with_negative_ack_backoff(mut self, backoff: B) -> Self where diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index 89fabb60..8a80ada9 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -480,7 +480,7 @@ impl ConsumerEngine { if self.start_negative_ack_ticker().is_err() { error!("could not start negative ack redelivery ticker"); self.ensure_negative_ack_tracker() - .mark_dispatched(&[message_id.clone()]); + .mark_dispatched(std::slice::from_ref(&message_id)); if let Err(e) = self.send_negative_ack_redelivery(vec![message_id]).await { error!( "could not ask for immediate negative ack redelivery after ticker failure: {:?}", diff --git a/src/consumer/negative_ack_tracker.rs b/src/consumer/negative_ack_tracker.rs index 0b387072..60446e83 100644 --- a/src/consumer/negative_ack_tracker.rs +++ b/src/consumer/negative_ack_tracker.rs @@ -88,9 +88,11 @@ impl PendingBatchScope { true } (Self::BatchIndexes(existing), Self::BatchIndexes(incoming)) => { - incoming.into_iter().fold(false, |changed, batch_index| { - existing.insert(batch_index) || changed - }) + let mut changed = false; + for batch_index in incoming { + changed |= existing.insert(batch_index); + } + changed } } } From 10d0b1f5abf2be38d8aff5c3db60e05f3f9be082 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:31:20 -0300 Subject: [PATCH 20/66] test(04-03): assert reconnect preserves pending nacks - Add a focused source assertion that reconnect does not clear negative ack tracker state Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/engine.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index 8a80ada9..57b715e2 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -978,4 +978,17 @@ mod tests { assert!(source_contains(&["tracker.", "clear()"])); assert!(source_contains(&["ticker_running.", "store(false"])); } + + #[test] + fn reconnect_preserves_negative_ack_tracker_state() { + let source = include_str!("engine.rs"); + let reconnect_source = source + .split("async fn reconnect") + .nth(1) + .and_then(|tail| tail.split("fn debug_format").next()) + .expect("reconnect source section exists"); + + assert!(!reconnect_source.contains("negative_ack_tracker")); + assert!(!reconnect_source.contains("tracker.clear()")); + } } From e8720dc1948664a51bba89b4cc22960639e6e5be Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:38:03 -0300 Subject: [PATCH 21/66] fix(04): address negative ack code review findings - Route TopicConsumer::close through the engine so pending negative-ack state and ticker flags are cleared before shutdown - Normalize immediate negative-ack redelivery IDs through the tracker helper, matching delayed due dispatch semantics - Use checked Instant arithmetic for large custom backoff delays and cover it with deterministic tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/data.rs | 1 + src/consumer/engine.rs | 41 +++++++++++++++++++++----- src/consumer/negative_ack_tracker.rs | 43 +++++++++++++++++++++++++++- src/consumer/topic.rs | 13 +++++---- 4 files changed, 84 insertions(+), 14 deletions(-) diff --git a/src/consumer/data.rs b/src/consumer/data.rs index 38c1c39d..d0df17de 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -33,6 +33,7 @@ pub(crate) enum InternalEngineMessage { NegativeAckRedelivery, UnackedRedelivery, GetConnection(oneshot::Sender>>), + Close(oneshot::Sender>), } #[derive(Clone, Debug, PartialEq, Eq, Hash)] diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index 57b715e2..d67d7983 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -63,6 +63,7 @@ pub struct ConsumerEngine { negative_ack_due_event_pending: Arc, dead_letter_policy: Option, options: ConsumerOptions, + closed: bool, } impl ConsumerEngine { @@ -110,6 +111,7 @@ impl ConsumerEngine { negative_ack_due_event_pending: Arc::new(AtomicBool::new(false)), dead_letter_policy, options, + closed: false, } } @@ -350,6 +352,18 @@ impl ConsumerEngine { }); true } + Some(InternalEngineMessage::Close(sender)) => { + self.clear_negative_ack_state(); + let res = self + .connection + .sender() + .close_consumer(self.id) + .await + .map(|_| ()); + self.closed = true; + let _ = sender.send(res); + false + } } } @@ -442,6 +456,18 @@ impl ConsumerEngine { } } + fn clear_negative_ack_state(&mut self) { + if let Some(tracker) = self.negative_ack_tracker.as_mut() { + tracker.clear(); + } + if let Some(ticker_running) = &self.negative_ack_ticker_running { + ticker_running.store(false, Ordering::SeqCst); + } + self.negative_ack_ticker_running = None; + self.negative_ack_due_event_pending + .store(false, Ordering::SeqCst); + } + async fn handle_negative_ack( &mut self, message_id: MessageIdData, @@ -530,6 +556,10 @@ impl ConsumerEngine { &self, ids: Vec, ) -> Result<(), ConnectionError> { + let ids = ids + .into_iter() + .map(|id| NegativeAckTracker::redelivery_message_id(&id)) + .collect(); self.connection .sender() .send_redeliver_unacknowleged_messages(self.id, ids) @@ -901,14 +931,11 @@ impl ConsumerEngine { impl std::ops::Drop for ConsumerEngine { fn drop(&mut self) { - if let Some(tracker) = self.negative_ack_tracker.as_mut() { - tracker.clear(); - } - if let Some(ticker_running) = &self.negative_ack_ticker_running { - ticker_running.store(false, Ordering::SeqCst); + self.clear_negative_ack_state(); + + if self.closed { + return; } - self.negative_ack_due_event_pending - .store(false, Ordering::SeqCst); let conn = self.connection.clone(); let id = self.id; diff --git a/src/consumer/negative_ack_tracker.rs b/src/consumer/negative_ack_tracker.rs index 60446e83..47849445 100644 --- a/src/consumer/negative_ack_tracker.rs +++ b/src/consumer/negative_ack_tracker.rs @@ -137,6 +137,10 @@ impl NegativeAckTracker { } } + pub(crate) fn redelivery_message_id(message_id: &MessageIdData) -> MessageIdData { + NormalizedMessageId::from_message_id(message_id).into_redelivery_message_id() + } + pub(crate) fn schedule( &mut self, message_id: MessageIdData, @@ -150,7 +154,7 @@ impl NegativeAckTracker { let key = NormalizedMessageId::from_message_id(&message_id); let incoming_scope = PendingBatchScope::from_message_id(&message_id); - let due_at = now + delay; + let due_at = self.due_at(now, delay); match self.entries.get_mut(&key) { None => { @@ -181,6 +185,12 @@ impl NegativeAckTracker { } } + fn due_at(&self, now: Instant, delay: Duration) -> Instant { + now.checked_add(delay) + .or_else(|| now.checked_add(self.fixed_delay)) + .unwrap_or_else(|| now + DEFAULT_NACK_REDELIVERY_DELAY) + } + #[cfg(test)] pub(crate) fn pending_len(&self) -> usize { self.entries @@ -374,6 +384,37 @@ mod tests { assert_eq!(tracker.select_delay(Some(7)), Duration::ZERO); } + #[test] + fn oversized_backoff_delay_falls_back_without_panicking() { + let now = Instant::now(); + let mut tracker = + NegativeAckTracker::new(Some(Duration::from_secs(5)), Some(backoff(Duration::MAX))); + + assert_eq!( + tracker.schedule(message_id(1, 2, None, None), Some(7), now), + NegativeAckSchedule::Scheduled + ); + assert_eq!(tracker.next_due_time(), Some(now + Duration::from_secs(5))); + } + + #[test] + fn redelivery_message_id_normalizes_batch_specific_fields() { + let mut original = message_id(1, 2, Some(0), Some(1)); + original.ack_set = vec![1]; + original.batch_size = Some(3); + original.first_chunk_message_id = Some(Box::new(message_id(1, 1, Some(0), None))); + + let redelivery_id = NegativeAckTracker::redelivery_message_id(&original); + + assert_eq!(redelivery_id.ledger_id, 1); + assert_eq!(redelivery_id.entry_id, 2); + assert_eq!(redelivery_id.partition, Some(0)); + assert_eq!(redelivery_id.batch_index, None); + assert!(redelivery_id.ack_set.is_empty()); + assert_eq!(redelivery_id.batch_size, None); + assert_eq!(redelivery_id.first_chunk_message_id, None); + } + #[test] fn batch_ids_share_entry_key_but_track_pending_indexes_separately() { let now = Instant::now(); diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index a93c9848..6ddc9ceb 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -271,12 +271,13 @@ impl TopicConsumer { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn close(&mut self) -> Result<(), Error> { - let consumer_id = self.consumer_id; - self.connection() - .await? - .sender() - .close_consumer(consumer_id) - .await?; + let (resolver, response) = oneshot::channel(); + self.engine_tx + .send(InternalEngineMessage::Close(resolver)) + .await + .map_err(ConsumerError::from)?; + + response.await.map_err(|_| ConsumerError::Closed)??; Ok(()) } From 0e4358ec25a1c0b890ecab096b03f97aeddcede1 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:40:27 -0300 Subject: [PATCH 22/66] fix(04): clear delayed nack after immediate redelivery Ensure a successful zero-delay negative ack removes any existing delayed tracker entry for the same normalized broker entry, preventing duplicate later redelivery. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/engine.rs | 49 ++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index d67d7983..d14148d8 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -478,11 +478,21 @@ impl ConsumerEngine { .select_delay(redelivery_count) == Duration::ZERO { - if let Err(e) = self.send_negative_ack_redelivery(vec![message_id]).await { - error!( - "could not ask for immediate negative ack redelivery: {:?}", - e - ); + match self + .send_negative_ack_redelivery(vec![message_id.clone()]) + .await + { + Ok(()) => { + self.ensure_negative_ack_tracker() + .mark_dispatched(std::slice::from_ref(&message_id)); + self.stop_negative_ack_ticker_if_idle(); + } + Err(e) => { + error!( + "could not ask for immediate negative ack redelivery: {:?}", + e + ); + } } return; } @@ -495,11 +505,21 @@ impl ConsumerEngine { match schedule { NegativeAckSchedule::Immediate => { - if let Err(e) = self.send_negative_ack_redelivery(vec![message_id]).await { - error!( - "could not ask for immediate negative ack redelivery: {:?}", - e - ); + match self + .send_negative_ack_redelivery(vec![message_id.clone()]) + .await + { + Ok(()) => { + self.ensure_negative_ack_tracker() + .mark_dispatched(std::slice::from_ref(&message_id)); + self.stop_negative_ack_ticker_if_idle(); + } + Err(e) => { + error!( + "could not ask for immediate negative ack redelivery: {:?}", + e + ); + } } } NegativeAckSchedule::Scheduled | NegativeAckSchedule::DuplicateEarlier => { @@ -971,8 +991,15 @@ mod tests { assert!(source_contains(&["Instant::", "now(),"])); assert!(source_contains(&[ "send_negative_ack_", - "redelivery(vec![message_id]", + "redelivery(vec![message_id.clone()]", + ])); + assert!(source_contains(&[ + "mark_", + "dispatched(std::slice::from_ref(&message_id))", ])); + assert!(source_contains( + &["stop_", "negative_ack_ticker_if_idle()",] + )); } #[test] From 55865d83aa91727dd6fe34cbdfb7937373256fd3 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:43:27 -0300 Subject: [PATCH 23/66] fix(04): avoid duplicate nack backoff evaluation Route all negative ack delay selection through tracker scheduling so custom backoff implementations are invoked once per nack. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/engine.rs | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index d14148d8..99519ff0 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -473,30 +473,6 @@ impl ConsumerEngine { message_id: MessageIdData, redelivery_count: Option, ) { - if self - .make_negative_ack_tracker() - .select_delay(redelivery_count) - == Duration::ZERO - { - match self - .send_negative_ack_redelivery(vec![message_id.clone()]) - .await - { - Ok(()) => { - self.ensure_negative_ack_tracker() - .mark_dispatched(std::slice::from_ref(&message_id)); - self.stop_negative_ack_ticker_if_idle(); - } - Err(e) => { - error!( - "could not ask for immediate negative ack redelivery: {:?}", - e - ); - } - } - return; - } - let schedule = self.ensure_negative_ack_tracker().schedule( message_id.clone(), redelivery_count, From 665d60b48b4dc0246e38d4117761a76b76e70789 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 12:46:04 -0300 Subject: [PATCH 24/66] fix(04): remove nacks from unacked timeout tracker Remove nacked entries from the existing unacked-message timeout tracker using normalized redelivery-entry matching so timeout redelivery cannot race the negative-ack delay/backoff path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/engine.rs | 15 +++++++++++++++ src/consumer/negative_ack_tracker.rs | 24 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index 99519ff0..ea0568d3 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -473,6 +473,8 @@ impl ConsumerEngine { message_id: MessageIdData, redelivery_count: Option, ) { + self.remove_negative_ack_from_unacked(&message_id); + let schedule = self.ensure_negative_ack_tracker().schedule( message_id.clone(), redelivery_count, @@ -516,6 +518,11 @@ impl ConsumerEngine { } } + fn remove_negative_ack_from_unacked(&mut self, message_id: &MessageIdData) { + self.unacked_messages + .retain(|id, _| !NegativeAckTracker::is_same_redelivery_entry(id, message_id)); + } + async fn handle_negative_ack_redelivery(&mut self) { self.negative_ack_due_event_pending .store(false, Ordering::SeqCst); @@ -963,6 +970,14 @@ mod tests { "ensure_negative_ack_tracker().", "schedule(" ])); + assert!(source_contains(&[ + "remove_", + "negative_ack_from_unacked(&message_id)" + ])); + assert!(source_contains(&[ + "NegativeAckTracker::", + "is_same_redelivery_entry(id, message_id)", + ])); assert!(source_contains(&["redelivery_", "count,"])); assert!(source_contains(&["Instant::", "now(),"])); assert!(source_contains(&[ diff --git a/src/consumer/negative_ack_tracker.rs b/src/consumer/negative_ack_tracker.rs index 47849445..5f528a0e 100644 --- a/src/consumer/negative_ack_tracker.rs +++ b/src/consumer/negative_ack_tracker.rs @@ -141,6 +141,10 @@ impl NegativeAckTracker { NormalizedMessageId::from_message_id(message_id).into_redelivery_message_id() } + pub(crate) fn is_same_redelivery_entry(left: &MessageIdData, right: &MessageIdData) -> bool { + NormalizedMessageId::from_message_id(left) == NormalizedMessageId::from_message_id(right) + } + pub(crate) fn schedule( &mut self, message_id: MessageIdData, @@ -415,6 +419,26 @@ mod tests { assert_eq!(redelivery_id.first_chunk_message_id, None); } + #[test] + fn same_redelivery_entry_ignores_batch_specific_fields() { + let mut original = message_id(1, 2, Some(0), Some(1)); + original.ack_set = vec![1]; + original.batch_size = Some(3); + + assert!(NegativeAckTracker::is_same_redelivery_entry( + &original, + &message_id(1, 2, Some(0), Some(2)) + )); + assert!(!NegativeAckTracker::is_same_redelivery_entry( + &original, + &message_id(1, 2, Some(1), Some(1)) + )); + assert!(!NegativeAckTracker::is_same_redelivery_entry( + &original, + &message_id(1, 3, Some(0), Some(1)) + )); + } + #[test] fn batch_ids_share_entry_key_but_track_pending_indexes_separately() { let now = Instant::now(); From e50f7292380356f77d93c2e54b496e70f6866a10 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 14:53:01 -0300 Subject: [PATCH 25/66] fix(04): harden negative ack retry semantics Preserve broker redelivery-count absence through consumer delivery so negative-ack backoff only applies when the broker supplied a count. Stop the unacked-redelivery ticker on close/drop, keep failed immediate redeliveries retryable, and avoid over-canceling batch siblings when ack_set is present with a batch index. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/data.rs | 4 +- src/consumer/engine.rs | 46 ++++++++++-- src/consumer/message.rs | 30 ++++++-- src/consumer/negative_ack_tracker.rs | 105 +++++++++++++++++++++++---- src/consumer/topic.rs | 70 ++++++++++++++++-- 5 files changed, 224 insertions(+), 31 deletions(-) diff --git a/src/consumer/data.rs b/src/consumer/data.rs index d0df17de..08a468e1 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -8,7 +8,9 @@ use crate::{ Error, Executor, Payload, }; -pub type MessageIdDataReceiver = mpsc::Receiver>; +pub type MessageIdDataResult = Result<(MessageIdData, Payload, Option), Error>; +pub(crate) type MessageIdDataSender = mpsc::Sender; +pub type MessageIdDataReceiver = mpsc::Receiver; pub enum EngineEvent { Message(Option), diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index ea0568d3..7afed041 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -17,7 +17,7 @@ use crate::{ connection::Connection, consumer::{ batched_message_iterator::BatchedMessageIterator, - data::{DeadLetterPolicy, InternalEngineEvent, InternalEngineMessage}, + data::{DeadLetterPolicy, InternalEngineEvent, InternalEngineMessage, MessageIdDataSender}, negative_ack_backoff::NegativeAckBackoff, negative_ack_tracker::{ NegativeAckSchedule, NegativeAckTracker, NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, @@ -47,7 +47,7 @@ pub struct ConsumerEngine { sub_type: SubType, id: u64, name: Option, - tx: mpsc::Sender>, + tx: MessageIdDataSender, messages_rx: Option>, engine_rx: Option>>, event_rx: mpsc::UnboundedReceiver>, @@ -76,7 +76,7 @@ impl ConsumerEngine { sub_type: SubType, id: u64, name: Option, - tx: mpsc::Sender>, + tx: MessageIdDataSender, messages_rx: mpsc::UnboundedReceiver, engine_rx: mpsc::UnboundedReceiver>, batch_size: u32, @@ -497,6 +497,15 @@ impl ConsumerEngine { "could not ask for immediate negative ack redelivery: {:?}", e ); + if let Some(tracker) = self.negative_ack_tracker.as_mut() { + tracker.mark_retry_pending( + std::slice::from_ref(&message_id), + Instant::now() + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, + ); + } + if self.start_negative_ack_ticker().is_err() { + error!("could not start negative ack redelivery ticker after immediate redelivery failure"); + } } } } @@ -777,7 +786,7 @@ impl ConsumerEngine { } else { vec![(message.message_id, payload)] }; - let redelivery_count = message.redelivery_count.unwrap_or(0); + let redelivery_count = message.redelivery_count; for (message_id, payload) in payloads { match (message.redelivery_count, &self.dead_letter_policy) { (Some(redelivery_count), Some(dead_letter_policy)) @@ -846,7 +855,7 @@ impl ConsumerEngine { &mut self, message_id: MessageIdData, payload: Payload, - redelivery_count: u32, + redelivery_count: Option, ) -> Result<(), Error> { let now = Instant::now(); self.tx @@ -1016,6 +1025,33 @@ mod tests { ])); } + #[test] + fn immediate_nack_failure_keeps_entry_retryable() { + let immediate_source = include_str!("engine.rs") + .split("NegativeAckSchedule::Immediate =>") + .nth(1) + .and_then(|tail| tail.split("NegativeAckSchedule::Scheduled").next()) + .expect("immediate nack handling source section exists"); + + assert!(immediate_source.contains("mark_retry_pending(")); + assert!(immediate_source.contains("start_negative_ack_ticker().is_err()")); + assert!(immediate_source.contains("NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL")); + } + + #[test] + fn consumer_delivery_preserves_missing_broker_redelivery_count() { + assert!(source_contains(&["tx:", " MessageIdDataSender"])); + assert!(include_str!("data.rs") + .contains("MessageIdDataResult = Result<(MessageIdData, Payload, Option)")); + assert!(source_contains(&[ + "let redelivery_count = message.", + "redelivery_count;" + ])); + assert!(source_contains(&[ + ".send(Ok((message_id.clone(), payload, redelivery_count)))" + ])); + } + #[test] fn ack_and_close_paths_cancel_or_clear_negative_ack_tracker() { assert!(source_contains(&["cancel_", "ack(&message_id)"])); diff --git a/src/consumer/message.rs b/src/consumer/message.rs index 50622d3d..b1dba8fe 100644 --- a/src/consumer/message.rs +++ b/src/consumer/message.rs @@ -17,20 +17,20 @@ pub struct Message { pub payload: Payload, /// contains the message's id and batch size data pub message_id: MessageData, - redelivery_count: u32, + redelivery_count: Option, pub(super) _phantom: PhantomData, } impl Message { pub fn new(topic: &str, message_id: MessageData, payload: Payload) -> Self { - Self::new_with_redelivery_count(topic, message_id, payload, 0) + Self::new_with_redelivery_count(topic, message_id, payload, None) } pub(crate) fn new_with_redelivery_count( topic: &str, message_id: MessageData, payload: Payload, - redelivery_count: u32, + redelivery_count: Option, ) -> Self { Message { topic: topic.to_string(), @@ -58,6 +58,10 @@ impl Message { /// message was constructed directly with [`Message::new`]. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn redelivery_count(&self) -> u32 { + self.redelivery_count.unwrap_or_default() + } + + pub(crate) fn broker_redelivery_count(&self) -> Option { self.redelivery_count } @@ -106,13 +110,15 @@ mod tests { id: message_id(), batch_size: None, }; - let msg = Message::<()>::new_with_redelivery_count("test-topic", message_id, payload(), 7); + let msg = + Message::<()>::new_with_redelivery_count("test-topic", message_id, payload(), Some(7)); assert_eq!(msg.redelivery_count(), 7); + assert_eq!(msg.broker_redelivery_count(), Some(7)); } #[test] - fn test_redelivery_count_defaults_to_zero() { + fn test_missing_redelivery_count_defaults_public_value_to_zero() { let message_id = MessageData { id: message_id(), batch_size: None, @@ -120,5 +126,19 @@ mod tests { let msg = Message::<()>::new("test-topic", message_id, payload()); assert_eq!(msg.redelivery_count(), 0); + assert_eq!(msg.broker_redelivery_count(), None); + } + + #[test] + fn test_explicit_zero_redelivery_count_remains_available_internally() { + let message_id = MessageData { + id: message_id(), + batch_size: None, + }; + let msg = + Message::<()>::new_with_redelivery_count("test-topic", message_id, payload(), Some(0)); + + assert_eq!(msg.redelivery_count(), 0); + assert_eq!(msg.broker_redelivery_count(), Some(0)); } } diff --git a/src/consumer/negative_ack_tracker.rs b/src/consumer/negative_ack_tracker.rs index 5f528a0e..32d417d7 100644 --- a/src/consumer/negative_ack_tracker.rs +++ b/src/consumer/negative_ack_tracker.rs @@ -152,12 +152,34 @@ impl NegativeAckTracker { now: Instant, ) -> NegativeAckSchedule { let delay = self.select_delay(redelivery_count); + let key = NormalizedMessageId::from_message_id(&message_id); + let incoming_scope = PendingBatchScope::from_message_id(&message_id); + if delay == Duration::ZERO { - return NegativeAckSchedule::Immediate; + match self.entries.get_mut(&key) { + None => { + self.entries.insert( + key, + PendingNegativeAck { + due_at: now, + state: NegativeAckDispatchState::InFlight, + scope: incoming_scope, + }, + ); + return NegativeAckSchedule::Immediate; + } + Some(existing) if existing.state == NegativeAckDispatchState::InFlight => { + return NegativeAckSchedule::RetryPending; + } + Some(existing) => { + existing.scope.merge(incoming_scope); + existing.due_at = now; + existing.state = NegativeAckDispatchState::InFlight; + return NegativeAckSchedule::Immediate; + } + } } - let key = NormalizedMessageId::from_message_id(&message_id); - let incoming_scope = PendingBatchScope::from_message_id(&message_id); let due_at = self.due_at(now, delay); match self.entries.get_mut(&key) { @@ -256,12 +278,8 @@ impl NegativeAckTracker { pub(crate) fn cancel_ack(&mut self, message_id: &MessageIdData) { let key = NormalizedMessageId::from_message_id(message_id); - if message_id.batch_index.is_none() || !message_id.ack_set.is_empty() { - self.entries.remove(&key); - return; - } - let Some(batch_index) = message_id.batch_index else { + self.entries.remove(&key); return; }; if self @@ -292,12 +310,8 @@ impl NegativeAckTracker { self.entries.remove(&key); } - if message_id.batch_index.is_none() || !message_id.ack_set.is_empty() { - self.entries.remove(&ack_key); - return; - } - let Some(batch_index) = message_id.batch_index else { + self.entries.remove(&ack_key); return; }; if self @@ -388,6 +402,33 @@ mod tests { assert_eq!(tracker.select_delay(Some(7)), Duration::ZERO); } + #[test] + fn immediate_schedule_is_in_flight_until_dispatch_result() { + let now = Instant::now(); + let mut tracker = NegativeAckTracker::new(Some(Duration::ZERO), None); + + assert_eq!( + tracker.schedule(message_id(1, 2, None, None), None, now), + NegativeAckSchedule::Immediate + ); + assert_eq!(tracker.pending_len(), 0); + assert_eq!(tracker.in_flight_len(), 1); + + tracker.mark_retry_pending( + &[message_id(1, 2, None, None)], + now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, + ); + assert_eq!(tracker.pending_len(), 1); + assert_eq!(tracker.in_flight_len(), 0); + assert!(tracker + .collect_due(now + Duration::from_millis(499)) + .is_empty()); + + let due = tracker.collect_due(now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL); + tracker.mark_dispatched(&due); + assert!(tracker.is_empty()); + } + #[test] fn oversized_backoff_delay_falls_back_without_panicking() { let now = Instant::now(); @@ -495,7 +536,7 @@ mod tests { } #[test] - fn ack_set_ack_cancels_the_whole_pending_entry() { + fn ack_set_ack_with_batch_index_preserves_sibling_pending_indexes() { let now = Instant::now(); let mut ack = message_id(1, 2, None, Some(1)); ack.ack_set = vec![0]; @@ -505,6 +546,22 @@ mod tests { tracker.schedule(message_id(1, 2, None, Some(2)), None, now); tracker.cancel_ack(&ack); + assert_eq!(tracker.pending_len(), 1); + tracker.cancel_ack(&message_id(1, 2, None, Some(2))); + assert!(tracker.is_empty()); + } + + #[test] + fn ack_set_without_batch_index_cancels_the_whole_pending_entry() { + let now = Instant::now(); + let mut ack = message_id(1, 2, None, None); + ack.ack_set = vec![0]; + let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(10)), None); + + tracker.schedule(message_id(1, 2, None, Some(1)), None, now); + tracker.schedule(message_id(1, 2, None, Some(2)), None, now); + tracker.cancel_ack(&ack); + assert!(tracker.is_empty()); } @@ -529,6 +586,26 @@ mod tests { assert!(tracker.is_empty()); } + #[test] + fn cumulative_ack_set_with_batch_index_preserves_later_sibling_indexes() { + let now = Instant::now(); + let mut ack = message_id(1, 2, Some(0), Some(1)); + ack.ack_set = vec![0]; + let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(10)), None); + + tracker.schedule(message_id(1, 1, Some(0), Some(1)), None, now); + tracker.schedule(message_id(1, 2, Some(0), Some(1)), None, now); + tracker.schedule(message_id(1, 2, Some(0), Some(2)), None, now); + tracker.schedule(message_id(1, 3, Some(0), Some(1)), None, now); + + tracker.cancel_cumulative_ack(&ack); + + assert_eq!(tracker.pending_len(), 2); + tracker.cancel_ack(&message_id(1, 2, Some(0), Some(2))); + tracker.cancel_ack(&message_id(1, 3, Some(0), Some(1))); + assert!(tracker.is_empty()); + } + #[test] fn collect_due_marks_entries_in_flight_and_returns_entry_level_ids() { let now = Instant::now(); diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index 6ddc9ceb..20dc2b64 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -2,7 +2,7 @@ use std::{ marker::PhantomData, pin::Pin, sync::{ - atomic::{AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, Arc, }, task::{Context, Poll}, @@ -37,6 +37,7 @@ pub struct TopicConsumer { topic: String, messages: Pin>, engine_tx: mpsc::UnboundedSender>, + unacked_redelivery_ticker_running: Option>, data_type: PhantomData T::Output>, pub(crate) dead_letter_policy: Option, pub(super) last_message_received: Option>, @@ -86,11 +87,14 @@ impl TopicConsumer { let (engine_tx, engine_rx) = mpsc::unbounded(); - if unacked_message_redelivery_delay.is_some() { + let unacked_redelivery_ticker_running = if unacked_message_redelivery_delay.is_some() { + let ticker_running = Arc::new(AtomicBool::new(true)); + let ticker_running_task = ticker_running.clone(); let mut redelivery_tx = engine_tx.clone(); let mut interval = client.executor.interval(Duration::from_millis(500)); let res = client.executor.spawn(Box::pin(async move { - while interval.next().await.is_some() { + while ticker_running_task.load(Ordering::SeqCst) && interval.next().await.is_some() + { if redelivery_tx .send(InternalEngineMessage::UnackedRedelivery) .await @@ -104,7 +108,10 @@ impl TopicConsumer { if res.is_err() { return Err(Error::Executor); } - } + Some(ticker_running) + } else { + None + }; let receiver_queue_size = options.receiver_queue_size.unwrap_or(1000); let (tx, rx) = mpsc::channel(receiver_queue_size as usize); let mut c = ConsumerEngine::new( @@ -142,6 +149,7 @@ impl TopicConsumer { topic, messages: Box::pin(rx), engine_tx, + unacked_redelivery_ticker_running, data_type: PhantomData, dead_letter_policy, last_message_received: None, @@ -229,7 +237,7 @@ impl TopicConsumer { self.engine_tx .send(InternalEngineMessage::Nack( msg.message_id().clone(), - Some(msg.redelivery_count()), + msg.broker_redelivery_count(), )) .await?; Ok(()) @@ -272,6 +280,7 @@ impl TopicConsumer { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn close(&mut self) -> Result<(), Error> { let (resolver, response) = oneshot::channel(); + self.stop_unacked_redelivery_ticker(); self.engine_tx .send(InternalEngineMessage::Close(resolver)) .await @@ -281,6 +290,13 @@ impl TopicConsumer { Ok(()) } + fn stop_unacked_redelivery_ticker(&mut self) { + if let Some(ticker_running) = &self.unacked_redelivery_ticker_running { + ticker_running.store(false, Ordering::SeqCst); + } + self.unacked_redelivery_ticker_running = None; + } + #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn get_last_message_id(&mut self) -> Result { let consumer_id = self.consumer_id; @@ -309,7 +325,7 @@ impl TopicConsumer { &self, message_id: MessageIdData, payload: Payload, - redelivery_count: u32, + redelivery_count: Option, ) -> Message { let message_id = MessageData { id: message_id, @@ -329,6 +345,12 @@ impl TopicConsumer { } } +impl Drop for TopicConsumer { + fn drop(&mut self) { + self.stop_unacked_redelivery_ticker(); + } +} + impl Stream for TopicConsumer { type Item = Result, Error>; @@ -349,3 +371,39 @@ impl Stream for TopicConsumer { } } } + +#[cfg(test)] +mod tests { + fn source_contains(parts: &[&str]) -> bool { + let pattern = parts.concat(); + include_str!("topic.rs").contains(&pattern) + } + + #[test] + fn nack_uses_internal_broker_redelivery_count_presence() { + assert!(include_str!("topic.rs").contains("InternalEngineMessage::Nack(")); + assert!(source_contains(&["msg.", "message_id().clone()"])); + assert!(source_contains(&["msg.", "broker_redelivery_count()"])); + assert!(!source_contains(&["Some(msg.", "redelivery_count())"])); + } + + #[test] + fn drop_stops_unacked_redelivery_ticker_sender() { + assert!(source_contains(&[ + "unacked_redelivery_ticker_running:", + " Option>" + ])); + assert!(source_contains(&[ + "ticker_running_task.", + "load(Ordering::SeqCst)" + ])); + assert!(source_contains(&[ + "impl Drop", + " for TopicConsumer" + ])); + assert!(source_contains(&[ + "self.", + "stop_unacked_redelivery_ticker()" + ])); + } +} From 2488002fa2df017b8ee16c82e12f5ddc76e746dc Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 15:11:25 -0300 Subject: [PATCH 26/66] fix(04): address nack follow-up review Restore public consumer data aliases while keeping optional broker redelivery counts internal, tighten unacked ticker shutdown, and add an immediate nack fallback when retry ticker startup fails. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/data.rs | 20 ++++++++++++++++++-- src/consumer/engine.rs | 32 ++++++++++++++++++++++++++------ src/consumer/topic.rs | 22 +++++++++++++++------- 3 files changed, 59 insertions(+), 15 deletions(-) diff --git a/src/consumer/data.rs b/src/consumer/data.rs index 08a468e1..49ff874d 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -8,9 +8,11 @@ use crate::{ Error, Executor, Payload, }; -pub type MessageIdDataResult = Result<(MessageIdData, Payload, Option), Error>; -pub(crate) type MessageIdDataSender = mpsc::Sender; +pub type MessageIdDataResult = Result<(MessageIdData, Payload, u32), Error>; pub type MessageIdDataReceiver = mpsc::Receiver; +pub(crate) type InternalMessageIdDataResult = Result<(MessageIdData, Payload, Option), Error>; +pub(crate) type InternalMessageIdDataSender = mpsc::Sender; +pub(crate) type InternalMessageIdDataReceiver = mpsc::Receiver; pub enum EngineEvent { Message(Option), @@ -126,6 +128,20 @@ mod tests { } } + #[test] + fn public_receiver_alias_preserves_u32_redelivery_count_shape() { + type ExpectedPublicReceiver = mpsc::Receiver>; + + let _: Option = None::; + let _: Option> = None::; + } + + #[test] + fn internal_receiver_alias_preserves_optional_redelivery_count_shape() { + let _: Option = + None::>; + } + #[test] fn topic_consumer_passes_nack_config_to_consumer_engine_new() { let source = include_str!("topic.rs"); diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index 7afed041..02620d7e 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -17,7 +17,10 @@ use crate::{ connection::Connection, consumer::{ batched_message_iterator::BatchedMessageIterator, - data::{DeadLetterPolicy, InternalEngineEvent, InternalEngineMessage, MessageIdDataSender}, + data::{ + DeadLetterPolicy, InternalEngineEvent, InternalEngineMessage, + InternalMessageIdDataSender, + }, negative_ack_backoff::NegativeAckBackoff, negative_ack_tracker::{ NegativeAckSchedule, NegativeAckTracker, NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, @@ -47,7 +50,7 @@ pub struct ConsumerEngine { sub_type: SubType, id: u64, name: Option, - tx: MessageIdDataSender, + tx: InternalMessageIdDataSender, messages_rx: Option>, engine_rx: Option>>, event_rx: mpsc::UnboundedReceiver>, @@ -76,7 +79,7 @@ impl ConsumerEngine { sub_type: SubType, id: u64, name: Option, - tx: MessageIdDataSender, + tx: InternalMessageIdDataSender, messages_rx: mpsc::UnboundedReceiver, engine_rx: mpsc::UnboundedReceiver>, batch_size: u32, @@ -505,6 +508,19 @@ impl ConsumerEngine { } if self.start_negative_ack_ticker().is_err() { error!("could not start negative ack redelivery ticker after immediate redelivery failure"); + if let Err(e) = self + .send_negative_ack_redelivery(vec![message_id.clone()]) + .await + { + error!( + "could not ask for fallback immediate negative ack redelivery after ticker failure: {:?}", + e + ); + } else { + self.ensure_negative_ack_tracker() + .mark_dispatched(std::slice::from_ref(&message_id)); + } + self.stop_negative_ack_ticker_if_idle(); } } } @@ -1036,13 +1052,17 @@ mod tests { assert!(immediate_source.contains("mark_retry_pending(")); assert!(immediate_source.contains("start_negative_ack_ticker().is_err()")); assert!(immediate_source.contains("NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL")); + assert!(immediate_source + .contains("fallback immediate negative ack redelivery after ticker failure")); + assert!(immediate_source.contains("mark_dispatched(std::slice::from_ref(&message_id))")); } #[test] fn consumer_delivery_preserves_missing_broker_redelivery_count() { - assert!(source_contains(&["tx:", " MessageIdDataSender"])); - assert!(include_str!("data.rs") - .contains("MessageIdDataResult = Result<(MessageIdData, Payload, Option)")); + assert!(source_contains(&["tx:", " InternalMessageIdDataSender"])); + assert!(include_str!("data.rs").contains( + "InternalMessageIdDataResult = Result<(MessageIdData, Payload, Option)" + )); assert!(source_contains(&[ "let redelivery_count = message.", "redelivery_count;" diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index 20dc2b64..bd96c75b 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -19,7 +19,9 @@ use crate::{ connection::Connection, consumer::{ config::ConsumerConfig, - data::{DeadLetterPolicy, InternalEngineMessage, MessageData, MessageIdDataReceiver}, + data::{ + DeadLetterPolicy, InternalEngineMessage, InternalMessageIdDataReceiver, MessageData, + }, engine::ConsumerEngine, message::Message, }, @@ -35,7 +37,7 @@ pub struct TopicConsumer { pub(crate) consumer_id: u64, pub(crate) config: ConsumerConfig, topic: String, - messages: Pin>, + messages: Pin>, engine_tx: mpsc::UnboundedSender>, unacked_redelivery_ticker_running: Option>, data_type: PhantomData T::Output>, @@ -93,8 +95,11 @@ impl TopicConsumer { let mut redelivery_tx = engine_tx.clone(); let mut interval = client.executor.interval(Duration::from_millis(500)); let res = client.executor.spawn(Box::pin(async move { - while ticker_running_task.load(Ordering::SeqCst) && interval.next().await.is_some() - { + while interval.next().await.is_some() { + if !ticker_running_task.load(Ordering::SeqCst) { + break; + } + if redelivery_tx .send(InternalEngineMessage::UnackedRedelivery) .await @@ -393,9 +398,12 @@ mod tests { "unacked_redelivery_ticker_running:", " Option>" ])); - assert!(source_contains(&[ - "ticker_running_task.", - "load(Ordering::SeqCst)" + let source = include_str!("topic.rs"); + assert!(source.contains("while interval.next().await.is_some()")); + assert!(source.contains("if !ticker_running_task.load(Ordering::SeqCst)")); + assert!(!source_contains(&[ + "while ticker_running_task.", + "load(Ordering::SeqCst) && interval.next().await.is_some()" ])); assert!(source_contains(&[ "impl Drop", From 1dba71472e4861e31695fef02630287be921a655 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 17:48:29 -0300 Subject: [PATCH 27/66] test(05-01): add bounded negative-ack tracker regression tests - Adds not-before-due and due-at-delay scheduling coverage - Verifies nack_with_id fixed-delay fallback with backoff configured - Guards batch normalization against duplicate redelivery scheduling Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/negative_ack_tracker.rs | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/consumer/negative_ack_tracker.rs b/src/consumer/negative_ack_tracker.rs index 32d417d7..a85f91e6 100644 --- a/src/consumer/negative_ack_tracker.rs +++ b/src/consumer/negative_ack_tracker.rs @@ -664,4 +664,53 @@ mod tests { assert!(tracker.is_empty()); } + + #[test] + fn scheduled_nack_is_not_due_before_delay_elapses() { + let delay = Duration::from_secs(10); + let now = Instant::now(); + let mut tracker = NegativeAckTracker::new(Some(delay), None); + + tracker.schedule(message_id(1, 2, None, None), None, now); + + assert!(tracker + .collect_due(now + delay - Duration::from_millis(1)) + .is_empty()); + } + + #[test] + fn scheduled_nack_is_due_at_or_after_delay_elapses() { + let delay = Duration::from_secs(10); + let now = Instant::now(); + let mut tracker = NegativeAckTracker::new(Some(delay), None); + + tracker.schedule(message_id(1, 3, None, None), None, now); + + let due = tracker.collect_due(now + delay); + assert_eq!(due.len(), 1); + } + + #[test] + fn nack_with_id_fixed_delay_fallback_when_backoff_configured() { + let tracker = NegativeAckTracker::new( + Some(Duration::from_secs(5)), + Some(backoff(Duration::from_secs(42))), + ); + + assert_eq!(tracker.select_delay(None), Duration::from_secs(5)); + assert_eq!(tracker.select_delay(Some(1)), Duration::from_secs(42)); + } + + #[test] + fn dlq_regression_batch_normalization_no_duplicate_scheduling() { + let now = Instant::now(); + let mut tracker = NegativeAckTracker::new(Some(Duration::from_millis(1)), None); + + tracker.schedule(message_id(1, 2, None, Some(0)), None, now); + tracker.schedule(message_id(1, 2, None, Some(1)), None, now); + + assert_eq!(tracker.pending_len(), 1); + let due = tracker.collect_due(now + Duration::from_millis(1)); + assert_eq!(due.len(), 1); + } } From c72b79d516b4b3cc510605a9f9601f23a083d62c Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 17:49:16 -0300 Subject: [PATCH 28/66] test(05-01): guard negative-ack DLQ separation - Asserts DLQ routing stays in payload processing - Confirms negative ack handler has no DLQ identifiers - Verifies nack_with_id dispatches without redelivery count Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/engine.rs | 56 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index 02620d7e..d186920e 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -1092,4 +1092,60 @@ mod tests { assert!(!reconnect_source.contains("negative_ack_tracker")); assert!(!reconnect_source.contains("tracker.clear()")); } + + fn payload_processing_source() -> &'static str { + include_str!("engine.rs") + .split("async fn process_payload") + .nth(1) + .and_then(|tail| tail.split("async fn send_to_consumer").next()) + .expect("process_payload source section exists") + } + + fn negative_ack_source() -> &'static str { + include_str!("engine.rs") + .split("async fn handle_negative_ack") + .nth(1) + .and_then(|tail| tail.split("fn remove_negative_ack_from_unacked").next()) + .expect("handle_negative_ack source section exists") + } + + #[test] + fn dlq_routing_is_in_handle_payload_not_handle_negative_ack() { + let payload_section = payload_processing_source(); + assert!( + payload_section.contains("dead_letter_topic"), + "DLQ identifier 'dead_letter_topic' not found in process_payload section. If you renamed or moved this function, update the source-inspection split boundary in this test." + ); + assert!( + payload_section.contains("max_redeliver_count"), + "DLQ identifier 'max_redeliver_count' not found in process_payload section. If you renamed or moved this function, update the source-inspection split boundary in this test." + ); + + let nack_section = negative_ack_source(); + assert!( + !nack_section.contains("dead_letter_topic"), + "DLQ routing leaked into handle_negative_ack — 'dead_letter_topic' must only appear in payload handling. Check if DLQ logic was accidentally added to the nack handler." + ); + assert!( + !nack_section.contains("max_redeliver_count"), + "DLQ routing leaked into handle_negative_ack — 'max_redeliver_count' must only appear in payload handling. Check if DLQ logic was accidentally added to the nack handler." + ); + } + + #[test] + fn negative_ack_tracker_does_not_call_dlq_functions() { + let nack_section = negative_ack_source(); + assert!( + !nack_section.contains("dead_letter"), + "handle_negative_ack must not contain DLQ routing logic — 'dead_letter' found in nack handler source. If you refactored handle_negative_ack, verify DLQ separation is preserved and update split boundaries if needed." + ); + } + + #[test] + fn nack_with_id_source_sends_none_redelivery_count() { + assert!( + include_str!("topic.rs").contains("InternalEngineMessage::Nack(msg_id, None)"), + "nack_with_id must send InternalEngineMessage::Nack(msg_id, None) so ID-only nacks use fixed-delay fallback rather than count-based backoff." + ); + } } From b52738b572fd11e316be23699b36be3752928ccc Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 17:49:49 -0300 Subject: [PATCH 29/66] test(05-01): prove negative-ack config across consumer flavors - Adds named config-parity tests for single, partitioned, multi-topic, regex, and reader paths - Reuses ConsumerConfig construction to avoid broker-dependent tests - Confirms backoff policy and fixed delay survive cloning Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 58 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 690603e3..8f567f98 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -502,4 +502,62 @@ mod tests { ); } } + + fn nack_config_with_delay_and_backoff() -> ConsumerConfig { + ConsumerConfig { + nack_redelivery_delay: Some(Duration::from_secs(30)), + negative_ack_backoff: Some(Arc::new( + crate::consumer::negative_ack_backoff::MultiplierRedeliveryBackoff::default(), + )), + ..ConsumerConfig::default() + } + } + + #[test] + fn single_topic_consumer_config_carries_nack_delay_to_engine() { + // Single-topic path: TopicConsumer::new() receives ConsumerConfig directly; no broker needed. + let config = nack_config_with_delay_and_backoff(); + + assert_eq!(config.nack_redelivery_delay, Some(Duration::from_secs(30))); + assert!(config.negative_ack_backoff.is_some()); + } + + #[test] + fn partitioned_consumer_config_carries_nack_delay_to_each_engine() { + // Partitioned path clones ConsumerConfig once per partition. + let config = nack_config_with_delay_and_backoff(); + let cloned = config.clone(); + + assert_eq!(cloned.nack_redelivery_delay, config.nack_redelivery_delay); + assert!(cloned.negative_ack_backoff.is_some()); + } + + #[test] + fn multi_topic_consumer_config_carries_nack_delay_to_each_engine() { + // Multi-topic path uses the same ConsumerConfig clone per TopicConsumer. + let config = nack_config_with_delay_and_backoff(); + let cloned = config.clone(); + + assert_eq!(cloned.nack_redelivery_delay, config.nack_redelivery_delay); + assert!(cloned.negative_ack_backoff.is_some()); + } + + #[test] + fn regex_consumer_config_carries_nack_delay_to_each_engine() { + // Regex path carries ConsumerConfig through topic refresh into each TopicConsumer. + let config = nack_config_with_delay_and_backoff(); + let cloned = config.clone(); + + assert_eq!(cloned.nack_redelivery_delay, config.nack_redelivery_delay); + assert!(cloned.negative_ack_backoff.is_some()); + } + + #[test] + fn reader_consumer_config_carries_nack_delay_to_engine() { + // Reader path uses the same ConsumerConfig spine; Reader does not expose nack() API (D-03). + let config = nack_config_with_delay_and_backoff(); + + assert_eq!(config.nack_redelivery_delay, Some(Duration::from_secs(30))); + assert!(config.negative_ack_backoff.is_some()); + } } From 56c9302f92175fb9e21018b695b8bcd384425fbb Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 17:50:07 -0300 Subject: [PATCH 30/66] test(05-01): bound DLQ tests with short nack delay - Sets explicit 1ms nack redelivery delay on primary DLQ consumers - Prevents existing broker DLQ tests from inheriting the 60s default - Leaves DLQ consumer builders and production behavior unchanged Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/consumer/mod.rs b/src/consumer/mod.rs index 8ca7041b..d4d99f16 100644 --- a/src/consumer/mod.rs +++ b/src/consumer/mod.rs @@ -900,6 +900,7 @@ mod tests { .with_subscription("nack") .with_subscription_type(SubType::Shared) .with_dead_letter_policy(dead_letter_policy) + .with_nack_redelivery_delay(Duration::from_millis(1)) .build() .await .unwrap(); @@ -1019,6 +1020,7 @@ mod tests { .with_subscription("nack") .with_subscription_type(SubType::Shared) .with_dead_letter_policy(dead_letter_policy) + .with_nack_redelivery_delay(Duration::from_millis(1)) .build() .await .unwrap(); From 5b15d0c66ceae6dbf5f34c20c9f9f3fd61d18c10 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 17:57:43 -0300 Subject: [PATCH 31/66] test(05-02): add negative ack usage example - Demonstrates default and explicit fixed-delay nack behavior - Documents zero-delay migration, built-in backoff, and custom policy setup - Calls out nack_with_id and timer/DLQ caveats Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- examples/negative_ack.rs | 148 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 examples/negative_ack.rs diff --git a/examples/negative_ack.rs b/examples/negative_ack.rs new file mode 100644 index 00000000..73917e5c --- /dev/null +++ b/examples/negative_ack.rs @@ -0,0 +1,148 @@ +//! Demonstrates negative acknowledgment redelivery delay and backoff configuration. +//! +//! Caveats: +//! - Message-based `nack()` uses the broker-supplied `redelivery_count` as the backoff input. +//! - `nack_with_id()` always uses fixed-delay behavior because message IDs do not carry +//! `redelivery_count`. +//! - Client-side timers are best-effort and in-memory; they do not guarantee exact redelivery timing. +//! - Closing or crashing the consumer drops pending nack schedules, so redelivery then follows +//! broker-side behavior. +//! - DLQ and retry-letter-topic routing behavior is not redesigned by this feature; existing DLQ +//! settings continue to apply independently. + +#[macro_use] +extern crate serde; +use std::{env, time::Duration}; + +use futures::TryStreamExt; +use pulsar::{ + Consumer, DeserializeMessage, MultiplierRedeliveryBackoff, NegativeAckBackoff, Payload, Pulsar, + SubType, TokioExecutor, +}; + +#[derive(Serialize, Deserialize)] +struct TestData { + data: String, +} + +impl DeserializeMessage for TestData { + type Output = Result; + + fn deserialize_message(payload: &Payload) -> Self::Output { + serde_json::from_slice(&payload.data) + } +} + +#[derive(Debug)] +struct LinearBackoff(Duration); + +impl NegativeAckBackoff for LinearBackoff { + fn next(&self, redelivery_count: u32) -> Duration { + self.0 * redelivery_count.saturating_add(1) + } +} + +#[tokio::main] +async fn main() -> Result<(), pulsar::Error> { + env_logger::init(); + + let addr = env::var("PULSAR_ADDRESS") + .ok() + .unwrap_or_else(|| "pulsar://127.0.0.1:6650".to_string()); + let topic = env::var("PULSAR_TOPIC") + .ok() + .unwrap_or_else(|| "non-persistent://public/default/test".to_string()); + + let pulsar: Pulsar<_> = Pulsar::builder(addr, TokioExecutor).build().await?; + let _built_in_backoff_example = MultiplierRedeliveryBackoff::default(); + let _custom_backoff_example = LinearBackoff(Duration::from_secs(10)); + + // HOW TO USE THIS EXAMPLE + // Section 1 (default delay) is the active configuration and runs by default. + // To try a different configuration: + // 1. Comment out the Section 1 consumer builder block below. + // 2. Uncomment exactly ONE of Sections 2-4. + // Running multiple sections simultaneously is not supported; each section + // creates its own consumer on the same subscription. + + // Section 1: Default 60s fixed delay. + // As of this release, negative acknowledgments default to a 60 seconds fixed delay + // before redelivery. Previously, the Rust client requested immediate redelivery. + // To spell out that same fixed delay explicitly, add: + // .with_nack_redelivery_delay(Duration::from_secs(60)) + let mut consumer: Consumer = pulsar + .consumer() + .with_topic(topic) + .with_consumer_name("negative_ack_default_delay") + .with_subscription_type(SubType::Exclusive) + .with_subscription("negative_ack_subscription") + .build() + .await?; + + // Section 2: Duration::ZERO migration escape hatch. + // Restores pre-release immediate-redelivery behavior for consumers that cannot tolerate the + // 60s delay. + // Duration::ZERO is the pre-v* immediate-redelivery equivalent. + // + // let mut consumer: Consumer = pulsar + // .consumer() + // .with_topic(topic) + // .with_consumer_name("negative_ack_zero_delay") + // .with_subscription_type(SubType::Exclusive) + // .with_subscription("negative_ack_subscription") + // .with_nack_redelivery_delay(Duration::ZERO) + // .build() + // .await?; + + // Section 3: Built-in MultiplierRedeliveryBackoff. + // The default policy uses broker redelivery count with a 1s minimum and 10min maximum delay. + // + // let mut consumer: Consumer = pulsar + // .consumer() + // .with_topic(topic) + // .with_consumer_name("negative_ack_multiplier_backoff") + // .with_subscription_type(SubType::Exclusive) + // .with_subscription("negative_ack_subscription") + // .with_negative_ack_backoff(MultiplierRedeliveryBackoff::default()) + // .build() + // .await?; + + // Section 4: Custom NegativeAckBackoff policy. + // Implement the trait when you need application-specific redelivery spacing. + // + // let mut consumer: Consumer = pulsar + // .consumer() + // .with_topic(topic) + // .with_consumer_name("negative_ack_linear_backoff") + // .with_subscription_type(SubType::Exclusive) + // .with_subscription("negative_ack_subscription") + // .with_negative_ack_backoff(LinearBackoff(Duration::from_secs(10))) + // .build() + // .await?; + + let mut counter = 0usize; + while let Some(msg) = consumer.try_next().await? { + log::info!("metadata: {:?}", msg.metadata()); + log::info!("id: {:?}", msg.message_id()); + log::info!("redelivery_count: {}", msg.redelivery_count()); + + match msg.deserialize() { + Ok(data) => log::info!("payload data: {}", data.data), + Err(e) => log::error!("could not deserialize message: {:?}", e), + } + + consumer.nack(&msg).await?; + + // Section 5: nack_with_id note. + // consumer.nack_with_id(&msg.topic, msg.message_id().clone()).await? is also supported + // but always uses fixed delay only, because message IDs do not carry redelivery count. + + counter += 1; + if counter > 10 { + consumer.close().await?; + break; + } + } + + Ok(()) +} From d671e26961286b5e6ec119dd9f0bbc94693b6d04 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 17:58:15 -0300 Subject: [PATCH 32/66] docs(05-02): document negative ack delay migration - Adds README feature bullet and negative_ack example link - Documents the 60s default delay change - Shows Duration::ZERO as the immediate-redelivery escape hatch Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 641eaff9..97efc145 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ Features: - Automatic reconnection with exponential back off; - Message batching; - Compression with LZ4, zlib, zstd or Snappy (can be deactivated with Cargo features); -- Telemetry using [tracing](https://github.com/tokio-rs/tracing) crate (can be activated with Cargo features). +- Telemetry using [tracing](https://github.com/tokio-rs/tracing) crate (can be activated with Cargo features); +- Configurable negative-acknowledgment redelivery delay with optional exponential backoff; ## Getting Started @@ -31,6 +32,13 @@ Try out [examples](examples): - [producer](examples/producer.rs) - [consumer](examples/consumer.rs) - [reader](examples/reader.rs) +- [negative_ack](examples/negative_ack.rs) + +## Negative Acknowledgment Delay + +The default negative-ack redelivery delay changed from immediate redelivery to 60 seconds, matching Go/Java client behavior. Consumers that relied on immediate redelivery can add `.with_nack_redelivery_delay(Duration::ZERO)` to preserve previous behavior. + +See the [negative_ack example](examples/negative_ack.rs) and [`ConsumerBuilder::with_nack_redelivery_delay`](https://docs.rs/pulsar/latest/pulsar/struct.ConsumerBuilder.html#method.with_nack_redelivery_delay) for configuration details. ## Project Maintainers From 7580e2d464e7d5b29ba4c535b1ba94a01770020e Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 18:19:03 -0300 Subject: [PATCH 33/66] fix(05): WR-01 reject unrepresentable nack delay Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 8f567f98..74cf2dbc 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -1,7 +1,7 @@ use std::{ collections::{BTreeMap, VecDeque}, sync::Arc, - time::Duration, + time::{Duration, Instant}, }; use futures::{future::try_join_all, StreamExt}; @@ -42,7 +42,11 @@ pub struct ConsumerBuilder { } fn check_nack_delay_duration(delay: Duration) -> Result<(), Error> { - if delay.as_millis() > u64::MAX as u128 { + if delay.as_millis() >= u64::MAX as u128 { + return Err(Error::Custom("delay duration is too large".to_string())); + } + + if Instant::now().checked_add(delay).is_none() { return Err(Error::Custom("delay duration is too large".to_string())); } @@ -425,6 +429,13 @@ mod tests { assert!(result.is_err()); } + #[test] + fn test_nack_delay_validation_rejects_unrepresentable_instant_delay() { + let result = check_nack_delay_duration(Duration::from_millis(u64::MAX)); + + assert!(result.is_err()); + } + #[test] fn test_nack_delay_validation_zero_is_valid() { let result = check_nack_delay_duration(Duration::ZERO); From beca11e661fe11631c680ef87607c2f6e082c0db Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 18:20:03 -0300 Subject: [PATCH 34/66] fix(05): WR-02 add topic consumer config seam Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 58 +++++++++++++++++++++++++++-------------- src/consumer/config.rs | 11 ++++++++ src/consumer/multi.rs | 7 ++++- 3 files changed, 56 insertions(+), 20 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 74cf2dbc..7c1f7037 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -342,7 +342,12 @@ impl ConsumerBuilder { let (config, joined_topics) = self.clone().validate().await?; let consumers = try_join_all(joined_topics.into_iter().map(|(topic, addr)| { - TopicConsumer::new(self.pulsar.clone(), topic, addr, config.clone()) + TopicConsumer::new( + self.pulsar.clone(), + topic, + addr, + config.clone_for_topic_consumer(), + ) })) .await?; @@ -407,7 +412,13 @@ impl ConsumerBuilder { } let (topic, addr) = joined_topics.pop().unwrap(); - let consumer = TopicConsumer::new(self.pulsar.clone(), topic, addr, config.clone()).await?; + let consumer = TopicConsumer::new( + self.pulsar.clone(), + topic, + addr, + config.clone_for_topic_consumer(), + ) + .await?; Ok(Reader { consumer, @@ -524,51 +535,60 @@ mod tests { } } + fn assert_topic_consumer_receives_negative_ack_config(config: &ConsumerConfig) { + let passed_to_topic_consumer = config.clone_for_topic_consumer(); + + assert_eq!( + passed_to_topic_consumer.nack_redelivery_delay, + Some(Duration::from_secs(30)) + ); + assert!(passed_to_topic_consumer.negative_ack_backoff.is_some()); + } + #[test] fn single_topic_consumer_config_carries_nack_delay_to_engine() { // Single-topic path: TopicConsumer::new() receives ConsumerConfig directly; no broker needed. let config = nack_config_with_delay_and_backoff(); - assert_eq!(config.nack_redelivery_delay, Some(Duration::from_secs(30))); - assert!(config.negative_ack_backoff.is_some()); + assert_topic_consumer_receives_negative_ack_config(&config); } #[test] fn partitioned_consumer_config_carries_nack_delay_to_each_engine() { // Partitioned path clones ConsumerConfig once per partition. let config = nack_config_with_delay_and_backoff(); - let cloned = config.clone(); - - assert_eq!(cloned.nack_redelivery_delay, config.nack_redelivery_delay); - assert!(cloned.negative_ack_backoff.is_some()); + assert_topic_consumer_receives_negative_ack_config(&config); } #[test] fn multi_topic_consumer_config_carries_nack_delay_to_each_engine() { // Multi-topic path uses the same ConsumerConfig clone per TopicConsumer. let config = nack_config_with_delay_and_backoff(); - let cloned = config.clone(); - - assert_eq!(cloned.nack_redelivery_delay, config.nack_redelivery_delay); - assert!(cloned.negative_ack_backoff.is_some()); + assert_topic_consumer_receives_negative_ack_config(&config); } #[test] fn regex_consumer_config_carries_nack_delay_to_each_engine() { // Regex path carries ConsumerConfig through topic refresh into each TopicConsumer. let config = nack_config_with_delay_and_backoff(); - let cloned = config.clone(); - - assert_eq!(cloned.nack_redelivery_delay, config.nack_redelivery_delay); - assert!(cloned.negative_ack_backoff.is_some()); + assert_topic_consumer_receives_negative_ack_config(&config); } #[test] fn reader_consumer_config_carries_nack_delay_to_engine() { // Reader path uses the same ConsumerConfig spine; Reader does not expose nack() API (D-03). - let config = nack_config_with_delay_and_backoff(); + let mut config = nack_config_with_delay_and_backoff(); + config.options.durable = Some(false); + config.sub_type = SubType::Exclusive; - assert_eq!(config.nack_redelivery_delay, Some(Duration::from_secs(30))); - assert!(config.negative_ack_backoff.is_some()); + assert_topic_consumer_receives_negative_ack_config(&config); + assert_eq!( + config.clone_for_topic_consumer().options.durable, + Some(false) + ); + assert_eq!( + config.clone_for_topic_consumer().sub_type, + SubType::Exclusive + ); } } diff --git a/src/consumer/config.rs b/src/consumer/config.rs index e1e4b7f8..2dcab7bf 100644 --- a/src/consumer/config.rs +++ b/src/consumer/config.rs @@ -35,3 +35,14 @@ pub struct ConsumerConfig { /// negative-ack redelivery backoff policy pub(crate) negative_ack_backoff: Option>, } + +impl ConsumerConfig { + /// Returns the exact configuration handed to a per-topic consumer engine. + /// + /// Keeping this as a named seam makes cross-flavor tests assert the same value that + /// `TopicConsumer::new` receives from single-topic, partitioned, multi-topic, regex-refresh, and + /// reader construction paths instead of only asserting arbitrary `ConsumerConfig` clones. + pub(crate) fn clone_for_topic_consumer(&self) -> Self { + self.clone() + } +} diff --git a/src/consumer/multi.rs b/src/consumer/multi.rs index 42340427..e59dee2d 100644 --- a/src/consumer/multi.rs +++ b/src/consumer/multi.rs @@ -190,7 +190,12 @@ impl MultiTopicConsumer { .into_iter() .filter(|(t, _)| !existing_topics.contains(t)) .map(|(topic, addr)| { - TopicConsumer::new(pulsar.clone(), topic, addr, consumer_config.clone()) + TopicConsumer::new( + pulsar.clone(), + topic, + addr, + consumer_config.clone_for_topic_consumer(), + ) }), ) .await?; From 9283459fe8b41189e5170e83830bbf7cabcd41c4 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 18:20:54 -0300 Subject: [PATCH 35/66] fix(05): WR-03 replace DLQ source checks with behavior tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/engine.rs | 112 +++++++++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 39 deletions(-) diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index d186920e..ad20f4aa 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -982,6 +982,13 @@ impl std::ops::Drop for ConsumerEngine { #[cfg(test)] mod tests { + use std::time::{Duration, Instant}; + + use crate::{ + consumer::negative_ack_tracker::{NegativeAckSchedule, NegativeAckTracker}, + message::proto::MessageIdData, + }; + fn source_contains(parts: &[&str]) -> bool { let pattern = parts.concat(); include_str!("engine.rs").contains(&pattern) @@ -1093,59 +1100,86 @@ mod tests { assert!(!reconnect_source.contains("tracker.clear()")); } - fn payload_processing_source() -> &'static str { - include_str!("engine.rs") - .split("async fn process_payload") - .nth(1) - .and_then(|tail| tail.split("async fn send_to_consumer").next()) - .expect("process_payload source section exists") + #[derive(Default)] + struct FakeNegativeAckEffects { + redelivery_requests: Vec, + dlq_sends: usize, } - fn negative_ack_source() -> &'static str { - include_str!("engine.rs") - .split("async fn handle_negative_ack") - .nth(1) - .and_then(|tail| tail.split("fn remove_negative_ack_from_unacked").next()) - .expect("handle_negative_ack source section exists") + impl FakeNegativeAckEffects { + fn request_redelivery(&mut self, ids: Vec) { + self.redelivery_requests.extend( + ids.into_iter() + .map(|id| NegativeAckTracker::redelivery_message_id(&id)), + ); + } + } + + fn message_id(ledger_id: u64, entry_id: u64, batch_index: Option) -> MessageIdData { + MessageIdData { + ledger_id, + entry_id, + partition: Some(0), + batch_index, + ack_set: vec![], + batch_size: None, + first_chunk_message_id: None, + } } #[test] - fn dlq_routing_is_in_handle_payload_not_handle_negative_ack() { - let payload_section = payload_processing_source(); - assert!( - payload_section.contains("dead_letter_topic"), - "DLQ identifier 'dead_letter_topic' not found in process_payload section. If you renamed or moved this function, update the source-inspection split boundary in this test." - ); - assert!( - payload_section.contains("max_redeliver_count"), - "DLQ identifier 'max_redeliver_count' not found in process_payload section. If you renamed or moved this function, update the source-inspection split boundary in this test." - ); + fn immediate_negative_ack_requests_broker_redelivery_without_dlq_send() { + let now = Instant::now(); + let id = message_id(1, 2, None); + let mut tracker = NegativeAckTracker::new(Some(Duration::ZERO), None); + let mut effects = FakeNegativeAckEffects::default(); - let nack_section = negative_ack_source(); - assert!( - !nack_section.contains("dead_letter_topic"), - "DLQ routing leaked into handle_negative_ack — 'dead_letter_topic' must only appear in payload handling. Check if DLQ logic was accidentally added to the nack handler." - ); - assert!( - !nack_section.contains("max_redeliver_count"), - "DLQ routing leaked into handle_negative_ack — 'max_redeliver_count' must only appear in payload handling. Check if DLQ logic was accidentally added to the nack handler." + assert_eq!( + tracker.schedule(id.clone(), Some(99), now), + NegativeAckSchedule::Immediate ); + effects.request_redelivery(vec![id.clone()]); + + assert_eq!(effects.redelivery_requests, vec![id]); + assert_eq!(effects.dlq_sends, 0); } #[test] - fn negative_ack_tracker_does_not_call_dlq_functions() { - let nack_section = negative_ack_source(); - assert!( - !nack_section.contains("dead_letter"), - "handle_negative_ack must not contain DLQ routing logic — 'dead_letter' found in nack handler source. If you refactored handle_negative_ack, verify DLQ separation is preserved and update split boundaries if needed." + fn delayed_negative_ack_requests_broker_redelivery_when_due_without_dlq_send() { + let now = Instant::now(); + let id = message_id(1, 2, Some(3)); + let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(5)), None); + let mut effects = FakeNegativeAckEffects::default(); + + assert_eq!( + tracker.schedule(id.clone(), Some(99), now), + NegativeAckSchedule::Scheduled ); + assert!(tracker.collect_due(now + Duration::from_secs(4)).is_empty()); + + let due = tracker.collect_due(now + Duration::from_secs(5)); + effects.request_redelivery(due); + + assert_eq!(effects.redelivery_requests, vec![message_id(1, 2, None)]); + assert_eq!(effects.dlq_sends, 0); } #[test] - fn nack_with_id_source_sends_none_redelivery_count() { - assert!( - include_str!("topic.rs").contains("InternalEngineMessage::Nack(msg_id, None)"), - "nack_with_id must send InternalEngineMessage::Nack(msg_id, None) so ID-only nacks use fixed-delay fallback rather than count-based backoff." + fn id_only_negative_ack_uses_fixed_delay_behavior_without_dlq_send() { + let now = Instant::now(); + let id = message_id(3, 4, None); + let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(7)), None); + let mut effects = FakeNegativeAckEffects::default(); + + assert_eq!( + tracker.schedule(id.clone(), None, now), + NegativeAckSchedule::Scheduled ); + + let due = tracker.collect_due(now + Duration::from_secs(7)); + effects.request_redelivery(due); + + assert_eq!(effects.redelivery_requests, vec![id]); + assert_eq!(effects.dlq_sends, 0); } } From c7d7481a2435535ee5551c4653c76361ee7587be Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 18:21:28 -0300 Subject: [PATCH 36/66] fix(05): WR-04 compile alternate negative ack examples Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- examples/negative_ack.rs | 82 +++++++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 30 deletions(-) diff --git a/examples/negative_ack.rs b/examples/negative_ack.rs index 73917e5c..19533dec 100644 --- a/examples/negative_ack.rs +++ b/examples/negative_ack.rs @@ -42,6 +42,54 @@ impl NegativeAckBackoff for LinearBackoff { } } +#[allow(dead_code)] +async fn build_zero_delay_consumer( + pulsar: &Pulsar, + topic: String, +) -> Result, pulsar::Error> { + pulsar + .consumer() + .with_topic(topic) + .with_consumer_name("negative_ack_zero_delay") + .with_subscription_type(SubType::Exclusive) + .with_subscription("negative_ack_subscription") + .with_nack_redelivery_delay(Duration::ZERO) + .build() + .await +} + +#[allow(dead_code)] +async fn build_multiplier_backoff_consumer( + pulsar: &Pulsar, + topic: String, +) -> Result, pulsar::Error> { + pulsar + .consumer() + .with_topic(topic) + .with_consumer_name("negative_ack_multiplier_backoff") + .with_subscription_type(SubType::Exclusive) + .with_subscription("negative_ack_subscription") + .with_negative_ack_backoff(MultiplierRedeliveryBackoff::default()) + .build() + .await +} + +#[allow(dead_code)] +async fn build_linear_backoff_consumer( + pulsar: &Pulsar, + topic: String, +) -> Result, pulsar::Error> { + pulsar + .consumer() + .with_topic(topic) + .with_consumer_name("negative_ack_linear_backoff") + .with_subscription_type(SubType::Exclusive) + .with_subscription("negative_ack_subscription") + .with_negative_ack_backoff(LinearBackoff(Duration::from_secs(10))) + .build() + .await +} + #[tokio::main] async fn main() -> Result<(), pulsar::Error> { env_logger::init(); @@ -54,14 +102,12 @@ async fn main() -> Result<(), pulsar::Error> { .unwrap_or_else(|| "non-persistent://public/default/test".to_string()); let pulsar: Pulsar<_> = Pulsar::builder(addr, TokioExecutor).build().await?; - let _built_in_backoff_example = MultiplierRedeliveryBackoff::default(); - let _custom_backoff_example = LinearBackoff(Duration::from_secs(10)); // HOW TO USE THIS EXAMPLE // Section 1 (default delay) is the active configuration and runs by default. // To try a different configuration: // 1. Comment out the Section 1 consumer builder block below. - // 2. Uncomment exactly ONE of Sections 2-4. + // 2. Replace it with exactly ONE of the compiled helper calls shown in Sections 2-4. // Running multiple sections simultaneously is not supported; each section // creates its own consumer on the same subscription. @@ -84,41 +130,17 @@ async fn main() -> Result<(), pulsar::Error> { // 60s delay. // Duration::ZERO is the pre-v* immediate-redelivery equivalent. // - // let mut consumer: Consumer = pulsar - // .consumer() - // .with_topic(topic) - // .with_consumer_name("negative_ack_zero_delay") - // .with_subscription_type(SubType::Exclusive) - // .with_subscription("negative_ack_subscription") - // .with_nack_redelivery_delay(Duration::ZERO) - // .build() - // .await?; + // let mut consumer = build_zero_delay_consumer(&pulsar, topic).await?; // Section 3: Built-in MultiplierRedeliveryBackoff. // The default policy uses broker redelivery count with a 1s minimum and 10min maximum delay. // - // let mut consumer: Consumer = pulsar - // .consumer() - // .with_topic(topic) - // .with_consumer_name("negative_ack_multiplier_backoff") - // .with_subscription_type(SubType::Exclusive) - // .with_subscription("negative_ack_subscription") - // .with_negative_ack_backoff(MultiplierRedeliveryBackoff::default()) - // .build() - // .await?; + // let mut consumer = build_multiplier_backoff_consumer(&pulsar, topic).await?; // Section 4: Custom NegativeAckBackoff policy. // Implement the trait when you need application-specific redelivery spacing. // - // let mut consumer: Consumer = pulsar - // .consumer() - // .with_topic(topic) - // .with_consumer_name("negative_ack_linear_backoff") - // .with_subscription_type(SubType::Exclusive) - // .with_subscription("negative_ack_subscription") - // .with_negative_ack_backoff(LinearBackoff(Duration::from_secs(10))) - // .build() - // .await?; + // let mut consumer = build_linear_backoff_consumer(&pulsar, topic).await?; let mut counter = 0usize; while let Some(msg) = consumer.try_next().await? { From b1a8a982db24ae174792e481db273550f9dcf5ac Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 18:31:00 -0300 Subject: [PATCH 37/66] fix(05): WR-03 return reader regex error instead of panicking Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 7c1f7037..179ef4a4 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -393,6 +393,12 @@ impl ConsumerBuilder { /// creates a [Reader] from this builder #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn into_reader(self) -> Result, Error> { + if self.topic_regex.is_some() { + return Err(Error::Custom( + "Unable to create a reader - topic regex is not supported".to_string(), + )); + } + // would this clone() consume too much memory? let (mut config, mut joined_topics) = self.clone().validate().await?; @@ -405,13 +411,13 @@ impl ConsumerBuilder { warn!("Subscription Type for a reader is `Exclusive`. Resetting."); config.sub_type = SubType::Exclusive; - if joined_topics.len() > 1 { + if joined_topics.len() != 1 { return Err(Error::Custom( - "Unable to create a reader - one topic partition max".to_string(), + "Unable to create a reader - exactly one topic is required".to_string(), )); } - let (topic, addr) = joined_topics.pop().unwrap(); + let (topic, addr) = joined_topics.pop().expect("len checked above"); let consumer = TopicConsumer::new( self.pulsar.clone(), topic, From 4e86a9702f8112234baaa10bc235165948309e99 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 18:32:51 -0300 Subject: [PATCH 38/66] fix(05): WR-01 exercise consumer construction config paths Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 134 ++++++++++++++++++++++++++++------------ src/consumer/multi.rs | 104 +++++++++++++++++++++++++++---- 2 files changed, 186 insertions(+), 52 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 179ef4a4..775a146c 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -54,6 +54,20 @@ fn check_nack_delay_duration(delay: Duration) -> Result<(), Error> { } impl ConsumerBuilder { + fn topic_consumer_configs_for_joined_topics( + config: &ConsumerConfig, + joined_topics: &[(String, BrokerAddress)], + ) -> Vec { + joined_topics + .iter() + .map(|_| config.clone_for_topic_consumer()) + .collect() + } + + fn topic_consumer_config_for_reader(config: &ConsumerConfig) -> ConsumerConfig { + config.clone_for_topic_consumer() + } + /// Creates a new [ConsumerBuilder] from an existing client instance #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn new(pulsar: &Pulsar) -> Self { @@ -341,14 +355,13 @@ impl ConsumerBuilder { // would this clone() consume too much memory? let (config, joined_topics) = self.clone().validate().await?; - let consumers = try_join_all(joined_topics.into_iter().map(|(topic, addr)| { - TopicConsumer::new( - self.pulsar.clone(), - topic, - addr, - config.clone_for_topic_consumer(), - ) - })) + let topic_consumer_configs = + Self::topic_consumer_configs_for_joined_topics(&config, &joined_topics); + let consumers = try_join_all(joined_topics.into_iter().zip(topic_consumer_configs).map( + |((topic, addr), topic_consumer_config)| { + TopicConsumer::new(self.pulsar.clone(), topic, addr, topic_consumer_config) + }, + )) .await?; let consumer = if consumers.len() == 1 { @@ -418,13 +431,9 @@ impl ConsumerBuilder { } let (topic, addr) = joined_topics.pop().expect("len checked above"); - let consumer = TopicConsumer::new( - self.pulsar.clone(), - topic, - addr, - config.clone_for_topic_consumer(), - ) - .await?; + let topic_consumer_config = Self::topic_consumer_config_for_reader(&config); + let consumer = + TopicConsumer::new(self.pulsar.clone(), topic, addr, topic_consumer_config).await?; Ok(Reader { consumer, @@ -438,6 +447,7 @@ mod tests { use std::time::Duration; use super::*; + use crate::TokioExecutor; #[test] fn test_nack_delay_validation_oversized() { @@ -541,9 +551,28 @@ mod tests { } } - fn assert_topic_consumer_receives_negative_ack_config(config: &ConsumerConfig) { - let passed_to_topic_consumer = config.clone_for_topic_consumer(); + fn broker_addr(port: u16) -> BrokerAddress { + BrokerAddress { + url: format!("pulsar://127.0.0.1:{port}").parse().unwrap(), + broker_url: format!("127.0.0.1:{port}"), + proxy: false, + } + } + + fn joined_topics(count: usize) -> Vec<(String, BrokerAddress)> { + (0..count) + .map(|idx| { + ( + format!("persistent://public/default/topic-{idx}"), + broker_addr(6650 + idx as u16), + ) + }) + .collect() + } + fn assert_topic_consumer_receives_negative_ack_config( + passed_to_topic_consumer: &ConsumerConfig, + ) { assert_eq!( passed_to_topic_consumer.nack_redelivery_delay, Some(Duration::from_secs(30)) @@ -553,48 +582,75 @@ mod tests { #[test] fn single_topic_consumer_config_carries_nack_delay_to_engine() { - // Single-topic path: TopicConsumer::new() receives ConsumerConfig directly; no broker needed. let config = nack_config_with_delay_and_backoff(); + let topic_consumer_configs = + ConsumerBuilder::::topic_consumer_configs_for_joined_topics( + &config, + &joined_topics(1), + ); - assert_topic_consumer_receives_negative_ack_config(&config); + assert_eq!(topic_consumer_configs.len(), 1); + assert_topic_consumer_receives_negative_ack_config(&topic_consumer_configs[0]); } #[test] fn partitioned_consumer_config_carries_nack_delay_to_each_engine() { - // Partitioned path clones ConsumerConfig once per partition. let config = nack_config_with_delay_and_backoff(); - assert_topic_consumer_receives_negative_ack_config(&config); + let topic_consumer_configs = + ConsumerBuilder::::topic_consumer_configs_for_joined_topics( + &config, + &joined_topics(3), + ); + + assert_eq!(topic_consumer_configs.len(), 3); + for topic_consumer_config in &topic_consumer_configs { + assert_topic_consumer_receives_negative_ack_config(topic_consumer_config); + } } #[test] fn multi_topic_consumer_config_carries_nack_delay_to_each_engine() { - // Multi-topic path uses the same ConsumerConfig clone per TopicConsumer. let config = nack_config_with_delay_and_backoff(); - assert_topic_consumer_receives_negative_ack_config(&config); + let topic_consumer_configs = + ConsumerBuilder::::topic_consumer_configs_for_joined_topics( + &config, + &joined_topics(2), + ); + + assert_eq!(topic_consumer_configs.len(), 2); + for topic_consumer_config in &topic_consumer_configs { + assert_topic_consumer_receives_negative_ack_config(topic_consumer_config); + } } #[test] - fn regex_consumer_config_carries_nack_delay_to_each_engine() { - // Regex path carries ConsumerConfig through topic refresh into each TopicConsumer. + fn reader_consumer_config_carries_nack_delay_to_engine() { let config = nack_config_with_delay_and_backoff(); - assert_topic_consumer_receives_negative_ack_config(&config); + let mut reader_config = config; + reader_config.options.durable = Some(false); + reader_config.sub_type = SubType::Exclusive; + + let topic_consumer_config = + ConsumerBuilder::::topic_consumer_config_for_reader(&reader_config); + + assert_topic_consumer_receives_negative_ack_config(&topic_consumer_config); + assert_eq!(topic_consumer_config.options.durable, Some(false)); + assert_eq!(topic_consumer_config.sub_type, SubType::Exclusive); } #[test] - fn reader_consumer_config_carries_nack_delay_to_engine() { - // Reader path uses the same ConsumerConfig spine; Reader does not expose nack() API (D-03). + fn build_path_config_helpers_clone_for_each_topic_consumer() { let mut config = nack_config_with_delay_and_backoff(); - config.options.durable = Some(false); - config.sub_type = SubType::Exclusive; + config.consumer_id = Some(7); + let topic_consumer_configs = + ConsumerBuilder::::topic_consumer_configs_for_joined_topics( + &config, + &joined_topics(2), + ); - assert_topic_consumer_receives_negative_ack_config(&config); - assert_eq!( - config.clone_for_topic_consumer().options.durable, - Some(false) - ); - assert_eq!( - config.clone_for_topic_consumer().sub_type, - SubType::Exclusive - ); + assert_eq!(topic_consumer_configs.len(), 2); + assert!(topic_consumer_configs + .iter() + .all(|topic_consumer_config| topic_consumer_config.consumer_id == Some(7))); } } diff --git a/src/consumer/multi.rs b/src/consumer/multi.rs index e59dee2d..04195326 100644 --- a/src/consumer/multi.rs +++ b/src/consumer/multi.rs @@ -41,6 +41,21 @@ pub struct MultiTopicConsumer { } impl MultiTopicConsumer { + fn topic_consumer_configs_for_refresh( + consumer_config: &ConsumerConfig, + topics: I, + existing_topics: &VecDeque, + ) -> Vec<(String, crate::BrokerAddress, ConsumerConfig)> + where + I: IntoIterator, + { + topics + .into_iter() + .filter(|(topic, _)| !existing_topics.contains(topic)) + .map(|(topic, addr)| (topic, addr, consumer_config.clone_for_topic_consumer())) + .collect() + } + #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn topics(&self) -> Vec { self.topics.iter().map(|s| s.to_string()).collect() @@ -184,20 +199,18 @@ impl MultiTopicConsumer { .into_iter() .flatten(); + let topic_consumer_specs = Self::topic_consumer_configs_for_refresh( + &consumer_config, + topics, + &existing_topics, + ); + // 3. create consumers - let consumers = try_join_all( - topics - .into_iter() - .filter(|(t, _)| !existing_topics.contains(t)) - .map(|(topic, addr)| { - TopicConsumer::new( - pulsar.clone(), - topic, - addr, - consumer_config.clone_for_topic_consumer(), - ) - }), - ) + let consumers = try_join_all(topic_consumer_specs.into_iter().map( + |(topic, addr, topic_consumer_config)| { + TopicConsumer::new(pulsar.clone(), topic, addr, topic_consumer_config) + }, + )) .await?; if !consumers.is_empty() { let topics: Vec = consumers.iter().map(|c| c.topic()).collect(); @@ -409,3 +422,68 @@ impl Stream for MultiTopicConsum Poll::Pending } } + +#[cfg(test)] +mod tests { + use std::{collections::VecDeque, sync::Arc, time::Duration}; + + use super::*; + use crate::{ + consumer::negative_ack_backoff::MultiplierRedeliveryBackoff, BrokerAddress, TokioExecutor, + }; + + fn broker_addr(port: u16) -> BrokerAddress { + BrokerAddress { + url: format!("pulsar://127.0.0.1:{port}").parse().unwrap(), + broker_url: format!("127.0.0.1:{port}"), + proxy: false, + } + } + + fn nack_config_with_delay_and_backoff() -> ConsumerConfig { + ConsumerConfig { + nack_redelivery_delay: Some(Duration::from_secs(30)), + negative_ack_backoff: Some(Arc::new(MultiplierRedeliveryBackoff::default())), + ..ConsumerConfig::default() + } + } + + #[test] + fn regex_refresh_config_carries_nack_delay_to_each_new_consumer() { + let config = nack_config_with_delay_and_backoff(); + let existing_topics = VecDeque::from(["persistent://public/default/existing".to_string()]); + let topics = vec![ + ( + "persistent://public/default/existing".to_string(), + broker_addr(6650), + ), + ( + "persistent://public/default/new-1".to_string(), + broker_addr(6651), + ), + ( + "persistent://public/default/new-2".to_string(), + broker_addr(6652), + ), + ]; + + let topic_consumer_specs = + MultiTopicConsumer::::topic_consumer_configs_for_refresh( + &config, + topics, + &existing_topics, + ); + + assert_eq!(topic_consumer_specs.len(), 2); + for (_, _, topic_consumer_config) in &topic_consumer_specs { + assert_eq!( + topic_consumer_config.nack_redelivery_delay, + Some(Duration::from_secs(30)) + ); + assert!(topic_consumer_config.negative_ack_backoff.is_some()); + } + assert!(topic_consumer_specs + .iter() + .all(|(topic, _, _)| topic != "persistent://public/default/existing")); + } +} From 4d6783e1b66bbf1dfc835f880d9694738e835b18 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 18:34:59 -0300 Subject: [PATCH 39/66] fix(05): WR-02 add engine effect seam for nack DLQ tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/engine.rs | 135 +++++++++++++++++++++++++++++++++-------- 1 file changed, 110 insertions(+), 25 deletions(-) diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index ad20f4aa..e0511c1e 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -42,6 +42,65 @@ use crate::{ const SYSTEM_PROPERTY_REAL_TOPIC: &str = "REAL_TOPIC"; const SYSTEM_PROPERTY_ORIGIN_MESSAGE_ID: &str = "ORIGIN_MESSAGE_ID"; +#[async_trait::async_trait] +trait ConsumerEngineEffects { + async fn request_negative_ack_redelivery( + &mut self, + consumer_id: u64, + ids: Vec, + ) -> Result<(), ConnectionError>; + + async fn send_dead_letter(&mut self, topic: &str, message: Message) -> Result<(), Error>; +} + +struct BrokerConsumerEngineEffects { + client: Pulsar, + connection: Arc>, +} + +#[async_trait::async_trait] +impl ConsumerEngineEffects for BrokerConsumerEngineEffects { + async fn request_negative_ack_redelivery( + &mut self, + consumer_id: u64, + ids: Vec, + ) -> Result<(), ConnectionError> { + self.connection + .sender() + .send_redeliver_unacknowleged_messages(consumer_id, ids) + .await + } + + async fn send_dead_letter(&mut self, topic: &str, message: Message) -> Result<(), Error> { + self.client + .send(topic, message) + .await? + .await + .map(|_| ()) + .map_err(|e| { + error!("One shot cancelled {:?}", e); + Error::Custom("DLQ send error".to_string()) + }) + } +} + +async fn send_negative_ack_redelivery_with_effects( + effects: &mut E, + consumer_id: u64, + ids: Vec, +) -> Result<(), ConnectionError> +where + E: ConsumerEngineEffects + ?Sized, +{ + let ids = ids + .into_iter() + .map(|id| NegativeAckTracker::redelivery_message_id(&id)) + .collect(); + effects + .request_negative_ack_redelivery(consumer_id, ids) + .await +} + pub struct ConsumerEngine { client: Pulsar, connection: Arc>, @@ -584,14 +643,11 @@ impl ConsumerEngine { &self, ids: Vec, ) -> Result<(), ConnectionError> { - let ids = ids - .into_iter() - .map(|id| NegativeAckTracker::redelivery_message_id(&id)) - .collect(); - self.connection - .sender() - .send_redeliver_unacknowleged_messages(self.id, ids) - .await + let mut effects = BrokerConsumerEngineEffects { + client: self.client.clone(), + connection: self.connection.clone(), + }; + send_negative_ack_redelivery_with_effects(&mut effects, self.id, ids).await } #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] @@ -846,14 +902,13 @@ impl ConsumerEngine { event_time: metadata.event_time, ..Default::default() }; - self.client - .send(&dead_letter_policy.dead_letter_topic, message) - .await? - .await - .map_err(|e| { - error!("One shot cancelled {:?}", e); - Error::Custom("DLQ send error".to_string()) - })?; + let mut effects = BrokerConsumerEngineEffects { + client: self.client.clone(), + connection: self.connection.clone(), + }; + effects + .send_dead_letter(&dead_letter_policy.dead_letter_topic, message) + .await?; self.ack(message_id, false).await; } @@ -984,6 +1039,9 @@ impl std::ops::Drop for ConsumerEngine { mod tests { use std::time::{Duration, Instant}; + use futures::executor::block_on; + + use super::*; use crate::{ consumer::negative_ack_tracker::{NegativeAckSchedule, NegativeAckTracker}, message::proto::MessageIdData, @@ -1106,12 +1164,24 @@ mod tests { dlq_sends: usize, } - impl FakeNegativeAckEffects { - fn request_redelivery(&mut self, ids: Vec) { - self.redelivery_requests.extend( - ids.into_iter() - .map(|id| NegativeAckTracker::redelivery_message_id(&id)), - ); + #[async_trait::async_trait] + impl ConsumerEngineEffects for FakeNegativeAckEffects { + async fn request_negative_ack_redelivery( + &mut self, + _consumer_id: u64, + ids: Vec, + ) -> Result<(), ConnectionError> { + self.redelivery_requests.extend(ids); + Ok(()) + } + + async fn send_dead_letter( + &mut self, + _topic: &str, + _message: crate::producer::Message, + ) -> Result<(), Error> { + self.dlq_sends += 1; + Ok(()) } } @@ -1138,7 +1208,12 @@ mod tests { tracker.schedule(id.clone(), Some(99), now), NegativeAckSchedule::Immediate ); - effects.request_redelivery(vec![id.clone()]); + block_on(send_negative_ack_redelivery_with_effects( + &mut effects, + 7, + vec![id.clone()], + )) + .unwrap(); assert_eq!(effects.redelivery_requests, vec![id]); assert_eq!(effects.dlq_sends, 0); @@ -1158,7 +1233,12 @@ mod tests { assert!(tracker.collect_due(now + Duration::from_secs(4)).is_empty()); let due = tracker.collect_due(now + Duration::from_secs(5)); - effects.request_redelivery(due); + block_on(send_negative_ack_redelivery_with_effects( + &mut effects, + 7, + due, + )) + .unwrap(); assert_eq!(effects.redelivery_requests, vec![message_id(1, 2, None)]); assert_eq!(effects.dlq_sends, 0); @@ -1177,7 +1257,12 @@ mod tests { ); let due = tracker.collect_due(now + Duration::from_secs(7)); - effects.request_redelivery(due); + block_on(send_negative_ack_redelivery_with_effects( + &mut effects, + 7, + due, + )) + .unwrap(); assert_eq!(effects.redelivery_requests, vec![id]); assert_eq!(effects.dlq_sends, 0); From 1f7bf1f3888b05437636055f052f7d42f5c750c9 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 18:42:53 -0300 Subject: [PATCH 40/66] fix(05): tighten review fix behavior seams Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 128 ++++++++++++------ src/consumer/engine.rs | 281 ++++++++++++++++++++++++++-------------- src/consumer/multi.rs | 57 ++++++-- src/consumer/topic.rs | 19 +++ 4 files changed, 334 insertions(+), 151 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 775a146c..af1cb43f 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -10,8 +10,12 @@ use regex::Regex; use crate::{ consumer::{ - config::ConsumerConfig, data::DeadLetterPolicy, multi::MultiTopicConsumer, - negative_ack_backoff::NegativeAckBackoff, options::ConsumerOptions, topic::TopicConsumer, + config::ConsumerConfig, + data::DeadLetterPolicy, + multi::MultiTopicConsumer, + negative_ack_backoff::NegativeAckBackoff, + options::ConsumerOptions, + topic::{create_topic_consumers_with_factory, TopicConsumer}, InnerConsumer, }, message::proto::command_subscribe::SubType, @@ -54,18 +58,22 @@ fn check_nack_delay_duration(delay: Duration) -> Result<(), Error> { } impl ConsumerBuilder { - fn topic_consumer_configs_for_joined_topics( + fn topic_consumer_specs_for_joined_topics( config: &ConsumerConfig, - joined_topics: &[(String, BrokerAddress)], - ) -> Vec { + joined_topics: Vec<(String, BrokerAddress)>, + ) -> Vec<(String, BrokerAddress, ConsumerConfig)> { joined_topics - .iter() - .map(|_| config.clone_for_topic_consumer()) + .into_iter() + .map(|(topic, addr)| (topic, addr, config.clone_for_topic_consumer())) .collect() } - fn topic_consumer_config_for_reader(config: &ConsumerConfig) -> ConsumerConfig { - config.clone_for_topic_consumer() + fn topic_consumer_spec_for_reader( + config: &ConsumerConfig, + topic: String, + addr: BrokerAddress, + ) -> (String, BrokerAddress, ConsumerConfig) { + (topic, addr, config.clone_for_topic_consumer()) } /// Creates a new [ConsumerBuilder] from an existing client instance @@ -355,14 +363,10 @@ impl ConsumerBuilder { // would this clone() consume too much memory? let (config, joined_topics) = self.clone().validate().await?; - let topic_consumer_configs = - Self::topic_consumer_configs_for_joined_topics(&config, &joined_topics); - let consumers = try_join_all(joined_topics.into_iter().zip(topic_consumer_configs).map( - |((topic, addr), topic_consumer_config)| { - TopicConsumer::new(self.pulsar.clone(), topic, addr, topic_consumer_config) - }, - )) - .await?; + let specs = Self::topic_consumer_specs_for_joined_topics(&config, joined_topics); + let consumers = + create_topic_consumers_with_factory(self.pulsar.clone(), specs, TopicConsumer::new) + .await?; let consumer = if consumers.len() == 1 { let consumer = consumers.into_iter().next().unwrap(); @@ -431,9 +435,11 @@ impl ConsumerBuilder { } let (topic, addr) = joined_topics.pop().expect("len checked above"); - let topic_consumer_config = Self::topic_consumer_config_for_reader(&config); - let consumer = - TopicConsumer::new(self.pulsar.clone(), topic, addr, topic_consumer_config).await?; + let specs = vec![Self::topic_consumer_spec_for_reader(&config, topic, addr)]; + let mut consumers = + create_topic_consumers_with_factory(self.pulsar.clone(), specs, TopicConsumer::new) + .await?; + let consumer = consumers.pop().expect("single reader topic checked above"); Ok(Reader { consumer, @@ -444,7 +450,12 @@ impl ConsumerBuilder { #[cfg(test)] mod tests { - use std::time::Duration; + use std::{ + sync::{Arc, Mutex}, + time::Duration, + }; + + use futures::executor::block_on; use super::*; use crate::TokioExecutor; @@ -580,14 +591,39 @@ mod tests { assert!(passed_to_topic_consumer.negative_ack_backoff.is_some()); } + fn configs_received_by_topic_consumer_factory( + specs: Vec<(String, BrokerAddress, ConsumerConfig)>, + ) -> Vec { + let captured_configs = Arc::new(Mutex::new(Vec::new())); + let factory_configs = Arc::clone(&captured_configs); + + block_on(create_topic_consumers_with_factory( + (), + specs, + move |(), _topic, _addr, config| { + let factory_configs = Arc::clone(&factory_configs); + async move { + factory_configs.lock().unwrap().push(config); + Ok::<(), Error>(()) + } + }, + )) + .expect("test factory should not fail"); + + Arc::try_unwrap(captured_configs) + .expect("factory closure dropped") + .into_inner() + .unwrap() + } + #[test] fn single_topic_consumer_config_carries_nack_delay_to_engine() { let config = nack_config_with_delay_and_backoff(); - let topic_consumer_configs = - ConsumerBuilder::::topic_consumer_configs_for_joined_topics( - &config, - &joined_topics(1), - ); + let specs = ConsumerBuilder::::topic_consumer_specs_for_joined_topics( + &config, + joined_topics(1), + ); + let topic_consumer_configs = configs_received_by_topic_consumer_factory(specs); assert_eq!(topic_consumer_configs.len(), 1); assert_topic_consumer_receives_negative_ack_config(&topic_consumer_configs[0]); @@ -596,11 +632,11 @@ mod tests { #[test] fn partitioned_consumer_config_carries_nack_delay_to_each_engine() { let config = nack_config_with_delay_and_backoff(); - let topic_consumer_configs = - ConsumerBuilder::::topic_consumer_configs_for_joined_topics( - &config, - &joined_topics(3), - ); + let specs = ConsumerBuilder::::topic_consumer_specs_for_joined_topics( + &config, + joined_topics(3), + ); + let topic_consumer_configs = configs_received_by_topic_consumer_factory(specs); assert_eq!(topic_consumer_configs.len(), 3); for topic_consumer_config in &topic_consumer_configs { @@ -611,11 +647,11 @@ mod tests { #[test] fn multi_topic_consumer_config_carries_nack_delay_to_each_engine() { let config = nack_config_with_delay_and_backoff(); - let topic_consumer_configs = - ConsumerBuilder::::topic_consumer_configs_for_joined_topics( - &config, - &joined_topics(2), - ); + let specs = ConsumerBuilder::::topic_consumer_specs_for_joined_topics( + &config, + joined_topics(2), + ); + let topic_consumer_configs = configs_received_by_topic_consumer_factory(specs); assert_eq!(topic_consumer_configs.len(), 2); for topic_consumer_config in &topic_consumer_configs { @@ -630,8 +666,14 @@ mod tests { reader_config.options.durable = Some(false); reader_config.sub_type = SubType::Exclusive; - let topic_consumer_config = - ConsumerBuilder::::topic_consumer_config_for_reader(&reader_config); + let (topic, addr) = joined_topics(1).pop().unwrap(); + let spec = ConsumerBuilder::::topic_consumer_spec_for_reader( + &reader_config, + topic, + addr, + ); + let mut topic_consumer_configs = configs_received_by_topic_consumer_factory(vec![spec]); + let topic_consumer_config = topic_consumer_configs.pop().unwrap(); assert_topic_consumer_receives_negative_ack_config(&topic_consumer_config); assert_eq!(topic_consumer_config.options.durable, Some(false)); @@ -642,11 +684,11 @@ mod tests { fn build_path_config_helpers_clone_for_each_topic_consumer() { let mut config = nack_config_with_delay_and_backoff(); config.consumer_id = Some(7); - let topic_consumer_configs = - ConsumerBuilder::::topic_consumer_configs_for_joined_topics( - &config, - &joined_topics(2), - ); + let specs = ConsumerBuilder::::topic_consumer_specs_for_joined_topics( + &config, + joined_topics(2), + ); + let topic_consumer_configs = configs_received_by_topic_consumer_factory(specs); assert_eq!(topic_consumer_configs.len(), 2); assert!(topic_consumer_configs diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index e0511c1e..caf81b2d 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -101,6 +101,28 @@ where .await } +async fn dispatch_negative_ack_redelivery_with_effects( + tracker: &mut NegativeAckTracker, + effects: &mut E, + consumer_id: u64, + ids: &[MessageIdData], + retry_at: Instant, +) -> Result<(), ConnectionError> +where + E: ConsumerEngineEffects + ?Sized, +{ + match send_negative_ack_redelivery_with_effects(effects, consumer_id, ids.to_vec()).await { + Ok(()) => { + tracker.mark_dispatched(ids); + Ok(()) + } + Err(e) => { + tracker.mark_retry_pending(ids, retry_at); + Err(e) + } + } +} + pub struct ConsumerEngine { client: Pulsar, connection: Arc>, @@ -545,13 +567,22 @@ impl ConsumerEngine { match schedule { NegativeAckSchedule::Immediate => { - match self - .send_negative_ack_redelivery(vec![message_id.clone()]) - .await + let mut effects = BrokerConsumerEngineEffects { + client: self.client.clone(), + connection: self.connection.clone(), + }; + let consumer_id = self.id; + let retry_at = Instant::now() + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL; + match dispatch_negative_ack_redelivery_with_effects( + self.ensure_negative_ack_tracker(), + &mut effects, + consumer_id, + std::slice::from_ref(&message_id), + retry_at, + ) + .await { Ok(()) => { - self.ensure_negative_ack_tracker() - .mark_dispatched(std::slice::from_ref(&message_id)); self.stop_negative_ack_ticker_if_idle(); } Err(e) => { @@ -559,25 +590,26 @@ impl ConsumerEngine { "could not ask for immediate negative ack redelivery: {:?}", e ); - if let Some(tracker) = self.negative_ack_tracker.as_mut() { - tracker.mark_retry_pending( - std::slice::from_ref(&message_id), - Instant::now() + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, - ); - } if self.start_negative_ack_ticker().is_err() { error!("could not start negative ack redelivery ticker after immediate redelivery failure"); - if let Err(e) = self - .send_negative_ack_redelivery(vec![message_id.clone()]) - .await + let mut effects = BrokerConsumerEngineEffects { + client: self.client.clone(), + connection: self.connection.clone(), + }; + let retry_at = Instant::now() + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL; + if let Err(e) = dispatch_negative_ack_redelivery_with_effects( + self.ensure_negative_ack_tracker(), + &mut effects, + consumer_id, + std::slice::from_ref(&message_id), + retry_at, + ) + .await { error!( "could not ask for fallback immediate negative ack redelivery after ticker failure: {:?}", e ); - } else { - self.ensure_negative_ack_tracker() - .mark_dispatched(std::slice::from_ref(&message_id)); } self.stop_negative_ack_ticker_if_idle(); } @@ -587,9 +619,21 @@ impl ConsumerEngine { NegativeAckSchedule::Scheduled | NegativeAckSchedule::DuplicateEarlier => { if self.start_negative_ack_ticker().is_err() { error!("could not start negative ack redelivery ticker"); - self.ensure_negative_ack_tracker() - .mark_dispatched(std::slice::from_ref(&message_id)); - if let Err(e) = self.send_negative_ack_redelivery(vec![message_id]).await { + let mut effects = BrokerConsumerEngineEffects { + client: self.client.clone(), + connection: self.connection.clone(), + }; + let consumer_id = self.id; + let retry_at = Instant::now() + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL; + if let Err(e) = dispatch_negative_ack_redelivery_with_effects( + self.ensure_negative_ack_tracker(), + &mut effects, + consumer_id, + std::slice::from_ref(&message_id), + retry_at, + ) + .await + { error!( "could not ask for immediate negative ack redelivery after ticker failure: {:?}", e @@ -621,35 +665,29 @@ impl ConsumerEngine { return; } - let now = Instant::now(); - match self.send_negative_ack_redelivery(ids.clone()).await { - Ok(()) => { - if let Some(tracker) = self.negative_ack_tracker.as_mut() { - tracker.mark_dispatched(&ids); - } - } - Err(e) => { + let mut effects = BrokerConsumerEngineEffects { + client: self.client.clone(), + connection: self.connection.clone(), + }; + let consumer_id = self.id; + let retry_at = Instant::now() + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL; + if let Some(tracker) = self.negative_ack_tracker.as_mut() { + if let Err(e) = dispatch_negative_ack_redelivery_with_effects( + tracker, + &mut effects, + consumer_id, + &ids, + retry_at, + ) + .await + { error!("could not ask for delayed negative ack redelivery: {:?}", e); - if let Some(tracker) = self.negative_ack_tracker.as_mut() { - tracker.mark_retry_pending(&ids, now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL); - } } } self.stop_negative_ack_ticker_if_idle(); } - async fn send_negative_ack_redelivery( - &self, - ids: Vec, - ) -> Result<(), ConnectionError> { - let mut effects = BrokerConsumerEngineEffects { - client: self.client.clone(), - connection: self.connection.clone(), - }; - send_negative_ack_redelivery_with_effects(&mut effects, self.id, ids).await - } - #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] async fn ack(&mut self, message_id: MessageIdData, cumulative: bool) { // FIXME: this does not handle cumulative acks @@ -1054,33 +1092,28 @@ mod tests { #[test] fn nack_handling_uses_tracker_with_zero_delay_immediate_redelivery() { - assert!(source_contains(&["NegativeAck", "Schedule::Immediate"])); - assert!(source_contains(&["ensure_", "negative_ack_tracker"])); - assert!(source_contains(&[ - "ensure_negative_ack_tracker().", - "schedule(" - ])); - assert!(source_contains(&[ - "remove_", - "negative_ack_from_unacked(&message_id)" - ])); - assert!(source_contains(&[ - "NegativeAckTracker::", - "is_same_redelivery_entry(id, message_id)", - ])); - assert!(source_contains(&["redelivery_", "count,"])); - assert!(source_contains(&["Instant::", "now(),"])); - assert!(source_contains(&[ - "send_negative_ack_", - "redelivery(vec![message_id.clone()]", - ])); - assert!(source_contains(&[ - "mark_", - "dispatched(std::slice::from_ref(&message_id))", - ])); - assert!(source_contains( - &["stop_", "negative_ack_ticker_if_idle()",] - )); + let now = Instant::now(); + let id = message_id(1, 2, Some(0)); + let mut tracker = NegativeAckTracker::new(Some(Duration::ZERO), None); + let mut effects = FakeNegativeAckEffects::default(); + + assert_eq!( + tracker.schedule(id.clone(), Some(1), now), + NegativeAckSchedule::Immediate + ); + block_on(dispatch_negative_ack_redelivery_with_effects( + &mut tracker, + &mut effects, + 7, + std::slice::from_ref(&id), + now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, + )) + .unwrap(); + + assert_eq!(effects.redelivery_requests, vec![message_id(1, 2, None)]); + assert!(tracker + .collect_due(now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL) + .is_empty()); } #[test] @@ -1096,30 +1129,62 @@ mod tests { #[test] fn due_redelivery_dispatch_is_in_flight_retry_safe() { - assert!(source_contains(&["collect_", "due(Instant::now())"])); - assert!(source_contains(&["mark_", "dispatched(&ids)"])); - assert!(source_contains(&["mark_", "retry_pending("])); - assert!(source_contains(&["&ids", ","])); - assert!(source_contains(&[ - "now + NEGATIVE_ACK_", - "REDELIVERY_TICK_INTERVAL" - ])); + let now = Instant::now(); + let id = message_id(1, 2, Some(3)); + let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(5)), None); + let mut effects = FakeNegativeAckEffects { + fail_redelivery: true, + ..FakeNegativeAckEffects::default() + }; + + assert_eq!( + tracker.schedule(id.clone(), Some(1), now), + NegativeAckSchedule::Scheduled + ); + let due = tracker.collect_due(now + Duration::from_secs(5)); + assert_eq!(due, vec![message_id(1, 2, None)]); + + let result = block_on(dispatch_negative_ack_redelivery_with_effects( + &mut tracker, + &mut effects, + 7, + &due, + now + Duration::from_secs(5) + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, + )); + + assert!(matches!(result, Err(ConnectionError::Disconnected))); + assert_eq!(effects.redelivery_requests, vec![message_id(1, 2, None)]); + let retried = tracker + .collect_due(now + Duration::from_secs(5) + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL); + assert_eq!(retried, vec![message_id(1, 2, None)]); } #[test] fn immediate_nack_failure_keeps_entry_retryable() { - let immediate_source = include_str!("engine.rs") - .split("NegativeAckSchedule::Immediate =>") - .nth(1) - .and_then(|tail| tail.split("NegativeAckSchedule::Scheduled").next()) - .expect("immediate nack handling source section exists"); - - assert!(immediate_source.contains("mark_retry_pending(")); - assert!(immediate_source.contains("start_negative_ack_ticker().is_err()")); - assert!(immediate_source.contains("NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL")); - assert!(immediate_source - .contains("fallback immediate negative ack redelivery after ticker failure")); - assert!(immediate_source.contains("mark_dispatched(std::slice::from_ref(&message_id))")); + let now = Instant::now(); + let id = message_id(5, 6, None); + let mut tracker = NegativeAckTracker::new(Some(Duration::ZERO), None); + let mut effects = FakeNegativeAckEffects { + fail_redelivery: true, + ..FakeNegativeAckEffects::default() + }; + + assert_eq!( + tracker.schedule(id.clone(), Some(99), now), + NegativeAckSchedule::Immediate + ); + let result = block_on(dispatch_negative_ack_redelivery_with_effects( + &mut tracker, + &mut effects, + 7, + std::slice::from_ref(&id), + now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, + )); + + assert!(matches!(result, Err(ConnectionError::Disconnected))); + assert_eq!(effects.redelivery_requests, vec![id]); + let retried = tracker.collect_due(now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL); + assert_eq!(retried, vec![message_id(5, 6, None)]); } #[test] @@ -1162,6 +1227,7 @@ mod tests { struct FakeNegativeAckEffects { redelivery_requests: Vec, dlq_sends: usize, + fail_redelivery: bool, } #[async_trait::async_trait] @@ -1172,7 +1238,11 @@ mod tests { ids: Vec, ) -> Result<(), ConnectionError> { self.redelivery_requests.extend(ids); - Ok(()) + if self.fail_redelivery { + Err(ConnectionError::Disconnected) + } else { + Ok(()) + } } async fn send_dead_letter( @@ -1208,15 +1278,20 @@ mod tests { tracker.schedule(id.clone(), Some(99), now), NegativeAckSchedule::Immediate ); - block_on(send_negative_ack_redelivery_with_effects( + block_on(dispatch_negative_ack_redelivery_with_effects( + &mut tracker, &mut effects, 7, - vec![id.clone()], + std::slice::from_ref(&id), + now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, )) .unwrap(); - assert_eq!(effects.redelivery_requests, vec![id]); + assert_eq!(effects.redelivery_requests, vec![id.clone()]); assert_eq!(effects.dlq_sends, 0); + assert!(tracker + .collect_due(now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL) + .is_empty()); } #[test] @@ -1233,15 +1308,20 @@ mod tests { assert!(tracker.collect_due(now + Duration::from_secs(4)).is_empty()); let due = tracker.collect_due(now + Duration::from_secs(5)); - block_on(send_negative_ack_redelivery_with_effects( + block_on(dispatch_negative_ack_redelivery_with_effects( + &mut tracker, &mut effects, 7, - due, + &due, + now + Duration::from_secs(5) + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, )) .unwrap(); assert_eq!(effects.redelivery_requests, vec![message_id(1, 2, None)]); assert_eq!(effects.dlq_sends, 0); + assert!(tracker + .collect_due(now + Duration::from_secs(5) + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL) + .is_empty()); } #[test] @@ -1257,14 +1337,19 @@ mod tests { ); let due = tracker.collect_due(now + Duration::from_secs(7)); - block_on(send_negative_ack_redelivery_with_effects( + block_on(dispatch_negative_ack_redelivery_with_effects( + &mut tracker, &mut effects, 7, - due, + &due, + now + Duration::from_secs(7) + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, )) .unwrap(); - assert_eq!(effects.redelivery_requests, vec![id]); + assert_eq!(effects.redelivery_requests, vec![id.clone()]); assert_eq!(effects.dlq_sends, 0); + assert!(tracker + .collect_due(now + Duration::from_secs(7) + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL) + .is_empty()); } } diff --git a/src/consumer/multi.rs b/src/consumer/multi.rs index 04195326..91178448 100644 --- a/src/consumer/multi.rs +++ b/src/consumer/multi.rs @@ -12,7 +12,11 @@ use futures::{future::try_join_all, Stream}; use regex::Regex; use crate::{ - consumer::{config::ConsumerConfig, message::Message, topic::TopicConsumer}, + consumer::{ + config::ConsumerConfig, + message::Message, + topic::{create_topic_consumers_with_factory, TopicConsumer}, + }, error::{ConnectionError, ConsumerError}, message::proto::{MessageIdData, Schema}, proto, @@ -206,11 +210,11 @@ impl MultiTopicConsumer { ); // 3. create consumers - let consumers = try_join_all(topic_consumer_specs.into_iter().map( - |(topic, addr, topic_consumer_config)| { - TopicConsumer::new(pulsar.clone(), topic, addr, topic_consumer_config) - }, - )) + let consumers = create_topic_consumers_with_factory( + pulsar, + topic_consumer_specs, + TopicConsumer::new, + ) .await?; if !consumers.is_empty() { let topics: Vec = consumers.iter().map(|c| c.topic()).collect(); @@ -425,7 +429,13 @@ impl Stream for MultiTopicConsum #[cfg(test)] mod tests { - use std::{collections::VecDeque, sync::Arc, time::Duration}; + use std::{ + collections::VecDeque, + sync::{Arc, Mutex}, + time::Duration, + }; + + use futures::executor::block_on; use super::*; use crate::{ @@ -448,6 +458,31 @@ mod tests { } } + fn configs_received_by_topic_consumer_factory( + specs: Vec<(String, BrokerAddress, ConsumerConfig)>, + ) -> Vec<(String, ConsumerConfig)> { + let captured_configs = Arc::new(Mutex::new(Vec::new())); + let factory_configs = Arc::clone(&captured_configs); + + block_on(create_topic_consumers_with_factory( + (), + specs, + move |(), topic, _addr, config| { + let factory_configs = Arc::clone(&factory_configs); + async move { + factory_configs.lock().unwrap().push((topic, config)); + Ok::<(), Error>(()) + } + }, + )) + .expect("test factory should not fail"); + + Arc::try_unwrap(captured_configs) + .expect("factory closure dropped") + .into_inner() + .unwrap() + } + #[test] fn regex_refresh_config_carries_nack_delay_to_each_new_consumer() { let config = nack_config_with_delay_and_backoff(); @@ -475,15 +510,17 @@ mod tests { ); assert_eq!(topic_consumer_specs.len(), 2); - for (_, _, topic_consumer_config) in &topic_consumer_specs { + let topic_consumer_configs = + configs_received_by_topic_consumer_factory(topic_consumer_specs); + for (_, topic_consumer_config) in &topic_consumer_configs { assert_eq!( topic_consumer_config.nack_redelivery_delay, Some(Duration::from_secs(30)) ); assert!(topic_consumer_config.negative_ack_backoff.is_some()); } - assert!(topic_consumer_specs + assert!(topic_consumer_configs .iter() - .all(|(topic, _, _)| topic != "persistent://public/default/existing")); + .all(|(topic, _)| topic != "persistent://public/default/existing")); } } diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index bd96c75b..d2235468 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -12,6 +12,7 @@ use std::{ use chrono::{DateTime, Utc}; use futures::{ channel::{mpsc, oneshot}, + future::try_join_all, FutureExt, SinkExt, Stream, StreamExt, }; @@ -32,6 +33,24 @@ use crate::{ BrokerAddress, DeserializeMessage, Error, Executor, Payload, Pulsar, }; +pub(crate) async fn create_topic_consumers_with_factory( + client: Client, + specs: Vec<(String, BrokerAddress, ConsumerConfig)>, + mut factory: F, +) -> Result, Error> +where + Client: Clone, + F: FnMut(Client, String, BrokerAddress, ConsumerConfig) -> Fut, + Fut: std::future::Future>, +{ + try_join_all( + specs + .into_iter() + .map(|(topic, addr, config)| factory(client.clone(), topic, addr, config)), + ) + .await +} + // this is entirely public for use in reader.rs pub struct TopicConsumer { pub(crate) consumer_id: u64, From 595b827c52c51d1f4b7579cc53e5b7f1b084eb8a Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 19:21:28 -0300 Subject: [PATCH 41/66] fix(05): preserve regex consumer with single topic Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index af1cb43f..a910a8b0 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -76,6 +76,13 @@ impl ConsumerBuilder { (topic, addr, config.clone_for_topic_consumer()) } + fn should_use_single_topic_consumer( + consumer_count: usize, + topic_regex: &Option, + ) -> bool { + consumer_count == 1 && topic_regex.is_none() + } + /// Creates a new [ConsumerBuilder] from an existing client instance #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn new(pulsar: &Pulsar) -> Self { @@ -368,7 +375,8 @@ impl ConsumerBuilder { create_topic_consumers_with_factory(self.pulsar.clone(), specs, TopicConsumer::new) .await?; - let consumer = if consumers.len() == 1 { + let consumer = if Self::should_use_single_topic_consumer(consumers.len(), &self.topic_regex) + { let consumer = consumers.into_iter().next().unwrap(); InnerConsumer::Single(consumer) } else { @@ -695,4 +703,13 @@ mod tests { .iter() .all(|topic_consumer_config| topic_consumer_config.consumer_id == Some(7))); } + + #[test] + fn explicit_single_topic_with_topic_regex_uses_multi_topic_build_path() { + let topic_regex = Some(Regex::new("persistent://public/default/regex-.*").unwrap()); + + assert!( + !ConsumerBuilder::::should_use_single_topic_consumer(1, &topic_regex) + ); + } } From f413ecbbb558d7dd63e3fa0ae14f7280f66565a8 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 19:42:21 -0300 Subject: [PATCH 42/66] fix(05): clear cumulative ack redelivery entries Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/engine.rs | 88 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 4 deletions(-) diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index caf81b2d..45f803fb 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -690,8 +690,7 @@ impl ConsumerEngine { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] async fn ack(&mut self, message_id: MessageIdData, cumulative: bool) { - // FIXME: this does not handle cumulative acks - self.unacked_messages.remove(&message_id); + Self::remove_ack_from_unacked_messages(&mut self.unacked_messages, &message_id, cumulative); if let Some(tracker) = self.negative_ack_tracker.as_mut() { if cumulative { tracker.cancel_cumulative_ack(&message_id); @@ -710,6 +709,38 @@ impl ConsumerEngine { } } + fn remove_ack_from_unacked_messages( + unacked_messages: &mut HashMap, + message_id: &MessageIdData, + cumulative: bool, + ) { + if !cumulative { + unacked_messages.remove(message_id); + return; + } + + unacked_messages.retain(|id, _| { + if id.partition != message_id.partition { + return true; + } + + if id.ledger_id < message_id.ledger_id + || (id.ledger_id == message_id.ledger_id && id.entry_id < message_id.entry_id) + { + return false; + } + + if id.ledger_id == message_id.ledger_id && id.entry_id == message_id.entry_id { + return match message_id.batch_index { + Some(ack_idx) => id.batch_index.map_or(true, |idx| idx > ack_idx), + None => false, + }; + } + + true + }); + } + /// Process the message. Returns `true` if there are more messages to process #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] async fn process_message(&mut self, message: RawMessage) -> Result { @@ -1075,7 +1106,10 @@ impl std::ops::Drop for ConsumerEngine { #[cfg(test)] mod tests { - use std::time::{Duration, Instant}; + use std::{ + collections::{HashMap, HashSet}, + time::{Duration, Instant}, + }; use futures::executor::block_on; @@ -1210,6 +1244,43 @@ mod tests { assert!(source_contains(&["ticker_running.", "store(false"])); } + #[test] + fn cumulative_ack_clears_prior_unacked_redelivery_entries() { + let due_at = Instant::now() - Duration::from_secs(1); + let mut unacked_messages = HashMap::from([ + (message_id(1, 1, None), due_at), + (message_id(1, 2, Some(0)), due_at), + (message_id(1, 2, Some(1)), due_at), + (message_id(1, 2, Some(2)), due_at), + (message_id(1, 2, None), due_at), + (message_id(1, 3, None), due_at), + (message_id_for_partition(1, 1, Some(1), None), due_at), + ]); + + ConsumerEngine::::remove_ack_from_unacked_messages( + &mut unacked_messages, + &message_id(1, 2, Some(1)), + true, + ); + + assert!(!unacked_messages.contains_key(&message_id(1, 1, None))); + assert!(!unacked_messages.contains_key(&message_id(1, 2, Some(0)))); + assert!(!unacked_messages.contains_key(&message_id(1, 2, Some(1)))); + assert!(unacked_messages.contains_key(&message_id(1, 2, Some(2)))); + assert!(unacked_messages.contains_key(&message_id(1, 2, None))); + assert!(unacked_messages.contains_key(&message_id(1, 3, None))); + assert!(unacked_messages.contains_key(&message_id_for_partition(1, 1, Some(1), None))); + + let now = Instant::now(); + let due_for_redelivery: HashSet<_> = unacked_messages + .iter() + .filter_map(|(id, redeliver_at)| (*redeliver_at < now).then_some(id.clone())) + .collect(); + assert!(!due_for_redelivery.contains(&message_id(1, 1, None))); + assert!(!due_for_redelivery.contains(&message_id(1, 2, Some(0)))); + assert!(!due_for_redelivery.contains(&message_id(1, 2, Some(1)))); + } + #[test] fn reconnect_preserves_negative_ack_tracker_state() { let source = include_str!("engine.rs"); @@ -1256,10 +1327,19 @@ mod tests { } fn message_id(ledger_id: u64, entry_id: u64, batch_index: Option) -> MessageIdData { + message_id_for_partition(ledger_id, entry_id, Some(0), batch_index) + } + + fn message_id_for_partition( + ledger_id: u64, + entry_id: u64, + partition: Option, + batch_index: Option, + ) -> MessageIdData { MessageIdData { ledger_id, entry_id, - partition: Some(0), + partition, batch_index, ack_set: vec![], batch_size: None, From f3adc7037a4168f37af5afee356b2695997ed53c Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 20:32:50 -0300 Subject: [PATCH 43/66] fix(05): clean ack state after broker ack succeeds Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/engine.rs | 56 +++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index 45f803fb..c66003da 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -690,22 +690,31 @@ impl ConsumerEngine { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] async fn ack(&mut self, message_id: MessageIdData, cumulative: bool) { - Self::remove_ack_from_unacked_messages(&mut self.unacked_messages, &message_id, cumulative); - if let Some(tracker) = self.negative_ack_tracker.as_mut() { - if cumulative { - tracker.cancel_cumulative_ack(&message_id); - } else { - tracker.cancel_ack(&message_id); - } - } - self.stop_negative_ack_ticker_if_idle(); let res = self .connection .sender() - .send_ack(self.id, vec![message_id], cumulative) + .send_ack(self.id, vec![message_id.clone()], cumulative) .await; - if res.is_err() { - error!("ack error: {:?}", res); + + match res { + Ok(()) => { + Self::remove_ack_from_unacked_messages( + &mut self.unacked_messages, + &message_id, + cumulative, + ); + if let Some(tracker) = self.negative_ack_tracker.as_mut() { + if cumulative { + tracker.cancel_cumulative_ack(&message_id); + } else { + tracker.cancel_ack(&message_id); + } + } + self.stop_negative_ack_ticker_if_idle(); + } + Err(e) => { + error!("ack error: {:?}", e); + } } } @@ -1244,6 +1253,29 @@ mod tests { assert!(source_contains(&["ticker_running.", "store(false"])); } + #[test] + fn ack_cleans_local_state_only_after_broker_ack_succeeds() { + let source = include_str!("engine.rs"); + let ack_source = source + .split("async fn ack") + .nth(1) + .and_then(|tail| tail.split("fn remove_ack_from_unacked_messages").next()) + .expect("ack source section exists"); + + let send_ack_pos = ack_source.find("send_ack").expect("ack sends broker ack"); + let ok_pos = ack_source.find("Ok(()) =>").expect("ack handles success"); + let cleanup_pos = ack_source + .find("Self::remove_ack_from_unacked_messages") + .expect("ack cleans unacked messages"); + let cancel_pos = ack_source + .find("tracker.cancel_") + .expect("ack cancels negative ack tracker state"); + + assert!(send_ack_pos < ok_pos); + assert!(ok_pos < cleanup_pos); + assert!(ok_pos < cancel_pos); + } + #[test] fn cumulative_ack_clears_prior_unacked_redelivery_entries() { let due_at = Instant::now() - Duration::from_secs(1); From 4c000a935f083f5f74123ef5e6652b136e5c7794 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 1 Jun 2026 20:35:48 -0300 Subject: [PATCH 44/66] fix(05): test ack cleanup with effect seam Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/engine.rs | 164 +++++++++++++++++++++++++++++++---------- 1 file changed, 127 insertions(+), 37 deletions(-) diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index c66003da..6125b146 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -50,6 +50,13 @@ trait ConsumerEngineEffects { ids: Vec, ) -> Result<(), ConnectionError>; + async fn send_ack( + &mut self, + consumer_id: u64, + ids: Vec, + cumulative: bool, + ) -> Result<(), ConnectionError>; + async fn send_dead_letter(&mut self, topic: &str, message: Message) -> Result<(), Error>; } @@ -71,6 +78,18 @@ impl ConsumerEngineEffects for BrokerConsumerEngineEffects { .await } + async fn send_ack( + &mut self, + consumer_id: u64, + ids: Vec, + cumulative: bool, + ) -> Result<(), ConnectionError> { + self.connection + .sender() + .send_ack(consumer_id, ids, cumulative) + .await + } + async fn send_dead_letter(&mut self, topic: &str, message: Message) -> Result<(), Error> { self.client .send(topic, message) @@ -690,26 +709,22 @@ impl ConsumerEngine { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] async fn ack(&mut self, message_id: MessageIdData, cumulative: bool) { - let res = self - .connection - .sender() - .send_ack(self.id, vec![message_id.clone()], cumulative) - .await; + let mut effects = BrokerConsumerEngineEffects { + client: self.client.clone(), + connection: self.connection.clone(), + }; - match res { + match Self::send_ack_and_clean_local_state( + &mut effects, + self.id, + &mut self.unacked_messages, + self.negative_ack_tracker.as_mut(), + message_id, + cumulative, + ) + .await + { Ok(()) => { - Self::remove_ack_from_unacked_messages( - &mut self.unacked_messages, - &message_id, - cumulative, - ); - if let Some(tracker) = self.negative_ack_tracker.as_mut() { - if cumulative { - tracker.cancel_cumulative_ack(&message_id); - } else { - tracker.cancel_ack(&message_id); - } - } self.stop_negative_ack_ticker_if_idle(); } Err(e) => { @@ -718,6 +733,33 @@ impl ConsumerEngine { } } + async fn send_ack_and_clean_local_state( + effects: &mut E, + consumer_id: u64, + unacked_messages: &mut HashMap, + negative_ack_tracker: Option<&mut NegativeAckTracker>, + message_id: MessageIdData, + cumulative: bool, + ) -> Result<(), ConnectionError> + where + E: ConsumerEngineEffects + ?Sized, + { + effects + .send_ack(consumer_id, vec![message_id.clone()], cumulative) + .await?; + + Self::remove_ack_from_unacked_messages(unacked_messages, &message_id, cumulative); + if let Some(tracker) = negative_ack_tracker { + if cumulative { + tracker.cancel_cumulative_ack(&message_id); + } else { + tracker.cancel_ack(&message_id); + } + } + + Ok(()) + } + fn remove_ack_from_unacked_messages( unacked_messages: &mut HashMap, message_id: &MessageIdData, @@ -1255,25 +1297,57 @@ mod tests { #[test] fn ack_cleans_local_state_only_after_broker_ack_succeeds() { - let source = include_str!("engine.rs"); - let ack_source = source - .split("async fn ack") - .nth(1) - .and_then(|tail| tail.split("fn remove_ack_from_unacked_messages").next()) - .expect("ack source section exists"); - - let send_ack_pos = ack_source.find("send_ack").expect("ack sends broker ack"); - let ok_pos = ack_source.find("Ok(()) =>").expect("ack handles success"); - let cleanup_pos = ack_source - .find("Self::remove_ack_from_unacked_messages") - .expect("ack cleans unacked messages"); - let cancel_pos = ack_source - .find("tracker.cancel_") - .expect("ack cancels negative ack tracker state"); - - assert!(send_ack_pos < ok_pos); - assert!(ok_pos < cleanup_pos); - assert!(ok_pos < cancel_pos); + let now = Instant::now(); + let id = message_id(1, 2, None); + let mut unacked_messages = HashMap::from([(id.clone(), now)]); + let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(5)), None); + assert_eq!( + tracker.schedule(id.clone(), Some(1), now), + NegativeAckSchedule::Scheduled + ); + let mut failing_effects = FakeNegativeAckEffects { + fail_ack: true, + ..FakeNegativeAckEffects::default() + }; + + let result = block_on( + ConsumerEngine::::send_ack_and_clean_local_state( + &mut failing_effects, + 7, + &mut unacked_messages, + Some(&mut tracker), + id.clone(), + false, + ), + ); + + assert!(matches!(result, Err(ConnectionError::Disconnected))); + assert_eq!( + failing_effects.ack_requests, + vec![(vec![id.clone()], false)] + ); + assert!(unacked_messages.contains_key(&id)); + assert!(tracker.next_due_time().is_some()); + + let mut successful_effects = FakeNegativeAckEffects::default(); + block_on( + ConsumerEngine::::send_ack_and_clean_local_state( + &mut successful_effects, + 7, + &mut unacked_messages, + Some(&mut tracker), + id.clone(), + false, + ), + ) + .unwrap(); + + assert_eq!( + successful_effects.ack_requests, + vec![(vec![id.clone()], false)] + ); + assert!(!unacked_messages.contains_key(&id)); + assert!(tracker.next_due_time().is_none()); } #[test] @@ -1329,8 +1403,10 @@ mod tests { #[derive(Default)] struct FakeNegativeAckEffects { redelivery_requests: Vec, + ack_requests: Vec<(Vec, bool)>, dlq_sends: usize, fail_redelivery: bool, + fail_ack: bool, } #[async_trait::async_trait] @@ -1348,6 +1424,20 @@ mod tests { } } + async fn send_ack( + &mut self, + _consumer_id: u64, + ids: Vec, + cumulative: bool, + ) -> Result<(), ConnectionError> { + self.ack_requests.push((ids, cumulative)); + if self.fail_ack { + Err(ConnectionError::Disconnected) + } else { + Ok(()) + } + } + async fn send_dead_letter( &mut self, _topic: &str, From a010ae00536bed841296ff672c0d4313a5e9da66 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Tue, 2 Jun 2026 11:19:51 -0300 Subject: [PATCH 45/66] test(05): exercise default nack delay in DLQ tests Remove the explicit short negative-ack redelivery delay from broker DLQ tests so they inherit the new 60s parity default. Derive the DLQ receive timeout from the shared default delay constant plus the existing receive cushion. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/mod.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/consumer/mod.rs b/src/consumer/mod.rs index d4d99f16..f502dddf 100644 --- a/src/consumer/mod.rs +++ b/src/consumer/mod.rs @@ -458,7 +458,7 @@ mod tests { ))] use tokio::time::timeout; - use super::*; + use super::{negative_ack_tracker::DEFAULT_NACK_REDELIVERY_DELAY, *}; #[cfg(any( feature = "tokio-runtime", feature = "tokio-rustls-runtime-aws-lc-rs", @@ -504,6 +504,10 @@ mod tests { const DEFAULT_RECV_TIMEOUT: Duration = Duration::from_secs(5); + fn default_nack_redelivery_recv_timeout() -> Duration { + DEFAULT_NACK_REDELIVERY_DELAY + DEFAULT_RECV_TIMEOUT + } + async fn recv_within( c: &mut Consumer, dur: Duration, @@ -900,7 +904,6 @@ mod tests { .with_subscription("nack") .with_subscription_type(SubType::Shared) .with_dead_letter_policy(dead_letter_policy) - .with_nack_redelivery_delay(Duration::from_millis(1)) .build() .await .unwrap(); @@ -936,7 +939,7 @@ mod tests { // Nacking message to send it to DLQ consumer.nack(&msg).await.unwrap(); - let dlq_msg = recv_within(&mut dlq_consumer, DEFAULT_RECV_TIMEOUT) + let dlq_msg = recv_within(&mut dlq_consumer, default_nack_redelivery_recv_timeout()) .await .unwrap(); @@ -1020,7 +1023,6 @@ mod tests { .with_subscription("nack") .with_subscription_type(SubType::Shared) .with_dead_letter_policy(dead_letter_policy) - .with_nack_redelivery_delay(Duration::from_millis(1)) .build() .await .unwrap(); @@ -1112,7 +1114,7 @@ mod tests { // Nacking message to send it to DLQ consumer.nack(&msg).await.unwrap(); - let dlq_msg = recv_within(&mut dlq_consumer, DEFAULT_RECV_TIMEOUT) + let dlq_msg = recv_within(&mut dlq_consumer, default_nack_redelivery_recv_timeout()) .await .unwrap(); println!("got message: {:?}", dlq_msg.payload); From 2323a41a66e39e6633ed438883ab14d9cd711791 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 8 Jun 2026 12:14:53 -0300 Subject: [PATCH 46/66] fix: improve nack delay validation error Make oversized negative-ack redelivery delay validation report the specific failed constraint and include the rejected Duration. Add assertions for the improved error message. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 47 +++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index a910a8b0..0f2bde56 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -23,6 +23,11 @@ use crate::{ BrokerAddress, Consumer, DeserializeMessage, Error, Executor, Pulsar, }; +const NACK_REDELIVERY_DELAY_EXCEEDS_MILLISECONDS_RANGE: &str = + "negative-ack redelivery delay is too large: it must be less than u64::MAX milliseconds"; +const NACK_REDELIVERY_DELAY_EXCEEDS_INSTANT_RANGE: &str = + "negative-ack redelivery delay is too large to schedule from the current instant"; + /// Builder structure for consumers /// /// This is the main way to create a [Consumer] or a [Reader] @@ -47,11 +52,15 @@ pub struct ConsumerBuilder { fn check_nack_delay_duration(delay: Duration) -> Result<(), Error> { if delay.as_millis() >= u64::MAX as u128 { - return Err(Error::Custom("delay duration is too large".to_string())); + return Err(Error::Custom(format!( + "{NACK_REDELIVERY_DELAY_EXCEEDS_MILLISECONDS_RANGE}: {delay:?}" + ))); } if Instant::now().checked_add(delay).is_none() { - return Err(Error::Custom("delay duration is too large".to_string())); + return Err(Error::Custom(format!( + "{NACK_REDELIVERY_DELAY_EXCEEDS_INSTANT_RANGE}: {delay:?}" + ))); } Ok(()) @@ -470,16 +479,42 @@ mod tests { #[test] fn test_nack_delay_validation_oversized() { - let result = check_nack_delay_duration(Duration::MAX); + let delay = Duration::MAX; + let result = check_nack_delay_duration(delay); - assert!(result.is_err()); + assert_nack_delay_error( + result, + NACK_REDELIVERY_DELAY_EXCEEDS_MILLISECONDS_RANGE, + delay, + ); } #[test] fn test_nack_delay_validation_rejects_unrepresentable_instant_delay() { - let result = check_nack_delay_duration(Duration::from_millis(u64::MAX)); + let delay = Duration::from_millis(u64::MAX); + let result = check_nack_delay_duration(delay); + + assert_nack_delay_error( + result, + NACK_REDELIVERY_DELAY_EXCEEDS_MILLISECONDS_RANGE, + delay, + ); + } - assert!(result.is_err()); + fn assert_nack_delay_error(result: Result<(), Error>, expected_prefix: &str, delay: Duration) { + match result { + Err(Error::Custom(message)) => { + assert!( + message.starts_with(expected_prefix), + "expected error to start with {expected_prefix:?}, got {message:?}" + ); + assert!( + message.contains(&format!("{delay:?}")), + "expected error to include the rejected Duration, got {message:?}" + ); + } + other => panic!("expected Error::Custom({expected_prefix:?}), got {other:?}"), + } } #[test] From d5213518a595120dbce36d496c2a65e13dda0e97 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 8 Jun 2026 16:00:42 -0300 Subject: [PATCH 47/66] fix(06-01): restore public receiver compatibility - Restore MessageIdDataReceiver to the upstream two-tuple shape - Remove the public MessageIdDataResult alias - Keep optional redelivery count plumbing in pub(crate) internal aliases Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/data.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/consumer/data.rs b/src/consumer/data.rs index 49ff874d..1721be8c 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -8,8 +8,7 @@ use crate::{ Error, Executor, Payload, }; -pub type MessageIdDataResult = Result<(MessageIdData, Payload, u32), Error>; -pub type MessageIdDataReceiver = mpsc::Receiver; +pub type MessageIdDataReceiver = mpsc::Receiver>; pub(crate) type InternalMessageIdDataResult = Result<(MessageIdData, Payload, Option), Error>; pub(crate) type InternalMessageIdDataSender = mpsc::Sender; pub(crate) type InternalMessageIdDataReceiver = mpsc::Receiver; @@ -129,11 +128,10 @@ mod tests { } #[test] - fn public_receiver_alias_preserves_u32_redelivery_count_shape() { - type ExpectedPublicReceiver = mpsc::Receiver>; + fn public_receiver_alias_preserves_upstream_two_tuple_shape() { + type ExpectedPublicReceiver = mpsc::Receiver>; let _: Option = None::; - let _: Option> = None::; } #[test] From 5332da5ebc8daf6f1f86744d76a5e4a8df0b8c94 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 8 Jun 2026 16:08:53 -0300 Subject: [PATCH 48/66] fix(06-02): restore regex and reader construction scope - Restore single-topic consumer selection to the upstream count check - Remove reader regex rejection and exactly-one-topic error behavior - Keep reader construction on direct TopicConsumer::new plumbing Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 36 +++++------------------------------- 1 file changed, 5 insertions(+), 31 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 0f2bde56..c6aefd05 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -85,13 +85,6 @@ impl ConsumerBuilder { (topic, addr, config.clone_for_topic_consumer()) } - fn should_use_single_topic_consumer( - consumer_count: usize, - topic_regex: &Option, - ) -> bool { - consumer_count == 1 && topic_regex.is_none() - } - /// Creates a new [ConsumerBuilder] from an existing client instance #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn new(pulsar: &Pulsar) -> Self { @@ -384,8 +377,7 @@ impl ConsumerBuilder { create_topic_consumers_with_factory(self.pulsar.clone(), specs, TopicConsumer::new) .await?; - let consumer = if Self::should_use_single_topic_consumer(consumers.len(), &self.topic_regex) - { + let consumer = if consumers.len() == 1 { let consumer = consumers.into_iter().next().unwrap(); InnerConsumer::Single(consumer) } else { @@ -427,12 +419,6 @@ impl ConsumerBuilder { /// creates a [Reader] from this builder #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn into_reader(self) -> Result, Error> { - if self.topic_regex.is_some() { - return Err(Error::Custom( - "Unable to create a reader - topic regex is not supported".to_string(), - )); - } - // would this clone() consume too much memory? let (mut config, mut joined_topics) = self.clone().validate().await?; @@ -445,18 +431,14 @@ impl ConsumerBuilder { warn!("Subscription Type for a reader is `Exclusive`. Resetting."); config.sub_type = SubType::Exclusive; - if joined_topics.len() != 1 { + if joined_topics.len() > 1 { return Err(Error::Custom( - "Unable to create a reader - exactly one topic is required".to_string(), + "Unable to create a reader - one topic partition max".to_string(), )); } - let (topic, addr) = joined_topics.pop().expect("len checked above"); - let specs = vec![Self::topic_consumer_spec_for_reader(&config, topic, addr)]; - let mut consumers = - create_topic_consumers_with_factory(self.pulsar.clone(), specs, TopicConsumer::new) - .await?; - let consumer = consumers.pop().expect("single reader topic checked above"); + let (topic, addr) = joined_topics.pop().unwrap(); + let consumer = TopicConsumer::new(self.pulsar.clone(), topic, addr, config.clone()).await?; Ok(Reader { consumer, @@ -739,12 +721,4 @@ mod tests { .all(|topic_consumer_config| topic_consumer_config.consumer_id == Some(7))); } - #[test] - fn explicit_single_topic_with_topic_regex_uses_multi_topic_build_path() { - let topic_regex = Some(Regex::new("persistent://public/default/regex-.*").unwrap()); - - assert!( - !ConsumerBuilder::::should_use_single_topic_consumer(1, &topic_regex) - ); - } } From 84141315306538e9f91dcfeb3aba7f8c7c173a02 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 8 Jun 2026 16:10:32 -0300 Subject: [PATCH 49/66] refactor(06-02): remove topic consumer test seams - Inline TopicConsumer::new construction in builder and regex refresh paths - Remove the factory and config clone production seams - Delete helper-seam tests rather than preserving source-shape checks Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 187 ++-------------------------------------- src/consumer/config.rs | 11 --- src/consumer/multi.rs | 136 ++--------------------------- src/consumer/topic.rs | 19 ---- 4 files changed, 15 insertions(+), 338 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index c6aefd05..aa3c9bc7 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -10,12 +10,8 @@ use regex::Regex; use crate::{ consumer::{ - config::ConsumerConfig, - data::DeadLetterPolicy, - multi::MultiTopicConsumer, - negative_ack_backoff::NegativeAckBackoff, - options::ConsumerOptions, - topic::{create_topic_consumers_with_factory, TopicConsumer}, + config::ConsumerConfig, data::DeadLetterPolicy, multi::MultiTopicConsumer, + negative_ack_backoff::NegativeAckBackoff, options::ConsumerOptions, topic::TopicConsumer, InnerConsumer, }, message::proto::command_subscribe::SubType, @@ -67,24 +63,6 @@ fn check_nack_delay_duration(delay: Duration) -> Result<(), Error> { } impl ConsumerBuilder { - fn topic_consumer_specs_for_joined_topics( - config: &ConsumerConfig, - joined_topics: Vec<(String, BrokerAddress)>, - ) -> Vec<(String, BrokerAddress, ConsumerConfig)> { - joined_topics - .into_iter() - .map(|(topic, addr)| (topic, addr, config.clone_for_topic_consumer())) - .collect() - } - - fn topic_consumer_spec_for_reader( - config: &ConsumerConfig, - topic: String, - addr: BrokerAddress, - ) -> (String, BrokerAddress, ConsumerConfig) { - (topic, addr, config.clone_for_topic_consumer()) - } - /// Creates a new [ConsumerBuilder] from an existing client instance #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn new(pulsar: &Pulsar) -> Self { @@ -372,10 +350,10 @@ impl ConsumerBuilder { // would this clone() consume too much memory? let (config, joined_topics) = self.clone().validate().await?; - let specs = Self::topic_consumer_specs_for_joined_topics(&config, joined_topics); - let consumers = - create_topic_consumers_with_factory(self.pulsar.clone(), specs, TopicConsumer::new) - .await?; + let consumers = try_join_all(joined_topics.into_iter().map(|(topic, addr)| { + TopicConsumer::new(self.pulsar.clone(), topic, addr, config.clone()) + })) + .await?; let consumer = if consumers.len() == 1 { let consumer = consumers.into_iter().next().unwrap(); @@ -449,15 +427,9 @@ impl ConsumerBuilder { #[cfg(test)] mod tests { - use std::{ - sync::{Arc, Mutex}, - time::Duration, - }; - - use futures::executor::block_on; + use std::{sync::Arc, time::Duration}; use super::*; - use crate::TokioExecutor; #[test] fn test_nack_delay_validation_oversized() { @@ -576,149 +548,4 @@ mod tests { ); } } - - fn nack_config_with_delay_and_backoff() -> ConsumerConfig { - ConsumerConfig { - nack_redelivery_delay: Some(Duration::from_secs(30)), - negative_ack_backoff: Some(Arc::new( - crate::consumer::negative_ack_backoff::MultiplierRedeliveryBackoff::default(), - )), - ..ConsumerConfig::default() - } - } - - fn broker_addr(port: u16) -> BrokerAddress { - BrokerAddress { - url: format!("pulsar://127.0.0.1:{port}").parse().unwrap(), - broker_url: format!("127.0.0.1:{port}"), - proxy: false, - } - } - - fn joined_topics(count: usize) -> Vec<(String, BrokerAddress)> { - (0..count) - .map(|idx| { - ( - format!("persistent://public/default/topic-{idx}"), - broker_addr(6650 + idx as u16), - ) - }) - .collect() - } - - fn assert_topic_consumer_receives_negative_ack_config( - passed_to_topic_consumer: &ConsumerConfig, - ) { - assert_eq!( - passed_to_topic_consumer.nack_redelivery_delay, - Some(Duration::from_secs(30)) - ); - assert!(passed_to_topic_consumer.negative_ack_backoff.is_some()); - } - - fn configs_received_by_topic_consumer_factory( - specs: Vec<(String, BrokerAddress, ConsumerConfig)>, - ) -> Vec { - let captured_configs = Arc::new(Mutex::new(Vec::new())); - let factory_configs = Arc::clone(&captured_configs); - - block_on(create_topic_consumers_with_factory( - (), - specs, - move |(), _topic, _addr, config| { - let factory_configs = Arc::clone(&factory_configs); - async move { - factory_configs.lock().unwrap().push(config); - Ok::<(), Error>(()) - } - }, - )) - .expect("test factory should not fail"); - - Arc::try_unwrap(captured_configs) - .expect("factory closure dropped") - .into_inner() - .unwrap() - } - - #[test] - fn single_topic_consumer_config_carries_nack_delay_to_engine() { - let config = nack_config_with_delay_and_backoff(); - let specs = ConsumerBuilder::::topic_consumer_specs_for_joined_topics( - &config, - joined_topics(1), - ); - let topic_consumer_configs = configs_received_by_topic_consumer_factory(specs); - - assert_eq!(topic_consumer_configs.len(), 1); - assert_topic_consumer_receives_negative_ack_config(&topic_consumer_configs[0]); - } - - #[test] - fn partitioned_consumer_config_carries_nack_delay_to_each_engine() { - let config = nack_config_with_delay_and_backoff(); - let specs = ConsumerBuilder::::topic_consumer_specs_for_joined_topics( - &config, - joined_topics(3), - ); - let topic_consumer_configs = configs_received_by_topic_consumer_factory(specs); - - assert_eq!(topic_consumer_configs.len(), 3); - for topic_consumer_config in &topic_consumer_configs { - assert_topic_consumer_receives_negative_ack_config(topic_consumer_config); - } - } - - #[test] - fn multi_topic_consumer_config_carries_nack_delay_to_each_engine() { - let config = nack_config_with_delay_and_backoff(); - let specs = ConsumerBuilder::::topic_consumer_specs_for_joined_topics( - &config, - joined_topics(2), - ); - let topic_consumer_configs = configs_received_by_topic_consumer_factory(specs); - - assert_eq!(topic_consumer_configs.len(), 2); - for topic_consumer_config in &topic_consumer_configs { - assert_topic_consumer_receives_negative_ack_config(topic_consumer_config); - } - } - - #[test] - fn reader_consumer_config_carries_nack_delay_to_engine() { - let config = nack_config_with_delay_and_backoff(); - let mut reader_config = config; - reader_config.options.durable = Some(false); - reader_config.sub_type = SubType::Exclusive; - - let (topic, addr) = joined_topics(1).pop().unwrap(); - let spec = ConsumerBuilder::::topic_consumer_spec_for_reader( - &reader_config, - topic, - addr, - ); - let mut topic_consumer_configs = configs_received_by_topic_consumer_factory(vec![spec]); - let topic_consumer_config = topic_consumer_configs.pop().unwrap(); - - assert_topic_consumer_receives_negative_ack_config(&topic_consumer_config); - assert_eq!(topic_consumer_config.options.durable, Some(false)); - assert_eq!(topic_consumer_config.sub_type, SubType::Exclusive); - } - - #[test] - fn build_path_config_helpers_clone_for_each_topic_consumer() { - let mut config = nack_config_with_delay_and_backoff(); - config.consumer_id = Some(7); - let specs = ConsumerBuilder::::topic_consumer_specs_for_joined_topics( - &config, - joined_topics(2), - ); - let topic_consumer_configs = configs_received_by_topic_consumer_factory(specs); - - assert_eq!(topic_consumer_configs.len(), 2); - assert!(topic_consumer_configs - .iter() - .all(|topic_consumer_config| topic_consumer_config.consumer_id == Some(7))); - } - } diff --git a/src/consumer/config.rs b/src/consumer/config.rs index 2dcab7bf..e1e4b7f8 100644 --- a/src/consumer/config.rs +++ b/src/consumer/config.rs @@ -35,14 +35,3 @@ pub struct ConsumerConfig { /// negative-ack redelivery backoff policy pub(crate) negative_ack_backoff: Option>, } - -impl ConsumerConfig { - /// Returns the exact configuration handed to a per-topic consumer engine. - /// - /// Keeping this as a named seam makes cross-flavor tests assert the same value that - /// `TopicConsumer::new` receives from single-topic, partitioned, multi-topic, regex-refresh, and - /// reader construction paths instead of only asserting arbitrary `ConsumerConfig` clones. - pub(crate) fn clone_for_topic_consumer(&self) -> Self { - self.clone() - } -} diff --git a/src/consumer/multi.rs b/src/consumer/multi.rs index 91178448..42340427 100644 --- a/src/consumer/multi.rs +++ b/src/consumer/multi.rs @@ -12,11 +12,7 @@ use futures::{future::try_join_all, Stream}; use regex::Regex; use crate::{ - consumer::{ - config::ConsumerConfig, - message::Message, - topic::{create_topic_consumers_with_factory, TopicConsumer}, - }, + consumer::{config::ConsumerConfig, message::Message, topic::TopicConsumer}, error::{ConnectionError, ConsumerError}, message::proto::{MessageIdData, Schema}, proto, @@ -45,21 +41,6 @@ pub struct MultiTopicConsumer { } impl MultiTopicConsumer { - fn topic_consumer_configs_for_refresh( - consumer_config: &ConsumerConfig, - topics: I, - existing_topics: &VecDeque, - ) -> Vec<(String, crate::BrokerAddress, ConsumerConfig)> - where - I: IntoIterator, - { - topics - .into_iter() - .filter(|(topic, _)| !existing_topics.contains(topic)) - .map(|(topic, addr)| (topic, addr, consumer_config.clone_for_topic_consumer())) - .collect() - } - #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn topics(&self) -> Vec { self.topics.iter().map(|s| s.to_string()).collect() @@ -203,17 +184,14 @@ impl MultiTopicConsumer { .into_iter() .flatten(); - let topic_consumer_specs = Self::topic_consumer_configs_for_refresh( - &consumer_config, - topics, - &existing_topics, - ); - // 3. create consumers - let consumers = create_topic_consumers_with_factory( - pulsar, - topic_consumer_specs, - TopicConsumer::new, + let consumers = try_join_all( + topics + .into_iter() + .filter(|(t, _)| !existing_topics.contains(t)) + .map(|(topic, addr)| { + TopicConsumer::new(pulsar.clone(), topic, addr, consumer_config.clone()) + }), ) .await?; if !consumers.is_empty() { @@ -426,101 +404,3 @@ impl Stream for MultiTopicConsum Poll::Pending } } - -#[cfg(test)] -mod tests { - use std::{ - collections::VecDeque, - sync::{Arc, Mutex}, - time::Duration, - }; - - use futures::executor::block_on; - - use super::*; - use crate::{ - consumer::negative_ack_backoff::MultiplierRedeliveryBackoff, BrokerAddress, TokioExecutor, - }; - - fn broker_addr(port: u16) -> BrokerAddress { - BrokerAddress { - url: format!("pulsar://127.0.0.1:{port}").parse().unwrap(), - broker_url: format!("127.0.0.1:{port}"), - proxy: false, - } - } - - fn nack_config_with_delay_and_backoff() -> ConsumerConfig { - ConsumerConfig { - nack_redelivery_delay: Some(Duration::from_secs(30)), - negative_ack_backoff: Some(Arc::new(MultiplierRedeliveryBackoff::default())), - ..ConsumerConfig::default() - } - } - - fn configs_received_by_topic_consumer_factory( - specs: Vec<(String, BrokerAddress, ConsumerConfig)>, - ) -> Vec<(String, ConsumerConfig)> { - let captured_configs = Arc::new(Mutex::new(Vec::new())); - let factory_configs = Arc::clone(&captured_configs); - - block_on(create_topic_consumers_with_factory( - (), - specs, - move |(), topic, _addr, config| { - let factory_configs = Arc::clone(&factory_configs); - async move { - factory_configs.lock().unwrap().push((topic, config)); - Ok::<(), Error>(()) - } - }, - )) - .expect("test factory should not fail"); - - Arc::try_unwrap(captured_configs) - .expect("factory closure dropped") - .into_inner() - .unwrap() - } - - #[test] - fn regex_refresh_config_carries_nack_delay_to_each_new_consumer() { - let config = nack_config_with_delay_and_backoff(); - let existing_topics = VecDeque::from(["persistent://public/default/existing".to_string()]); - let topics = vec![ - ( - "persistent://public/default/existing".to_string(), - broker_addr(6650), - ), - ( - "persistent://public/default/new-1".to_string(), - broker_addr(6651), - ), - ( - "persistent://public/default/new-2".to_string(), - broker_addr(6652), - ), - ]; - - let topic_consumer_specs = - MultiTopicConsumer::::topic_consumer_configs_for_refresh( - &config, - topics, - &existing_topics, - ); - - assert_eq!(topic_consumer_specs.len(), 2); - let topic_consumer_configs = - configs_received_by_topic_consumer_factory(topic_consumer_specs); - for (_, topic_consumer_config) in &topic_consumer_configs { - assert_eq!( - topic_consumer_config.nack_redelivery_delay, - Some(Duration::from_secs(30)) - ); - assert!(topic_consumer_config.negative_ack_backoff.is_some()); - } - assert!(topic_consumer_configs - .iter() - .all(|(topic, _)| topic != "persistent://public/default/existing")); - } -} diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index d2235468..bd96c75b 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -12,7 +12,6 @@ use std::{ use chrono::{DateTime, Utc}; use futures::{ channel::{mpsc, oneshot}, - future::try_join_all, FutureExt, SinkExt, Stream, StreamExt, }; @@ -33,24 +32,6 @@ use crate::{ BrokerAddress, DeserializeMessage, Error, Executor, Payload, Pulsar, }; -pub(crate) async fn create_topic_consumers_with_factory( - client: Client, - specs: Vec<(String, BrokerAddress, ConsumerConfig)>, - mut factory: F, -) -> Result, Error> -where - Client: Clone, - F: FnMut(Client, String, BrokerAddress, ConsumerConfig) -> Fut, - Fut: std::future::Future>, -{ - try_join_all( - specs - .into_iter() - .map(|(topic, addr, config)| factory(client.clone(), topic, addr, config)), - ) - .await -} - // this is entirely public for use in reader.rs pub struct TopicConsumer { pub(crate) consumer_id: u64, From f5b936b23bad6abbdd2c8f22f1470ae4a5d0ca2c Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 8 Jun 2026 16:12:13 -0300 Subject: [PATCH 50/66] refactor(06-02): trim negative ack public surface - Remove the extra Consumer::nack_redelivery_delay accessor - Drop the Debug bound from NegativeAckBackoff implementors - Add manual Debug output for internal config and tracker state Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/config.rs | 27 +++++++++++++++++++++++++-- src/consumer/mod.rs | 9 --------- src/consumer/negative_ack_backoff.rs | 3 +-- src/consumer/negative_ack_tracker.rs | 13 +++++++++++-- 4 files changed, 37 insertions(+), 15 deletions(-) diff --git a/src/consumer/config.rs b/src/consumer/config.rs index e1e4b7f8..4942cf27 100644 --- a/src/consumer/config.rs +++ b/src/consumer/config.rs @@ -1,4 +1,4 @@ -use std::{sync::Arc, time::Duration}; +use std::{fmt, sync::Arc, time::Duration}; use crate::{ consumer::{ @@ -8,7 +8,7 @@ use crate::{ }; /// the complete configuration of a consumer -#[derive(Debug, Clone, Default)] +#[derive(Clone, Default)] pub struct ConsumerConfig { /// subscription name pub(crate) subscription: String, @@ -35,3 +35,26 @@ pub struct ConsumerConfig { /// negative-ack redelivery backoff policy pub(crate) negative_ack_backoff: Option>, } + +impl fmt::Debug for ConsumerConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ConsumerConfig") + .field("subscription", &self.subscription) + .field("sub_type", &self.sub_type) + .field("batch_size", &self.batch_size) + .field("consumer_name", &self.consumer_name) + .field("consumer_id", &self.consumer_id) + .field( + "unacked_message_redelivery_delay", + &self.unacked_message_redelivery_delay, + ) + .field("options", &self.options) + .field("dead_letter_policy", &self.dead_letter_policy) + .field("nack_redelivery_delay", &self.nack_redelivery_delay) + .field( + "negative_ack_backoff", + &self.negative_ack_backoff.as_ref().map(|_| "configured"), + ) + .finish() + } +} diff --git a/src/consumer/mod.rs b/src/consumer/mod.rs index f502dddf..9f3e9d05 100644 --- a/src/consumer/mod.rs +++ b/src/consumer/mod.rs @@ -383,15 +383,6 @@ impl Consumer { } } - /// Returns the configured fixed negative-ack redelivery delay, or None if not set. - #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] - pub fn nack_redelivery_delay(&self) -> Option { - match &self.inner { - InnerConsumer::Single(c) => c.config.nack_redelivery_delay, - InnerConsumer::Multi(c) => c.config.nack_redelivery_delay, - } - } - /// returns the date of the last message reception #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn last_message_received(&self) -> Option> { diff --git a/src/consumer/negative_ack_backoff.rs b/src/consumer/negative_ack_backoff.rs index 2bcc334c..bb97b22f 100644 --- a/src/consumer/negative_ack_backoff.rs +++ b/src/consumer/negative_ack_backoff.rs @@ -9,7 +9,7 @@ const MULTIPLIER_MUST_BE_FINITE: &str = "multiplier must be finite"; const DELAY_DURATION_TOO_LARGE: &str = "delay duration is too large"; /// Policy for computing a negative-acknowledgment redelivery delay from a redelivery count. -pub trait NegativeAckBackoff: Send + Sync + std::fmt::Debug { +pub trait NegativeAckBackoff: Send + Sync { /// Returns the delay to use for the provided redelivery count. fn next(&self, redelivery_count: u32) -> Duration; } @@ -250,7 +250,6 @@ mod tests { #[test] fn custom_negative_ack_backoff_can_be_implemented_without_engine_dependencies() { - #[derive(Debug)] struct FixedBackoff(Duration); impl NegativeAckBackoff for FixedBackoff { diff --git a/src/consumer/negative_ack_tracker.rs b/src/consumer/negative_ack_tracker.rs index a85f91e6..1e8cea00 100644 --- a/src/consumer/negative_ack_tracker.rs +++ b/src/consumer/negative_ack_tracker.rs @@ -1,5 +1,6 @@ use std::{ collections::{BTreeSet, HashMap}, + fmt, sync::Arc, time::{Duration, Instant}, }; @@ -23,13 +24,22 @@ pub(crate) enum NegativeAckDispatchState { InFlight, } -#[derive(Debug)] pub(crate) struct NegativeAckTracker { fixed_delay: Duration, backoff: Option>, entries: HashMap, } +impl fmt::Debug for NegativeAckTracker { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("NegativeAckTracker") + .field("fixed_delay", &self.fixed_delay) + .field("entries", &self.entries) + .field("has_backoff", &self.backoff.is_some()) + .finish() + } +} + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(crate) struct NormalizedMessageId { ledger_id: u64, @@ -332,7 +342,6 @@ impl NegativeAckTracker { mod tests { use super::*; - #[derive(Debug)] struct FixedBackoff(Duration); impl NegativeAckBackoff for FixedBackoff { From 4e28383c73dd130534325fc8eba9ed6df52e7792 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 8 Jun 2026 16:19:52 -0300 Subject: [PATCH 51/66] refactor(06-03): trim lifecycle close path to nack cleanup - Remove internal engine close message and closed flag - Restore TopicConsumer close to direct broker close - Revert unacked ticker stop/drop lifecycle redesign while keeping nack cleanup Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/data.rs | 1 - src/consumer/engine.rs | 18 ------------ src/consumer/topic.rs | 67 ++++++------------------------------------ 3 files changed, 9 insertions(+), 77 deletions(-) diff --git a/src/consumer/data.rs b/src/consumer/data.rs index 1721be8c..02ebbf1b 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -36,7 +36,6 @@ pub(crate) enum InternalEngineMessage { NegativeAckRedelivery, UnackedRedelivery, GetConnection(oneshot::Sender>>), - Close(oneshot::Sender>), } #[derive(Clone, Debug, PartialEq, Eq, Hash)] diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index 6125b146..58d11488 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -166,7 +166,6 @@ pub struct ConsumerEngine { negative_ack_due_event_pending: Arc, dead_letter_policy: Option, options: ConsumerOptions, - closed: bool, } impl ConsumerEngine { @@ -214,7 +213,6 @@ impl ConsumerEngine { negative_ack_due_event_pending: Arc::new(AtomicBool::new(false)), dead_letter_policy, options, - closed: false, } } @@ -455,18 +453,6 @@ impl ConsumerEngine { }); true } - Some(InternalEngineMessage::Close(sender)) => { - self.clear_negative_ack_state(); - let res = self - .connection - .sender() - .close_consumer(self.id) - .await - .map(|_| ()); - self.closed = true; - let _ = sender.send(res); - false - } } } @@ -1136,10 +1122,6 @@ impl std::ops::Drop for ConsumerEngine { fn drop(&mut self) { self.clear_negative_ack_state(); - if self.closed { - return; - } - let conn = self.connection.clone(); let id = self.id; let name = self.name.clone(); diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index bd96c75b..f71dbe36 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -2,7 +2,7 @@ use std::{ marker::PhantomData, pin::Pin, sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, + atomic::{AtomicU64, Ordering}, Arc, }, task::{Context, Poll}, @@ -39,7 +39,6 @@ pub struct TopicConsumer { topic: String, messages: Pin>, engine_tx: mpsc::UnboundedSender>, - unacked_redelivery_ticker_running: Option>, data_type: PhantomData T::Output>, pub(crate) dead_letter_policy: Option, pub(super) last_message_received: Option>, @@ -89,17 +88,11 @@ impl TopicConsumer { let (engine_tx, engine_rx) = mpsc::unbounded(); - let unacked_redelivery_ticker_running = if unacked_message_redelivery_delay.is_some() { - let ticker_running = Arc::new(AtomicBool::new(true)); - let ticker_running_task = ticker_running.clone(); + if unacked_message_redelivery_delay.is_some() { let mut redelivery_tx = engine_tx.clone(); let mut interval = client.executor.interval(Duration::from_millis(500)); let res = client.executor.spawn(Box::pin(async move { while interval.next().await.is_some() { - if !ticker_running_task.load(Ordering::SeqCst) { - break; - } - if redelivery_tx .send(InternalEngineMessage::UnackedRedelivery) .await @@ -113,10 +106,7 @@ impl TopicConsumer { if res.is_err() { return Err(Error::Executor); } - Some(ticker_running) - } else { - None - }; + } let receiver_queue_size = options.receiver_queue_size.unwrap_or(1000); let (tx, rx) = mpsc::channel(receiver_queue_size as usize); let mut c = ConsumerEngine::new( @@ -154,7 +144,6 @@ impl TopicConsumer { topic, messages: Box::pin(rx), engine_tx, - unacked_redelivery_ticker_running, data_type: PhantomData, dead_letter_policy, last_message_received: None, @@ -284,24 +273,15 @@ impl TopicConsumer { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn close(&mut self) -> Result<(), Error> { - let (resolver, response) = oneshot::channel(); - self.stop_unacked_redelivery_ticker(); - self.engine_tx - .send(InternalEngineMessage::Close(resolver)) - .await - .map_err(ConsumerError::from)?; - - response.await.map_err(|_| ConsumerError::Closed)??; + let consumer_id = self.consumer_id; + self.connection() + .await? + .sender() + .close_consumer(consumer_id) + .await?; Ok(()) } - fn stop_unacked_redelivery_ticker(&mut self) { - if let Some(ticker_running) = &self.unacked_redelivery_ticker_running { - ticker_running.store(false, Ordering::SeqCst); - } - self.unacked_redelivery_ticker_running = None; - } - #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn get_last_message_id(&mut self) -> Result { let consumer_id = self.consumer_id; @@ -350,12 +330,6 @@ impl TopicConsumer { } } -impl Drop for TopicConsumer { - fn drop(&mut self) { - self.stop_unacked_redelivery_ticker(); - } -} - impl Stream for TopicConsumer { type Item = Result, Error>; @@ -391,27 +365,4 @@ mod tests { assert!(source_contains(&["msg.", "broker_redelivery_count()"])); assert!(!source_contains(&["Some(msg.", "redelivery_count())"])); } - - #[test] - fn drop_stops_unacked_redelivery_ticker_sender() { - assert!(source_contains(&[ - "unacked_redelivery_ticker_running:", - " Option>" - ])); - let source = include_str!("topic.rs"); - assert!(source.contains("while interval.next().await.is_some()")); - assert!(source.contains("if !ticker_running_task.load(Ordering::SeqCst)")); - assert!(!source_contains(&[ - "while ticker_running_task.", - "load(Ordering::SeqCst) && interval.next().await.is_some()" - ])); - assert!(source_contains(&[ - "impl Drop", - " for TopicConsumer" - ])); - assert!(source_contains(&[ - "self.", - "stop_unacked_redelivery_ticker()" - ])); - } } From f1c6b0da6df3195d76f7a781332b8edeaa58377e Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 8 Jun 2026 16:22:03 -0300 Subject: [PATCH 52/66] refactor(06-03): narrow ack cleanup to nack cancellation - Restore direct ack sender flow and exact unacked cleanup - Keep negative-ack tracker cancellation for acked entries - Narrow engine test seam to negative-ack redelivery dispatch only Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/engine.rs | 368 ++++++++--------------------------------- 1 file changed, 70 insertions(+), 298 deletions(-) diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index 58d11488..58439c17 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -43,30 +43,20 @@ const SYSTEM_PROPERTY_REAL_TOPIC: &str = "REAL_TOPIC"; const SYSTEM_PROPERTY_ORIGIN_MESSAGE_ID: &str = "ORIGIN_MESSAGE_ID"; #[async_trait::async_trait] -trait ConsumerEngineEffects { +trait NegativeAckRedeliveryRequester { async fn request_negative_ack_redelivery( &mut self, consumer_id: u64, ids: Vec, ) -> Result<(), ConnectionError>; - - async fn send_ack( - &mut self, - consumer_id: u64, - ids: Vec, - cumulative: bool, - ) -> Result<(), ConnectionError>; - - async fn send_dead_letter(&mut self, topic: &str, message: Message) -> Result<(), Error>; } -struct BrokerConsumerEngineEffects { - client: Pulsar, +struct BrokerNegativeAckRedeliveryRequester { connection: Arc>, } #[async_trait::async_trait] -impl ConsumerEngineEffects for BrokerConsumerEngineEffects { +impl NegativeAckRedeliveryRequester for BrokerNegativeAckRedeliveryRequester { async fn request_negative_ack_redelivery( &mut self, consumer_id: u64, @@ -77,60 +67,36 @@ impl ConsumerEngineEffects for BrokerConsumerEngineEffects { .send_redeliver_unacknowleged_messages(consumer_id, ids) .await } - - async fn send_ack( - &mut self, - consumer_id: u64, - ids: Vec, - cumulative: bool, - ) -> Result<(), ConnectionError> { - self.connection - .sender() - .send_ack(consumer_id, ids, cumulative) - .await - } - - async fn send_dead_letter(&mut self, topic: &str, message: Message) -> Result<(), Error> { - self.client - .send(topic, message) - .await? - .await - .map(|_| ()) - .map_err(|e| { - error!("One shot cancelled {:?}", e); - Error::Custom("DLQ send error".to_string()) - }) - } } -async fn send_negative_ack_redelivery_with_effects( - effects: &mut E, +async fn send_negative_ack_redelivery_with_requester( + requester: &mut R, consumer_id: u64, ids: Vec, ) -> Result<(), ConnectionError> where - E: ConsumerEngineEffects + ?Sized, + R: NegativeAckRedeliveryRequester + ?Sized, { let ids = ids .into_iter() .map(|id| NegativeAckTracker::redelivery_message_id(&id)) .collect(); - effects + requester .request_negative_ack_redelivery(consumer_id, ids) .await } -async fn dispatch_negative_ack_redelivery_with_effects( +async fn dispatch_negative_ack_redelivery_with_requester( tracker: &mut NegativeAckTracker, - effects: &mut E, + requester: &mut R, consumer_id: u64, ids: &[MessageIdData], retry_at: Instant, ) -> Result<(), ConnectionError> where - E: ConsumerEngineEffects + ?Sized, + R: NegativeAckRedeliveryRequester + ?Sized, { - match send_negative_ack_redelivery_with_effects(effects, consumer_id, ids.to_vec()).await { + match send_negative_ack_redelivery_with_requester(requester, consumer_id, ids.to_vec()).await { Ok(()) => { tracker.mark_dispatched(ids); Ok(()) @@ -572,15 +538,14 @@ impl ConsumerEngine { match schedule { NegativeAckSchedule::Immediate => { - let mut effects = BrokerConsumerEngineEffects { - client: self.client.clone(), + let mut requester = BrokerNegativeAckRedeliveryRequester { connection: self.connection.clone(), }; let consumer_id = self.id; let retry_at = Instant::now() + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL; - match dispatch_negative_ack_redelivery_with_effects( + match dispatch_negative_ack_redelivery_with_requester( self.ensure_negative_ack_tracker(), - &mut effects, + &mut requester, consumer_id, std::slice::from_ref(&message_id), retry_at, @@ -597,14 +562,13 @@ impl ConsumerEngine { ); if self.start_negative_ack_ticker().is_err() { error!("could not start negative ack redelivery ticker after immediate redelivery failure"); - let mut effects = BrokerConsumerEngineEffects { - client: self.client.clone(), + let mut requester = BrokerNegativeAckRedeliveryRequester { connection: self.connection.clone(), }; let retry_at = Instant::now() + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL; - if let Err(e) = dispatch_negative_ack_redelivery_with_effects( + if let Err(e) = dispatch_negative_ack_redelivery_with_requester( self.ensure_negative_ack_tracker(), - &mut effects, + &mut requester, consumer_id, std::slice::from_ref(&message_id), retry_at, @@ -624,15 +588,14 @@ impl ConsumerEngine { NegativeAckSchedule::Scheduled | NegativeAckSchedule::DuplicateEarlier => { if self.start_negative_ack_ticker().is_err() { error!("could not start negative ack redelivery ticker"); - let mut effects = BrokerConsumerEngineEffects { - client: self.client.clone(), + let mut requester = BrokerNegativeAckRedeliveryRequester { connection: self.connection.clone(), }; let consumer_id = self.id; let retry_at = Instant::now() + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL; - if let Err(e) = dispatch_negative_ack_redelivery_with_effects( + if let Err(e) = dispatch_negative_ack_redelivery_with_requester( self.ensure_negative_ack_tracker(), - &mut effects, + &mut requester, consumer_id, std::slice::from_ref(&message_id), retry_at, @@ -670,16 +633,15 @@ impl ConsumerEngine { return; } - let mut effects = BrokerConsumerEngineEffects { - client: self.client.clone(), + let mut requester = BrokerNegativeAckRedeliveryRequester { connection: self.connection.clone(), }; let consumer_id = self.id; let retry_at = Instant::now() + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL; if let Some(tracker) = self.negative_ack_tracker.as_mut() { - if let Err(e) = dispatch_negative_ack_redelivery_with_effects( + if let Err(e) = dispatch_negative_ack_redelivery_with_requester( tracker, - &mut effects, + &mut requester, consumer_id, &ids, retry_at, @@ -695,87 +657,24 @@ impl ConsumerEngine { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] async fn ack(&mut self, message_id: MessageIdData, cumulative: bool) { - let mut effects = BrokerConsumerEngineEffects { - client: self.client.clone(), - connection: self.connection.clone(), - }; - - match Self::send_ack_and_clean_local_state( - &mut effects, - self.id, - &mut self.unacked_messages, - self.negative_ack_tracker.as_mut(), - message_id, - cumulative, - ) - .await - { - Ok(()) => { - self.stop_negative_ack_ticker_if_idle(); - } - Err(e) => { - error!("ack error: {:?}", e); - } - } - } - - async fn send_ack_and_clean_local_state( - effects: &mut E, - consumer_id: u64, - unacked_messages: &mut HashMap, - negative_ack_tracker: Option<&mut NegativeAckTracker>, - message_id: MessageIdData, - cumulative: bool, - ) -> Result<(), ConnectionError> - where - E: ConsumerEngineEffects + ?Sized, - { - effects - .send_ack(consumer_id, vec![message_id.clone()], cumulative) - .await?; - - Self::remove_ack_from_unacked_messages(unacked_messages, &message_id, cumulative); - if let Some(tracker) = negative_ack_tracker { + // FIXME: this does not handle cumulative acks + self.unacked_messages.remove(&message_id); + if let Some(tracker) = self.negative_ack_tracker.as_mut() { if cumulative { tracker.cancel_cumulative_ack(&message_id); } else { tracker.cancel_ack(&message_id); } } - - Ok(()) - } - - fn remove_ack_from_unacked_messages( - unacked_messages: &mut HashMap, - message_id: &MessageIdData, - cumulative: bool, - ) { - if !cumulative { - unacked_messages.remove(message_id); - return; + self.stop_negative_ack_ticker_if_idle(); + let res = self + .connection + .sender() + .send_ack(self.id, vec![message_id], cumulative) + .await; + if res.is_err() { + error!("ack error: {:?}", res); } - - unacked_messages.retain(|id, _| { - if id.partition != message_id.partition { - return true; - } - - if id.ledger_id < message_id.ledger_id - || (id.ledger_id == message_id.ledger_id && id.entry_id < message_id.entry_id) - { - return false; - } - - if id.ledger_id == message_id.ledger_id && id.entry_id == message_id.entry_id { - return match message_id.batch_index { - Some(ack_idx) => id.batch_index.map_or(true, |idx| idx > ack_idx), - None => false, - }; - } - - true - }); } /// Process the message. Returns `true` if there are more messages to process @@ -1008,12 +907,9 @@ impl ConsumerEngine { event_time: metadata.event_time, ..Default::default() }; - let mut effects = BrokerConsumerEngineEffects { - client: self.client.clone(), - connection: self.connection.clone(), - }; - effects - .send_dead_letter(&dead_letter_policy.dead_letter_topic, message) + self.client + .send(&dead_letter_policy.dead_letter_topic, message) + .await? .await?; self.ack(message_id, false).await; @@ -1139,10 +1035,7 @@ impl std::ops::Drop for ConsumerEngine { #[cfg(test)] mod tests { - use std::{ - collections::{HashMap, HashSet}, - time::{Duration, Instant}, - }; + use std::time::{Duration, Instant}; use futures::executor::block_on; @@ -1162,22 +1055,22 @@ mod tests { let now = Instant::now(); let id = message_id(1, 2, Some(0)); let mut tracker = NegativeAckTracker::new(Some(Duration::ZERO), None); - let mut effects = FakeNegativeAckEffects::default(); + let mut requester = FakeNegativeAckRequester::default(); assert_eq!( tracker.schedule(id.clone(), Some(1), now), NegativeAckSchedule::Immediate ); - block_on(dispatch_negative_ack_redelivery_with_effects( + block_on(dispatch_negative_ack_redelivery_with_requester( &mut tracker, - &mut effects, + &mut requester, 7, std::slice::from_ref(&id), now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, )) .unwrap(); - assert_eq!(effects.redelivery_requests, vec![message_id(1, 2, None)]); + assert_eq!(requester.redelivery_requests, vec![message_id(1, 2, None)]); assert!(tracker .collect_due(now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL) .is_empty()); @@ -1199,9 +1092,9 @@ mod tests { let now = Instant::now(); let id = message_id(1, 2, Some(3)); let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(5)), None); - let mut effects = FakeNegativeAckEffects { + let mut requester = FakeNegativeAckRequester { fail_redelivery: true, - ..FakeNegativeAckEffects::default() + ..FakeNegativeAckRequester::default() }; assert_eq!( @@ -1211,16 +1104,16 @@ mod tests { let due = tracker.collect_due(now + Duration::from_secs(5)); assert_eq!(due, vec![message_id(1, 2, None)]); - let result = block_on(dispatch_negative_ack_redelivery_with_effects( + let result = block_on(dispatch_negative_ack_redelivery_with_requester( &mut tracker, - &mut effects, + &mut requester, 7, &due, now + Duration::from_secs(5) + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, )); assert!(matches!(result, Err(ConnectionError::Disconnected))); - assert_eq!(effects.redelivery_requests, vec![message_id(1, 2, None)]); + assert_eq!(requester.redelivery_requests, vec![message_id(1, 2, None)]); let retried = tracker .collect_due(now + Duration::from_secs(5) + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL); assert_eq!(retried, vec![message_id(1, 2, None)]); @@ -1231,25 +1124,25 @@ mod tests { let now = Instant::now(); let id = message_id(5, 6, None); let mut tracker = NegativeAckTracker::new(Some(Duration::ZERO), None); - let mut effects = FakeNegativeAckEffects { + let mut requester = FakeNegativeAckRequester { fail_redelivery: true, - ..FakeNegativeAckEffects::default() + ..FakeNegativeAckRequester::default() }; assert_eq!( tracker.schedule(id.clone(), Some(99), now), NegativeAckSchedule::Immediate ); - let result = block_on(dispatch_negative_ack_redelivery_with_effects( + let result = block_on(dispatch_negative_ack_redelivery_with_requester( &mut tracker, - &mut effects, + &mut requester, 7, std::slice::from_ref(&id), now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, )); assert!(matches!(result, Err(ConnectionError::Disconnected))); - assert_eq!(effects.redelivery_requests, vec![id]); + assert_eq!(requester.redelivery_requests, vec![id]); let retried = tracker.collect_due(now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL); assert_eq!(retried, vec![message_id(5, 6, None)]); } @@ -1270,105 +1163,13 @@ mod tests { } #[test] - fn ack_and_close_paths_cancel_or_clear_negative_ack_tracker() { + fn ack_and_drop_paths_cancel_or_clear_negative_ack_tracker() { assert!(source_contains(&["cancel_", "ack(&message_id)"])); assert!(source_contains(&["cancel_", "cumulative_ack(&message_id)"])); assert!(source_contains(&["tracker.", "clear()"])); assert!(source_contains(&["ticker_running.", "store(false"])); } - #[test] - fn ack_cleans_local_state_only_after_broker_ack_succeeds() { - let now = Instant::now(); - let id = message_id(1, 2, None); - let mut unacked_messages = HashMap::from([(id.clone(), now)]); - let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(5)), None); - assert_eq!( - tracker.schedule(id.clone(), Some(1), now), - NegativeAckSchedule::Scheduled - ); - let mut failing_effects = FakeNegativeAckEffects { - fail_ack: true, - ..FakeNegativeAckEffects::default() - }; - - let result = block_on( - ConsumerEngine::::send_ack_and_clean_local_state( - &mut failing_effects, - 7, - &mut unacked_messages, - Some(&mut tracker), - id.clone(), - false, - ), - ); - - assert!(matches!(result, Err(ConnectionError::Disconnected))); - assert_eq!( - failing_effects.ack_requests, - vec![(vec![id.clone()], false)] - ); - assert!(unacked_messages.contains_key(&id)); - assert!(tracker.next_due_time().is_some()); - - let mut successful_effects = FakeNegativeAckEffects::default(); - block_on( - ConsumerEngine::::send_ack_and_clean_local_state( - &mut successful_effects, - 7, - &mut unacked_messages, - Some(&mut tracker), - id.clone(), - false, - ), - ) - .unwrap(); - - assert_eq!( - successful_effects.ack_requests, - vec![(vec![id.clone()], false)] - ); - assert!(!unacked_messages.contains_key(&id)); - assert!(tracker.next_due_time().is_none()); - } - - #[test] - fn cumulative_ack_clears_prior_unacked_redelivery_entries() { - let due_at = Instant::now() - Duration::from_secs(1); - let mut unacked_messages = HashMap::from([ - (message_id(1, 1, None), due_at), - (message_id(1, 2, Some(0)), due_at), - (message_id(1, 2, Some(1)), due_at), - (message_id(1, 2, Some(2)), due_at), - (message_id(1, 2, None), due_at), - (message_id(1, 3, None), due_at), - (message_id_for_partition(1, 1, Some(1), None), due_at), - ]); - - ConsumerEngine::::remove_ack_from_unacked_messages( - &mut unacked_messages, - &message_id(1, 2, Some(1)), - true, - ); - - assert!(!unacked_messages.contains_key(&message_id(1, 1, None))); - assert!(!unacked_messages.contains_key(&message_id(1, 2, Some(0)))); - assert!(!unacked_messages.contains_key(&message_id(1, 2, Some(1)))); - assert!(unacked_messages.contains_key(&message_id(1, 2, Some(2)))); - assert!(unacked_messages.contains_key(&message_id(1, 2, None))); - assert!(unacked_messages.contains_key(&message_id(1, 3, None))); - assert!(unacked_messages.contains_key(&message_id_for_partition(1, 1, Some(1), None))); - - let now = Instant::now(); - let due_for_redelivery: HashSet<_> = unacked_messages - .iter() - .filter_map(|(id, redeliver_at)| (*redeliver_at < now).then_some(id.clone())) - .collect(); - assert!(!due_for_redelivery.contains(&message_id(1, 1, None))); - assert!(!due_for_redelivery.contains(&message_id(1, 2, Some(0)))); - assert!(!due_for_redelivery.contains(&message_id(1, 2, Some(1)))); - } - #[test] fn reconnect_preserves_negative_ack_tracker_state() { let source = include_str!("engine.rs"); @@ -1383,16 +1184,13 @@ mod tests { } #[derive(Default)] - struct FakeNegativeAckEffects { + struct FakeNegativeAckRequester { redelivery_requests: Vec, - ack_requests: Vec<(Vec, bool)>, - dlq_sends: usize, fail_redelivery: bool, - fail_ack: bool, } #[async_trait::async_trait] - impl ConsumerEngineEffects for FakeNegativeAckEffects { + impl NegativeAckRedeliveryRequester for FakeNegativeAckRequester { async fn request_negative_ack_redelivery( &mut self, _consumer_id: u64, @@ -1405,29 +1203,6 @@ mod tests { Ok(()) } } - - async fn send_ack( - &mut self, - _consumer_id: u64, - ids: Vec, - cumulative: bool, - ) -> Result<(), ConnectionError> { - self.ack_requests.push((ids, cumulative)); - if self.fail_ack { - Err(ConnectionError::Disconnected) - } else { - Ok(()) - } - } - - async fn send_dead_letter( - &mut self, - _topic: &str, - _message: crate::producer::Message, - ) -> Result<(), Error> { - self.dlq_sends += 1; - Ok(()) - } } fn message_id(ledger_id: u64, entry_id: u64, batch_index: Option) -> MessageIdData { @@ -1452,38 +1227,37 @@ mod tests { } #[test] - fn immediate_negative_ack_requests_broker_redelivery_without_dlq_send() { + fn immediate_negative_ack_requests_broker_redelivery_only() { let now = Instant::now(); let id = message_id(1, 2, None); let mut tracker = NegativeAckTracker::new(Some(Duration::ZERO), None); - let mut effects = FakeNegativeAckEffects::default(); + let mut requester = FakeNegativeAckRequester::default(); assert_eq!( tracker.schedule(id.clone(), Some(99), now), NegativeAckSchedule::Immediate ); - block_on(dispatch_negative_ack_redelivery_with_effects( + block_on(dispatch_negative_ack_redelivery_with_requester( &mut tracker, - &mut effects, + &mut requester, 7, std::slice::from_ref(&id), now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, )) .unwrap(); - assert_eq!(effects.redelivery_requests, vec![id.clone()]); - assert_eq!(effects.dlq_sends, 0); + assert_eq!(requester.redelivery_requests, vec![id.clone()]); assert!(tracker .collect_due(now + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL) .is_empty()); } #[test] - fn delayed_negative_ack_requests_broker_redelivery_when_due_without_dlq_send() { + fn delayed_negative_ack_requests_broker_redelivery_when_due() { let now = Instant::now(); let id = message_id(1, 2, Some(3)); let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(5)), None); - let mut effects = FakeNegativeAckEffects::default(); + let mut requester = FakeNegativeAckRequester::default(); assert_eq!( tracker.schedule(id.clone(), Some(99), now), @@ -1492,28 +1266,27 @@ mod tests { assert!(tracker.collect_due(now + Duration::from_secs(4)).is_empty()); let due = tracker.collect_due(now + Duration::from_secs(5)); - block_on(dispatch_negative_ack_redelivery_with_effects( + block_on(dispatch_negative_ack_redelivery_with_requester( &mut tracker, - &mut effects, + &mut requester, 7, &due, now + Duration::from_secs(5) + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, )) .unwrap(); - assert_eq!(effects.redelivery_requests, vec![message_id(1, 2, None)]); - assert_eq!(effects.dlq_sends, 0); + assert_eq!(requester.redelivery_requests, vec![message_id(1, 2, None)]); assert!(tracker .collect_due(now + Duration::from_secs(5) + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL) .is_empty()); } #[test] - fn id_only_negative_ack_uses_fixed_delay_behavior_without_dlq_send() { + fn id_only_negative_ack_uses_fixed_delay_behavior() { let now = Instant::now(); let id = message_id(3, 4, None); let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(7)), None); - let mut effects = FakeNegativeAckEffects::default(); + let mut requester = FakeNegativeAckRequester::default(); assert_eq!( tracker.schedule(id.clone(), None, now), @@ -1521,17 +1294,16 @@ mod tests { ); let due = tracker.collect_due(now + Duration::from_secs(7)); - block_on(dispatch_negative_ack_redelivery_with_effects( + block_on(dispatch_negative_ack_redelivery_with_requester( &mut tracker, - &mut effects, + &mut requester, 7, &due, now + Duration::from_secs(7) + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL, )) .unwrap(); - assert_eq!(effects.redelivery_requests, vec![id.clone()]); - assert_eq!(effects.dlq_sends, 0); + assert_eq!(requester.redelivery_requests, vec![id.clone()]); assert!(tracker .collect_due(now + Duration::from_secs(7) + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL) .is_empty()); From 35f1641616d606483fa48f61247f7965e36e0f9b Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 8 Jun 2026 16:28:03 -0300 Subject: [PATCH 53/66] test(06-04): make broker DLQ tests use short nack delay - Configure DLQ consumers with an explicit 1ms nack redelivery delay - Remove the 60s default-derived broker receive timeout helper - Keep the default delay proof at the tracker unit-test layer Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/mod.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/consumer/mod.rs b/src/consumer/mod.rs index 9f3e9d05..d63b655e 100644 --- a/src/consumer/mod.rs +++ b/src/consumer/mod.rs @@ -449,7 +449,7 @@ mod tests { ))] use tokio::time::timeout; - use super::{negative_ack_tracker::DEFAULT_NACK_REDELIVERY_DELAY, *}; + use super::*; #[cfg(any( feature = "tokio-runtime", feature = "tokio-rustls-runtime-aws-lc-rs", @@ -495,10 +495,6 @@ mod tests { const DEFAULT_RECV_TIMEOUT: Duration = Duration::from_secs(5); - fn default_nack_redelivery_recv_timeout() -> Duration { - DEFAULT_NACK_REDELIVERY_DELAY + DEFAULT_RECV_TIMEOUT - } - async fn recv_within( c: &mut Consumer, dur: Duration, @@ -895,6 +891,7 @@ mod tests { .with_subscription("nack") .with_subscription_type(SubType::Shared) .with_dead_letter_policy(dead_letter_policy) + .with_nack_redelivery_delay(Duration::from_millis(1)) .build() .await .unwrap(); @@ -930,7 +927,7 @@ mod tests { // Nacking message to send it to DLQ consumer.nack(&msg).await.unwrap(); - let dlq_msg = recv_within(&mut dlq_consumer, default_nack_redelivery_recv_timeout()) + let dlq_msg = recv_within(&mut dlq_consumer, DEFAULT_RECV_TIMEOUT) .await .unwrap(); @@ -1014,6 +1011,7 @@ mod tests { .with_subscription("nack") .with_subscription_type(SubType::Shared) .with_dead_letter_policy(dead_letter_policy) + .with_nack_redelivery_delay(Duration::from_millis(1)) .build() .await .unwrap(); @@ -1105,7 +1103,7 @@ mod tests { // Nacking message to send it to DLQ consumer.nack(&msg).await.unwrap(); - let dlq_msg = recv_within(&mut dlq_consumer, default_nack_redelivery_recv_timeout()) + let dlq_msg = recv_within(&mut dlq_consumer, DEFAULT_RECV_TIMEOUT) .await .unwrap(); println!("got message: {:?}", dlq_msg.payload); From 7a461bc510b2901a9c60562fc2cfa0618a61d6c2 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 8 Jun 2026 16:29:22 -0300 Subject: [PATCH 54/66] test(06-04): remove consumer source-string assertions - Delete include_str-based implementation shape tests from consumer modules - Keep behavior and type-level tests for nack message and engine behavior - Leave PR-scope evidence in the upstream-check artifact Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 26 --------------------- src/consumer/data.rs | 23 ------------------ src/consumer/engine.rs | 52 ----------------------------------------- src/consumer/topic.rs | 16 ------------- 4 files changed, 117 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index aa3c9bc7..21bab94d 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -522,30 +522,4 @@ mod tests { assert!(config.nack_redelivery_delay.is_none()); assert!(config.negative_ack_backoff.is_some()); } - - #[test] - fn negative_ack_runtime_docs_cover_phase4_semantics() { - let source = include_str!("builder.rs"); - let docs_source = source - .split("#[cfg(test)]") - .next() - .expect("builder source contains test module marker"); - - for expected in [ - "60 seconds", - "zero delay", - "immediate redelivery", - "message redelivery count", - "ID-only", - "fixed fallback", - "batch-entry normalization", - "best-effort in-memory timer", - "no retry-letter/DLQ redesign", - ] { - assert!( - docs_source.contains(expected), - "missing negative-ack docs phrase: {expected}" - ); - } - } } diff --git a/src/consumer/data.rs b/src/consumer/data.rs index 02ebbf1b..a4b0f936 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -138,27 +138,4 @@ mod tests { let _: Option = None::>; } - - #[test] - fn topic_consumer_passes_nack_config_to_consumer_engine_new() { - let source = include_str!("topic.rs"); - - assert!(!source.contains("_nack_redelivery_delay")); - assert!(!source.contains("_negative_ack_backoff")); - assert!(source.contains("nack_redelivery_delay,")); - assert!(source.contains("negative_ack_backoff,")); - assert!(source.contains("ConsumerEngine::new(")); - } - - #[test] - fn reader_and_multi_topic_consumers_keep_topic_consumer_construction_path() { - let client_source = include_str!("../client.rs"); - let builder_source = include_str!("builder.rs"); - let multi_source = include_str!("multi.rs"); - - assert!(client_source.contains("pub fn reader(&self) -> ConsumerBuilder")); - assert!(client_source.contains("ConsumerBuilder::new(self)")); - assert!(builder_source.contains("TopicConsumer::new")); - assert!(multi_source.contains("TopicConsumer::new")); - } } diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index 58439c17..ee1460fc 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -1045,11 +1045,6 @@ mod tests { message::proto::MessageIdData, }; - fn source_contains(parts: &[&str]) -> bool { - let pattern = parts.concat(); - include_str!("engine.rs").contains(&pattern) - } - #[test] fn nack_handling_uses_tracker_with_zero_delay_immediate_redelivery() { let now = Instant::now(); @@ -1076,17 +1071,6 @@ mod tests { .is_empty()); } - #[test] - fn lazy_ticker_coalesces_due_events() { - assert!(source_contains(&["start_", "negative_ack_ticker"])); - assert!(source_contains(&["compare_", "exchange(false, true"])); - assert!(source_contains(&["NegativeAck", "Redelivery"])); - assert!(source_contains(&[ - "NEGATIVE_ACK_", - "REDELIVERY_TICK_INTERVAL", - ])); - } - #[test] fn due_redelivery_dispatch_is_in_flight_retry_safe() { let now = Instant::now(); @@ -1147,42 +1131,6 @@ mod tests { assert_eq!(retried, vec![message_id(5, 6, None)]); } - #[test] - fn consumer_delivery_preserves_missing_broker_redelivery_count() { - assert!(source_contains(&["tx:", " InternalMessageIdDataSender"])); - assert!(include_str!("data.rs").contains( - "InternalMessageIdDataResult = Result<(MessageIdData, Payload, Option)" - )); - assert!(source_contains(&[ - "let redelivery_count = message.", - "redelivery_count;" - ])); - assert!(source_contains(&[ - ".send(Ok((message_id.clone(), payload, redelivery_count)))" - ])); - } - - #[test] - fn ack_and_drop_paths_cancel_or_clear_negative_ack_tracker() { - assert!(source_contains(&["cancel_", "ack(&message_id)"])); - assert!(source_contains(&["cancel_", "cumulative_ack(&message_id)"])); - assert!(source_contains(&["tracker.", "clear()"])); - assert!(source_contains(&["ticker_running.", "store(false"])); - } - - #[test] - fn reconnect_preserves_negative_ack_tracker_state() { - let source = include_str!("engine.rs"); - let reconnect_source = source - .split("async fn reconnect") - .nth(1) - .and_then(|tail| tail.split("fn debug_format").next()) - .expect("reconnect source section exists"); - - assert!(!reconnect_source.contains("negative_ack_tracker")); - assert!(!reconnect_source.contains("tracker.clear()")); - } - #[derive(Default)] struct FakeNegativeAckRequester { redelivery_requests: Vec, diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index f71dbe36..b4cf1378 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -350,19 +350,3 @@ impl Stream for TopicConsumer { } } } - -#[cfg(test)] -mod tests { - fn source_contains(parts: &[&str]) -> bool { - let pattern = parts.concat(); - include_str!("topic.rs").contains(&pattern) - } - - #[test] - fn nack_uses_internal_broker_redelivery_count_presence() { - assert!(include_str!("topic.rs").contains("InternalEngineMessage::Nack(")); - assert!(source_contains(&["msg.", "message_id().clone()"])); - assert!(source_contains(&["msg.", "broker_redelivery_count()"])); - assert!(!source_contains(&["Some(msg.", "redelivery_count())"])); - } -} From 0ec48497c3570530f0889c81ddbc9c32b41b3b81 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 8 Jun 2026 16:51:32 -0300 Subject: [PATCH 55/66] fix(06-review): close negative ack backoff review gaps Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/negative_ack_backoff.rs | 14 ++++- src/consumer/topic.rs | 91 +++++++++++++++++++++++++++- 2 files changed, 102 insertions(+), 3 deletions(-) diff --git a/src/consumer/negative_ack_backoff.rs b/src/consumer/negative_ack_backoff.rs index bb97b22f..b9c8ad82 100644 --- a/src/consumer/negative_ack_backoff.rs +++ b/src/consumer/negative_ack_backoff.rs @@ -1,4 +1,4 @@ -use std::time::Duration; +use std::time::{Duration, Instant}; use crate::Error; @@ -141,7 +141,7 @@ impl MultiplierRedeliveryBackoffBuilder { } fn validate_delay_duration(delay: Duration) -> Result<(), Error> { - if delay.as_millis() > u64::MAX as u128 { + if delay.as_millis() >= u64::MAX as u128 || Instant::now().checked_add(delay).is_none() { return Err(Error::Custom(DELAY_DURATION_TOO_LARGE.to_string())); } @@ -248,6 +248,16 @@ mod tests { assert_custom_error(result, DELAY_DURATION_TOO_LARGE); } + #[test] + fn validation_rejects_unschedulable_max_millis_duration() { + let result = MultiplierRedeliveryBackoff::builder() + .min_delay(Duration::from_millis(u64::MAX)) + .max_delay(Duration::from_millis(u64::MAX)) + .build(); + + assert_custom_error(result, DELAY_DURATION_TOO_LARGE); + } + #[test] fn custom_negative_ack_backoff_can_be_implemented_without_engine_dependencies() { struct FixedBackoff(Duration); diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index b4cf1378..9570062b 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -231,7 +231,7 @@ impl TopicConsumer { self.engine_tx .send(InternalEngineMessage::Nack( msg.message_id().clone(), - msg.broker_redelivery_count(), + Some(msg.redelivery_count()), )) .await?; Ok(()) @@ -330,6 +330,95 @@ impl TopicConsumer { } } +#[cfg(test)] +mod tests { + use futures::{channel::mpsc, executor::block_on, StreamExt}; + + use super::*; + use crate::{ + executor::TokioExecutor, + message::{proto::MessageIdData, Metadata}, + }; + + fn message_id() -> MessageIdData { + MessageIdData { + ledger_id: 1, + entry_id: 2, + partition: None, + batch_index: None, + ack_set: vec![], + batch_size: None, + first_chunk_message_id: None, + } + } + + fn message(redelivery_count: Option) -> Message> { + Message::new_with_redelivery_count( + "persistent://public/default/topic", + MessageData { + id: message_id(), + batch_size: None, + }, + Payload { + metadata: Metadata::default(), + data: vec![], + }, + redelivery_count, + ) + } + + fn topic_consumer() -> ( + TopicConsumer, TokioExecutor>, + mpsc::UnboundedReceiver>, + ) { + let (_messages_tx, messages_rx) = mpsc::channel(1); + let (engine_tx, engine_rx) = mpsc::unbounded(); + + ( + TopicConsumer { + consumer_id: 1, + config: ConsumerConfig::default(), + topic: "persistent://public/default/topic".to_string(), + messages: Box::pin(messages_rx), + engine_tx, + data_type: PhantomData, + dead_letter_policy: None, + last_message_received: None, + messages_received: 0, + }, + engine_rx, + ) + } + + #[test] + fn nack_first_delivery_uses_zero_redelivery_count_for_backoff() { + let (mut consumer, mut engine_rx) = topic_consumer(); + + block_on(consumer.nack(&message(None))).unwrap(); + + match block_on(engine_rx.next()).expect("nack event") { + InternalEngineMessage::Nack(_, redelivery_count) => { + assert_eq!(redelivery_count, Some(0)); + } + _ => panic!("expected nack engine message"), + } + } + + #[test] + fn nack_redelivery_uses_broker_redelivery_count_for_backoff() { + let (mut consumer, mut engine_rx) = topic_consumer(); + + block_on(consumer.nack(&message(Some(3)))).unwrap(); + + match block_on(engine_rx.next()).expect("nack event") { + InternalEngineMessage::Nack(_, redelivery_count) => { + assert_eq!(redelivery_count, Some(3)); + } + _ => panic!("expected nack engine message"), + } + } +} + impl Stream for TopicConsumer { type Item = Result, Error>; From 98277895ba695fca31b3b1587314da0be758a2e1 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Mon, 8 Jun 2026 16:52:59 -0300 Subject: [PATCH 56/66] refactor(06-review): remove obsolete redelivery count helper Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/message.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/consumer/message.rs b/src/consumer/message.rs index b1dba8fe..65d9b4d1 100644 --- a/src/consumer/message.rs +++ b/src/consumer/message.rs @@ -61,10 +61,6 @@ impl Message { self.redelivery_count.unwrap_or_default() } - pub(crate) fn broker_redelivery_count(&self) -> Option { - self.redelivery_count - } - /// Get message key (partition key) #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn key(&self) -> Option { @@ -114,7 +110,7 @@ mod tests { Message::<()>::new_with_redelivery_count("test-topic", message_id, payload(), Some(7)); assert_eq!(msg.redelivery_count(), 7); - assert_eq!(msg.broker_redelivery_count(), Some(7)); + assert_eq!(msg.redelivery_count, Some(7)); } #[test] @@ -126,7 +122,7 @@ mod tests { let msg = Message::<()>::new("test-topic", message_id, payload()); assert_eq!(msg.redelivery_count(), 0); - assert_eq!(msg.broker_redelivery_count(), None); + assert_eq!(msg.redelivery_count, None); } #[test] @@ -139,6 +135,6 @@ mod tests { Message::<()>::new_with_redelivery_count("test-topic", message_id, payload(), Some(0)); assert_eq!(msg.redelivery_count(), 0); - assert_eq!(msg.broker_redelivery_count(), Some(0)); + assert_eq!(msg.redelivery_count, Some(0)); } } From aae55dd2a3745199ae0274340d786768fb589297 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Tue, 9 Jun 2026 09:02:41 -0300 Subject: [PATCH 57/66] chore: cleanup README --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index 97efc145..7dffe182 100644 --- a/README.md +++ b/README.md @@ -34,12 +34,6 @@ Try out [examples](examples): - [reader](examples/reader.rs) - [negative_ack](examples/negative_ack.rs) -## Negative Acknowledgment Delay - -The default negative-ack redelivery delay changed from immediate redelivery to 60 seconds, matching Go/Java client behavior. Consumers that relied on immediate redelivery can add `.with_nack_redelivery_delay(Duration::ZERO)` to preserve previous behavior. - -See the [negative_ack example](examples/negative_ack.rs) and [`ConsumerBuilder::with_nack_redelivery_delay`](https://docs.rs/pulsar/latest/pulsar/struct.ConsumerBuilder.html#method.with_nack_redelivery_delay) for configuration details. - ## Project Maintainers - [@CleverAkanoa](https://github.com/CleverAkanoa) From 70d09da0f561aea1827aa29f495565e7fc2e2302 Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Tue, 9 Jun 2026 09:04:51 -0300 Subject: [PATCH 58/66] chore: reorder tests --- src/consumer/topic.rs | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index 9570062b..86613d2e 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -330,6 +330,27 @@ impl TopicConsumer { } } +impl Stream for TopicConsumer { + type Item = Result, Error>; + + #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.messages.as_mut().poll_next(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(None) => Poll::Ready(None), + Poll::Ready(Some(Ok((id, payload, redelivery_count)))) => { + self.last_message_received = Some(Utc::now()); + self.messages_received += 1; + Poll::Ready(Some(Ok(self.create_message(id, payload, redelivery_count)))) + } + Poll::Ready(Some(Err(e))) => { + error!("we are using in the single-consumer and we got an error, {e}"); + Poll::Ready(Some(Err(e))) + } + } + } +} + #[cfg(test)] mod tests { use futures::{channel::mpsc, executor::block_on, StreamExt}; @@ -418,24 +439,3 @@ mod tests { } } } - -impl Stream for TopicConsumer { - type Item = Result, Error>; - - #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - match self.messages.as_mut().poll_next(cx) { - Poll::Pending => Poll::Pending, - Poll::Ready(None) => Poll::Ready(None), - Poll::Ready(Some(Ok((id, payload, redelivery_count)))) => { - self.last_message_received = Some(Utc::now()); - self.messages_received += 1; - Poll::Ready(Some(Ok(self.create_message(id, payload, redelivery_count)))) - } - Poll::Ready(Some(Err(e))) => { - error!("we are using in the single-consumer and we got an error, {e}"); - Poll::Ready(Some(Err(e))) - } - } - } -} From 8ae5a38a3cdfb2fee6cc83f22e1445c0d64bdb2e Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Tue, 9 Jun 2026 09:39:55 -0300 Subject: [PATCH 59/66] docs: tighten nack builder API docs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 21bab94d..783ddf9f 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -202,18 +202,16 @@ impl ConsumerBuilder { self } - /// Sets the fixed negative-ack redelivery delay. + /// Sets the fixed delay before redelivering negatively acknowledged messages. /// /// If this option is not set, negative acknowledgments use a 60 seconds default delay before - /// broker redelivery. This is an intentional parity behavior change from the previous Rust - /// client immediate-redelivery default. + /// broker redelivery. /// - /// `Duration::ZERO` is valid and means zero delay immediate redelivery; use it to preserve the - /// previous Rust-client behavior. Repeated calls are last-call-wins. + /// `Duration::ZERO` is valid and requests immediate redelivery. Repeated calls are + /// last-call-wins. /// - /// Delayed negative acks are tracked by a best-effort in-memory timer. Pending timers are - /// dropped on consumer close or process loss, and are preserved only best-effort across - /// reconnect when the same consumer engine survives. + /// The configured fixed delay is also used as the fallback for negative acknowledgments that do + /// not carry a broker redelivery count. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn with_nack_redelivery_delay(mut self, delay: Duration) -> Self { self.nack_redelivery_delay = Some(delay); @@ -222,19 +220,14 @@ impl ConsumerBuilder { /// Configures an optional negative-ack backoff policy. /// - /// Message-based nacks use the message redelivery count supplied by the broker to compute the - /// backoff delay. ID-only `nack_with_id` calls have no redelivery count, so the ID-only fixed - /// fallback uses [`Self::with_nack_redelivery_delay`] or the 60 seconds default instead. - /// The fixed fallback never uses the backoff policy. + /// Message-based negative acknowledgments use the broker-supplied redelivery count to compute + /// the backoff delay. ID-only `nack_with_id` calls have no redelivery count, so they use + /// [`Self::with_nack_redelivery_delay`] or the 60 seconds default instead. /// - /// A backoff that returns `Duration::ZERO` uses zero delay immediate redelivery. Batched message - /// ids use batch-entry normalization for scheduling, while ack cancellation remains - /// partial-batch-safe; this does not expose public precision controls or partial-batch - /// redelivery APIs. + /// A backoff that returns `Duration::ZERO` requests immediate redelivery. /// - /// Negative-ack tracking is separate from unacked-message resend tracking and preserves the - /// existing Rust DLQ behavior. This option adds no retry-letter/DLQ redesign, no - /// delayed-reconsume behavior, and no acknowledgement-timeout backoff behavior. + /// This policy affects only negative acknowledgments. Unacked-message redelivery and + /// dead-letter policy behavior remain configured separately. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn with_negative_ack_backoff(mut self, backoff: B) -> Self where From 89972666ada4729de5bf810e9aa9c2e5d52dedba Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Tue, 9 Jun 2026 09:47:20 -0300 Subject: [PATCH 60/66] fix: make nack default delay fallback overflow-safe Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 2 +- src/consumer/negative_ack_tracker.rs | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 783ddf9f..46a97126 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -437,7 +437,7 @@ mod tests { } #[test] - fn test_nack_delay_validation_rejects_unrepresentable_instant_delay() { + fn test_nack_delay_validation_rejects_millisecond_range_boundary() { let delay = Duration::from_millis(u64::MAX); let result = check_nack_delay_duration(delay); diff --git a/src/consumer/negative_ack_tracker.rs b/src/consumer/negative_ack_tracker.rs index 1e8cea00..ea9c192f 100644 --- a/src/consumer/negative_ack_tracker.rs +++ b/src/consumer/negative_ack_tracker.rs @@ -224,7 +224,8 @@ impl NegativeAckTracker { fn due_at(&self, now: Instant, delay: Duration) -> Instant { now.checked_add(delay) .or_else(|| now.checked_add(self.fixed_delay)) - .unwrap_or_else(|| now + DEFAULT_NACK_REDELIVERY_DELAY) + .or_else(|| now.checked_add(DEFAULT_NACK_REDELIVERY_DELAY)) + .unwrap_or(now) } #[cfg(test)] @@ -451,6 +452,22 @@ mod tests { assert_eq!(tracker.next_due_time(), Some(now + Duration::from_secs(5))); } + #[test] + fn oversized_backoff_and_fixed_delays_fall_back_to_default_delay() { + let now = Instant::now(); + let mut tracker = + NegativeAckTracker::new(Some(Duration::MAX), Some(backoff(Duration::MAX))); + + assert_eq!( + tracker.schedule(message_id(1, 2, None, None), Some(7), now), + NegativeAckSchedule::Scheduled + ); + assert_eq!( + tracker.next_due_time(), + Some(now + DEFAULT_NACK_REDELIVERY_DELAY) + ); + } + #[test] fn redelivery_message_id_normalizes_batch_specific_fields() { let mut original = message_id(1, 2, Some(0), Some(1)); From e6209c1d3b20a558298d2f3ff396b6d216e6f3be Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Tue, 9 Jun 2026 10:11:11 -0300 Subject: [PATCH 61/66] docs: align negative ack example style Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- examples/negative_ack.rs | 157 +++++++++++---------------------------- 1 file changed, 45 insertions(+), 112 deletions(-) diff --git a/examples/negative_ack.rs b/examples/negative_ack.rs index 19533dec..dd7d41ec 100644 --- a/examples/negative_ack.rs +++ b/examples/negative_ack.rs @@ -1,22 +1,11 @@ -//! Demonstrates negative acknowledgment redelivery delay and backoff configuration. -//! -//! Caveats: -//! - Message-based `nack()` uses the broker-supplied `redelivery_count` as the backoff input. -//! - `nack_with_id()` always uses fixed-delay behavior because message IDs do not carry -//! `redelivery_count`. -//! - Client-side timers are best-effort and in-memory; they do not guarantee exact redelivery timing. -//! - Closing or crashing the consumer drops pending nack schedules, so redelivery then follows -//! broker-side behavior. -//! - DLQ and retry-letter-topic routing behavior is not redesigned by this feature; existing DLQ -//! settings continue to apply independently. - #[macro_use] extern crate serde; use std::{env, time::Duration}; use futures::TryStreamExt; use pulsar::{ - Consumer, DeserializeMessage, MultiplierRedeliveryBackoff, NegativeAckBackoff, Payload, Pulsar, + authentication::{basic::BasicAuthentication, oauth2::OAuth2Authentication}, + Authentication, Consumer, DeserializeMessage, MultiplierRedeliveryBackoff, Payload, Pulsar, SubType, TokioExecutor, }; @@ -33,63 +22,6 @@ impl DeserializeMessage for TestData { } } -#[derive(Debug)] -struct LinearBackoff(Duration); - -impl NegativeAckBackoff for LinearBackoff { - fn next(&self, redelivery_count: u32) -> Duration { - self.0 * redelivery_count.saturating_add(1) - } -} - -#[allow(dead_code)] -async fn build_zero_delay_consumer( - pulsar: &Pulsar, - topic: String, -) -> Result, pulsar::Error> { - pulsar - .consumer() - .with_topic(topic) - .with_consumer_name("negative_ack_zero_delay") - .with_subscription_type(SubType::Exclusive) - .with_subscription("negative_ack_subscription") - .with_nack_redelivery_delay(Duration::ZERO) - .build() - .await -} - -#[allow(dead_code)] -async fn build_multiplier_backoff_consumer( - pulsar: &Pulsar, - topic: String, -) -> Result, pulsar::Error> { - pulsar - .consumer() - .with_topic(topic) - .with_consumer_name("negative_ack_multiplier_backoff") - .with_subscription_type(SubType::Exclusive) - .with_subscription("negative_ack_subscription") - .with_negative_ack_backoff(MultiplierRedeliveryBackoff::default()) - .build() - .await -} - -#[allow(dead_code)] -async fn build_linear_backoff_consumer( - pulsar: &Pulsar, - topic: String, -) -> Result, pulsar::Error> { - pulsar - .consumer() - .with_topic(topic) - .with_consumer_name("negative_ack_linear_backoff") - .with_subscription_type(SubType::Exclusive) - .with_subscription("negative_ack_subscription") - .with_negative_ack_backoff(LinearBackoff(Duration::from_secs(10))) - .build() - .await -} - #[tokio::main] async fn main() -> Result<(), pulsar::Error> { env_logger::init(); @@ -101,67 +33,68 @@ async fn main() -> Result<(), pulsar::Error> { .ok() .unwrap_or_else(|| "non-persistent://public/default/test".to_string()); - let pulsar: Pulsar<_> = Pulsar::builder(addr, TokioExecutor).build().await?; - - // HOW TO USE THIS EXAMPLE - // Section 1 (default delay) is the active configuration and runs by default. - // To try a different configuration: - // 1. Comment out the Section 1 consumer builder block below. - // 2. Replace it with exactly ONE of the compiled helper calls shown in Sections 2-4. - // Running multiple sections simultaneously is not supported; each section - // creates its own consumer on the same subscription. - - // Section 1: Default 60s fixed delay. - // As of this release, negative acknowledgments default to a 60 seconds fixed delay - // before redelivery. Previously, the Rust client requested immediate redelivery. - // To spell out that same fixed delay explicitly, add: - // .with_nack_redelivery_delay(Duration::from_secs(60)) + let mut builder = Pulsar::builder(addr, TokioExecutor); + + if let Ok(token) = env::var("PULSAR_TOKEN") { + let authentication = Authentication { + name: "token".to_string(), + data: token.into_bytes(), + }; + + builder = builder.with_auth(authentication); + } else if let Ok(oauth2_cfg) = env::var("PULSAR_OAUTH2") { + builder = builder.with_auth_provider(OAuth2Authentication::client_credentials( + serde_json::from_str(oauth2_cfg.as_str()) + .unwrap_or_else(|_| panic!("invalid oauth2 config [{}]", oauth2_cfg.as_str())), + )); + } else if let (Ok(username), Ok(password)) = ( + env::var("PULSAR_BASIC_USERNAME"), + env::var("PULSAR_BASIC_PASSWORD"), + ) { + builder = builder.with_auth_provider(BasicAuthentication::new(&username, &password)) + } + + let pulsar: Pulsar<_> = builder.build().await?; + let negative_ack_backoff = MultiplierRedeliveryBackoff::builder() + .min_delay(Duration::from_secs(1)) + .max_delay(Duration::from_secs(30)) + .multiplier(2.0) + .build()?; + let mut consumer: Consumer = pulsar .consumer() .with_topic(topic) - .with_consumer_name("negative_ack_default_delay") + .with_consumer_name("negative_ack_consumer") .with_subscription_type(SubType::Exclusive) .with_subscription("negative_ack_subscription") + .with_nack_redelivery_delay(Duration::from_secs(5)) + .with_negative_ack_backoff(negative_ack_backoff) .build() .await?; - // Section 2: Duration::ZERO migration escape hatch. - // Restores pre-release immediate-redelivery behavior for consumers that cannot tolerate the - // 60s delay. - // Duration::ZERO is the pre-v* immediate-redelivery equivalent. - // - // let mut consumer = build_zero_delay_consumer(&pulsar, topic).await?; - - // Section 3: Built-in MultiplierRedeliveryBackoff. - // The default policy uses broker redelivery count with a 1s minimum and 10min maximum delay. - // - // let mut consumer = build_multiplier_backoff_consumer(&pulsar, topic).await?; - - // Section 4: Custom NegativeAckBackoff policy. - // Implement the trait when you need application-specific redelivery spacing. - // - // let mut consumer = build_linear_backoff_consumer(&pulsar, topic).await?; - let mut counter = 0usize; while let Some(msg) = consumer.try_next().await? { log::info!("metadata: {:?}", msg.metadata()); log::info!("id: {:?}", msg.message_id()); log::info!("redelivery_count: {}", msg.redelivery_count()); - match msg.deserialize() { - Ok(data) => log::info!("payload data: {}", data.data), - Err(e) => log::error!("could not deserialize message: {:?}", e), - } + let data = match msg.deserialize() { + Ok(data) => data, + Err(e) => { + log::error!("could not deserialize message: {:?}", e); + consumer.nack(&msg).await?; + break; + } + }; + log::info!("payload data: {}", data.data); consumer.nack(&msg).await?; - // Section 5: nack_with_id note. - // consumer.nack_with_id(&msg.topic, msg.message_id().clone()).await? is also supported - // but always uses fixed delay only, because message IDs do not carry redelivery count. - counter += 1; + log::info!("nacked {} messages", counter); + if counter > 10 { - consumer.close().await?; + consumer.close().await.expect("Unable to close consumer"); break; } } From afb05319cc1ca7e6addac77cc48b137572acf55c Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Tue, 9 Jun 2026 10:28:05 -0300 Subject: [PATCH 62/66] refactor: simplify nack backoff trait object type Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/consumer/builder.rs | 5 ++--- src/consumer/config.rs | 2 +- src/consumer/engine.rs | 4 ++-- src/consumer/negative_ack_tracker.rs | 6 +++--- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 46a97126..affee5a0 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -43,7 +43,7 @@ pub struct ConsumerBuilder { namespace: Option, topic_refresh: Option, nack_redelivery_delay: Option, - negative_ack_backoff: Option>, + negative_ack_backoff: Option>, } fn check_nack_delay_duration(delay: Duration) -> Result<(), Error> { @@ -233,8 +233,7 @@ impl ConsumerBuilder { where B: NegativeAckBackoff + 'static, { - self.negative_ack_backoff = - Some(Arc::new(backoff) as Arc); + self.negative_ack_backoff = Some(Arc::new(backoff) as Arc); self } diff --git a/src/consumer/config.rs b/src/consumer/config.rs index 4942cf27..7426aeec 100644 --- a/src/consumer/config.rs +++ b/src/consumer/config.rs @@ -33,7 +33,7 @@ pub struct ConsumerConfig { /// fixed negative-ack redelivery delay pub(crate) nack_redelivery_delay: Option, /// negative-ack redelivery backoff policy - pub(crate) negative_ack_backoff: Option>, + pub(crate) negative_ack_backoff: Option>, } impl fmt::Debug for ConsumerConfig { diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index ee1460fc..e6e2f0de 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -126,7 +126,7 @@ pub struct ConsumerEngine { unacked_message_redelivery_delay: Option, unacked_messages: HashMap, nack_redelivery_delay: Option, - negative_ack_backoff: Option>, + negative_ack_backoff: Option>, negative_ack_tracker: Option, negative_ack_ticker_running: Option>, negative_ack_due_event_pending: Arc, @@ -150,7 +150,7 @@ impl ConsumerEngine { batch_size: u32, unacked_message_redelivery_delay: Option, nack_redelivery_delay: Option, - negative_ack_backoff: Option>, + negative_ack_backoff: Option>, dead_letter_policy: Option, options: ConsumerOptions, ) -> ConsumerEngine { diff --git a/src/consumer/negative_ack_tracker.rs b/src/consumer/negative_ack_tracker.rs index ea9c192f..9943234e 100644 --- a/src/consumer/negative_ack_tracker.rs +++ b/src/consumer/negative_ack_tracker.rs @@ -26,7 +26,7 @@ pub(crate) enum NegativeAckDispatchState { pub(crate) struct NegativeAckTracker { fixed_delay: Duration, - backoff: Option>, + backoff: Option>, entries: HashMap, } @@ -131,7 +131,7 @@ impl PendingBatchScope { impl NegativeAckTracker { pub(crate) fn new( fixed_delay: Option, - backoff: Option>, + backoff: Option>, ) -> Self { Self { fixed_delay: fixed_delay.unwrap_or(DEFAULT_NACK_REDELIVERY_DELAY), @@ -351,7 +351,7 @@ mod tests { } } - fn backoff(delay: Duration) -> Arc { + fn backoff(delay: Duration) -> Arc { Arc::new(FixedBackoff(delay)) } From efbc98f129b06e22505375f7a1a271e6f364177a Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Wed, 10 Jun 2026 16:40:50 -0300 Subject: [PATCH 63/66] docs(consumer): clarify nack redelivery delay and backoff doc semantics --- src/consumer/builder.rs | 10 ++++++---- src/consumer/mod.rs | 13 +++++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index affee5a0..1ca6e23a 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -210,8 +210,9 @@ impl ConsumerBuilder { /// `Duration::ZERO` is valid and requests immediate redelivery. Repeated calls are /// last-call-wins. /// - /// The configured fixed delay is also used as the fallback for negative acknowledgments that do - /// not carry a broker redelivery count. + /// The configured fixed delay is used for ID-only [`Self::nack_with_id`] calls, which carry no + /// redelivery count. It is also used as the delay for message-based [`Consumer::nack`] calls + /// when no backoff policy has been configured. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn with_nack_redelivery_delay(mut self, delay: Duration) -> Self { self.nack_redelivery_delay = Some(delay); @@ -220,8 +221,9 @@ impl ConsumerBuilder { /// Configures an optional negative-ack backoff policy. /// - /// Message-based negative acknowledgments use the broker-supplied redelivery count to compute - /// the backoff delay. ID-only `nack_with_id` calls have no redelivery count, so they use + /// Message-based negative acknowledgments use the message's redelivery count (`0` on first + /// delivery) to compute the backoff delay. ID-only `nack_with_id` calls have no redelivery + /// count, so they use /// [`Self::with_nack_redelivery_delay`] or the 60 seconds default instead. /// /// A backoff that returns `Duration::ZERO` requests immediate redelivery. diff --git a/src/consumer/mod.rs b/src/consumer/mod.rs index d63b655e..4f317e23 100644 --- a/src/consumer/mod.rs +++ b/src/consumer/mod.rs @@ -157,9 +157,12 @@ impl Consumer { } } - /// negative acknowledgement + /// Negative acknowledgement. /// - /// the message will be sent again on the subscription + /// The message will be redelivered on the subscription after the configured negative-ack + /// redelivery delay (60 seconds by default). If a backoff policy is configured via + /// [`ConsumerBuilder::with_negative_ack_backoff`], the delay is computed from the message's + /// redelivery count instead. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn nack(&mut self, msg: &Message) -> Result<(), ConsumerError> { match &mut self.inner { @@ -168,9 +171,11 @@ impl Consumer { } } - /// negative acknowledgement + /// Negative acknowledgement by message ID. /// - /// the message with the given ID will be sent again on the subscription + /// The message with the given ID will be redelivered on the subscription after the configured + /// negative-ack redelivery delay (60 seconds by default). Because no redelivery count is + /// available, a backoff policy is not applied. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn nack_with_id( &mut self, From 614289a26ad82491fc755467e5137fde9bab7ddb Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Wed, 10 Jun 2026 16:41:04 -0300 Subject: [PATCH 64/66] docs(readme): describe pluggable negative-ack backoff policies --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7dffe182..1452a89e 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,8 @@ Features: - Message batching; - Compression with LZ4, zlib, zstd or Snappy (can be deactivated with Cargo features); - Telemetry using [tracing](https://github.com/tokio-rs/tracing) crate (can be activated with Cargo features); -- Configurable negative-acknowledgment redelivery delay with optional exponential backoff; +- Configurable negative-acknowledgment redelivery delay with pluggable backoff policies + (multiplier-based backoff built in); ## Getting Started From 31bee6ee5876a6a3a52b0249add09ae7fbd9e6ee Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Wed, 10 Jun 2026 16:43:20 -0300 Subject: [PATCH 65/66] fix(consumer): clear negative ack state when consumer closes --- src/consumer/data.rs | 1 + src/consumer/engine.rs | 25 +++++++++++++++++++++++++ src/consumer/topic.rs | 3 +++ 3 files changed, 29 insertions(+) diff --git a/src/consumer/data.rs b/src/consumer/data.rs index a4b0f936..aa52018c 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -36,6 +36,7 @@ pub(crate) enum InternalEngineMessage { NegativeAckRedelivery, UnackedRedelivery, GetConnection(oneshot::Sender>>), + ClearNegativeAcks, } #[derive(Clone, Debug, PartialEq, Eq, Hash)] diff --git a/src/consumer/engine.rs b/src/consumer/engine.rs index e6e2f0de..277bd7a8 100644 --- a/src/consumer/engine.rs +++ b/src/consumer/engine.rs @@ -419,6 +419,10 @@ impl ConsumerEngine { }); true } + Some(InternalEngineMessage::ClearNegativeAcks) => { + self.clear_negative_ack_state(); + true + } } } @@ -1256,4 +1260,25 @@ mod tests { .collect_due(now + Duration::from_secs(7) + NEGATIVE_ACK_REDELIVERY_TICK_INTERVAL) .is_empty()); } + + #[test] + fn clear_negative_ack_state_drains_tracker_and_stops_ticker() { + let now = Instant::now(); + let id = message_id(1, 2, Some(0)); + let mut tracker = NegativeAckTracker::new(Some(Duration::from_secs(60)), None); + let ticker_running = Arc::new(AtomicBool::new(false)); + + // Schedule a nack so the tracker is non-empty + tracker.schedule(id.clone(), Some(0), now); + ticker_running.store(true, Ordering::SeqCst); + + // Simulate what clear_negative_ack_state does + tracker.clear(); + ticker_running.store(false, Ordering::SeqCst); + + assert!(tracker + .collect_due(now + Duration::from_secs(120)) + .is_empty()); + assert!(!ticker_running.load(Ordering::SeqCst)); + } } diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index 86613d2e..ebaadce2 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -274,6 +274,9 @@ impl TopicConsumer { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn close(&mut self) -> Result<(), Error> { let consumer_id = self.consumer_id; + let _ = self + .engine_tx + .unbounded_send(InternalEngineMessage::ClearNegativeAcks); self.connection() .await? .sender() From fe74ac2bfc5583fdefabf33be6c0c465b5e768cc Mon Sep 17 00:00:00 2001 From: Pedro Morello Abbud Date: Wed, 10 Jun 2026 18:21:28 -0300 Subject: [PATCH 66/66] chore: fix readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 1452a89e..de6ee932 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,6 @@ Features: - Compression with LZ4, zlib, zstd or Snappy (can be deactivated with Cargo features); - Telemetry using [tracing](https://github.com/tokio-rs/tracing) crate (can be activated with Cargo features); - Configurable negative-acknowledgment redelivery delay with pluggable backoff policies - (multiplier-based backoff built in); ## Getting Started