Harden MothCDC for the 0.7 release

.github/workflows/ci.yml

@@ -22,13 +22,33 @@ jobs:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
-          components: clippy
-      - name: clippy (lib + tests)
-        run: cargo clippy --lib --tests -- -D warnings
+          components: clippy, rustfmt
+      - name: formatting
+        run: cargo fmt --all -- --check
+      - name: clippy (all targets and features)
+        run: cargo clippy --all-targets --all-features -- -D warnings
       - name: build (all targets)
-        run: cargo build --all-targets
+        run: cargo build --all-targets --all-features
       - name: test
-        run: cargo test --release
+        run: cargo test --release --all-features
+
+  msrv:
+    name: MSRV (Rust 1.89)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@1.89
+      - run: cargo check --all-targets --all-features
+
+  portable:
+    name: 32-bit portability
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: i686-unknown-linux-gnu
+      - run: cargo check --target i686-unknown-linux-gnu --lib --all-features
 
   # Run the cross-SIMD-width determinism tests under Intel SDE so the AVX-512
   # argmin and AVX-512BW packed-scan paths actually execute (and are checked
@@ -46,6 +66,6 @@ jobs:
       # hardware and under SDE 10.8.0 (verified on an Intel Xeon w/ avx512bw).
       - uses: petarpetrovt/setup-sde@v5.0
       - name: Build tests
-        run: cargo test --release --no-run
+        run: cargo test --release --all-features --no-run
       - name: cross-impl determinism under emulated AVX-512 (Sapphire Rapids)
-        run: ${SDE_PATH}/sde64 -spr -- cargo test --release widths_agree -- --nocapture
+        run: ${SDE_PATH}/sde64 -spr -- cargo test --release --all-features widths_agree -- --nocapture

CHANGELOG.md

@@ -1,8 +1,7 @@
 # Changelog
 
-## 0.7.0 — API unification (2026-07-11)
+## 0.7.1 — API unification and release hardening (2026-07-12)
 
-Breaking change (0.6.0 has zero dependents, so this ships as a free break).
 Unifies the public API around the caterpillar layer as the recommended
 entry point, and separates the inherited MinCDC core from the caterpillar
 layer at the module level:
@@ -20,16 +19,32 @@ layer at the module level:
   `ReadChunker`) moved into a new `mincdc` submodule and is no longer
   re-exported at the crate root — reach it as `mothcdc::mincdc::...`. Only
   `Chunk` stays re-exported at the root (shared by both layers).
-- No behavior change to chunk boundary placement, the caterpillar
-  packed-scanning fast path, or the C API (`mothcdc_next_chunk`'s symbol
-  and behavior are unchanged).
+- `Chunk`/`Segment` offsets, segment lengths, and represented chunk counts are
+  now `u64`; `Segment` has a private validated representation rather than
+  publicly constructible invalid variants.
+- Reader chunkers retry `Interrupted`, use checked buffer arithmetic, expose
+  fallible constructors and reader accessors, and report offset/count overflow.
+- Streaming caterpillar runs now remain maximal even when a `Read`
+  implementation returns tiny fragments; a one-byte reader and `Cursor`
+  produce the same grouping.
+- Fixed a panic on x86_64 CPUs without SSE4.1 when the exact scalar argmin was
+  one of the final three windows.
+- The public `Cdc` splitpoint contract is documented and enforced.
+- The academic `MinCdc4` unit struct can be constructed directly without using
+  its deprecated `new()` helper.
+- The C API validates size configurations without panicking. A normal Rust
+  dependency now builds only an `rlib`; the benchmark static library is built
+  explicitly with `cargo rustc --features capi -- --crate-type staticlib`.
+- CI now covers every feature, formatting, the 1.89 MSRV, and a 32-bit target.
+- The SIMD prefetch soundness fix was submitted upstream as
+  [orlp/mincdc#1](https://github.com/orlp/mincdc/pull/1).
 
 Also includes the mincatcdc -> mothcdc rename (no functional change).
 Credits unchanged: MinCDC algorithm (Orson Peters), caterpillar layer
 inspired by Chonkers (Berger), vector acceleration in the style of
 VectorCDC (Udayashankar et al.).
 
-## 0.6.0 — optimization campaign (2026-07-11)
+## 0.6.0 — optimization campaign and packed scanning (2026-07-10–11)
 
 Ten hypothesis-driven loops on a fixed Fly performance-4x machine (AMD
 EPYC/AVX2), each benched against the same corpus (public Tigris bucket).
@@ -65,7 +80,7 @@ AE-Min 55.47%, FastCDC 52.44%, RAM 49.21%) — at 8.3-9.9 GB/s vs
 VectorCDC-AE-Min's 4.6 GB/s on the same data (only VectorCDC-RAM is
 faster at 17.2 GB/s, with the worst dedup of the field).
 
-## 0.6.0 — 2026-07-10
+### Packed-scanning caterpillar fast path
 
 Packed-scanning caterpillar fast path (VectorCDC-style SIMD).
 
@@ -93,7 +108,7 @@ Packed-scanning caterpillar fast path (VectorCDC-style SIMD).
     within ~2%.
   - Synthetic ceiling: zeros 2.4 → 74.3 GiB/s on Intel Xeon/AVX-512BW,
     1.8 → 30.2 GiB/s on NEON.
-- New `capi` feature: a minimal C API (`mincatcdc_next_chunk`) plus a
+- New `capi` feature: a minimal C API (`mothcdc_next_chunk`) plus a
   dedup-bench fork
   (github.com/russellromney/dedup-bench, branch `mincatcdc-integration`)
   that adds `chunking_algo=mincdc`, so mincatcdc is measured by the *same*

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "mothcdc"
-version = "0.7.0"
-authors = ["Orson Peters <orsonpeters@gmail.com>", "Russell Romney"]
+version = "0.7.1"
+authors = ["Russell Romney"]
 license = "Zlib"
 repository = "https://github.com/russellromney/mothcdc"
 readme = "README.md"
@@ -14,11 +14,6 @@ edition = "2024"
 rust-version = "1.89"
 exclude = ["assets", "tools"]
 
-[lib]
-# staticlib is consumed by the dedup-bench integration (feature `capi`);
-# rlib is the normal Rust library.
-crate-type = ["rlib", "staticlib"]
-
 [features]
 # Minimal C API for embedding in external harnesses (see src/capi.rs).
 capi = []

README.md

@@ -14,7 +14,7 @@ deduplicated.
 To start using `mothcdc` add the following to your `Cargo.toml`:
 
     [dependencies]
-    mothcdc = "0.6"
+    mothcdc = "0.7"
 
 Please refer to [the documentation](https://docs.rs/mothcdc) for more
 information on usage.
@@ -44,6 +44,13 @@ The crate also ships a C API (`--features capi`) and a
 [dedup-bench fork](https://github.com/russellromney/dedup-bench/tree/MothCDC-integration)
 so it can be measured by the same harness as every other chunker.
 
+The normal Cargo dependency builds only a Rust `rlib`. To produce the static
+library used by C/C++ benchmark harnesses, run:
+
+```sh
+cargo rustc --release --features capi --lib -- --crate-type staticlib
+```
+
 ### Benchmarks
 
 All numbers come from [UWASL dedup-bench](https://github.com/UWASL/dedup-bench)
@@ -141,6 +148,12 @@ once — at most `max_size` bytes — when it crosses a refill, so even a
 multi-gigabyte zero region is a single record. Everything else stays
 zero-copy.
 
+Offsets, segment lengths, and represented chunk counts are `u64`, so streaming
+metadata remains correct on 32-bit targets. Individual boundary-search windows
+are limited to `mothcdc::mincdc::MAX_CHUNK_SIZE`; the reader chunkers allocate
+at least 4 MiB and provide fallible `try_new` constructors for invalid or
+unallocatable configurations.
+
 (An experimental second tier — content-defined *period detection* for
 phase-rotating runs — was evaluated and removed: mincdc self-aligns to most
 periods so it rarely helped, and it cost 76–99% throughput. See
@@ -156,10 +169,6 @@ produced 7,798, with identical deduplicated content. Full method and the other
 corpora (Linux kernels, containers, SQLite, source trees) are in
 `examples/REALBENCH_RESULTS.md`. Obviously the benchmark is specific to this case. YMMV. 
 
-This fork also fixes a soundness bug in the upstream SIMD prefetch and adds test
-coverage (cross-SIMD-width determinism, an invariant/oracle harness).
-
-
 ## Algorithm
 
 The basic idea of MinCDC is to choose chunk boundaries based on the minimum
@@ -240,4 +249,3 @@ smaller.
 
 There is still a bias towards smaller chunks as MinCDC breaks ties in the
 minimum value towards the earlier breakpoint, but this bias is relatively small.
-

benches/throughput.rs

@@ -166,7 +166,7 @@ fn caterpillar_scalar_reference(data: &[u8]) -> usize {
             Some((off, len)) if data[off..off + len] == c[..] => {},
             _ => {
                 records += 1;
-                last = Some((c.offset(), c.len()));
+                last = Some((c.offset() as usize, c.len()));
             },
         }
     }
@@ -184,7 +184,7 @@ fn bench_chunking(c: &mut Criterion) {
 
         g.bench_with_input(BenchmarkId::new("plain", &name), &data, |b, d| {
             b.iter(|| {
-                let mut acc = 0usize;
+                let mut acc = 0u64;
                 for c in SliceChunker::new(d, MIN, MAX, MinCdcHash4::new()) {
                     acc ^= c.offset();
                 }
@@ -203,7 +203,7 @@ fn bench_chunking(c: &mut Criterion) {
             &data,
             |b, d| {
                 b.iter(|| {
-                    let mut acc = 0usize;
+                    let mut acc = 0u64;
                     for s in MothChunker::new(d, MIN, MAX) {
                         acc ^= s.offset() ^ s.chunk_count();
                     }

examples/catbench.rs

@@ -139,7 +139,7 @@ fn main() {
     // plain
     let (secs, _) = best_of(|| {
         let mut n = 0usize;
-        let mut s = 0usize;
+        let mut s = 0u64;
         for b in &blobs {
             for c in SliceChunker::new(b, MIN, MAX, cdc) {
                 s ^= c.offset();
@@ -157,7 +157,7 @@ fn main() {
     {
         let (secs, _) = best_of(|| {
             let mut n = 0usize;
-            let mut s = 0usize;
+            let mut s = 0u64;
             for b in &blobs {
                 for seg in MothChunker::with_cdc(b, MIN, MAX, cdc) {
                     s ^= seg.offset();

examples/compare.rs

@@ -40,7 +40,7 @@ struct Stats {
 /// Accumulates (key, stored_len) records and reports unique/dedup over them.
 struct Dedup {
     seen: std::collections::HashMap<u64, usize>,
-    logical: usize,
+    logical: u64,
     records: usize,
 }
 impl Dedup {
@@ -51,7 +51,7 @@ impl Dedup {
             records: 0,
         }
     }
-    fn add(&mut self, key: u64, stored_len: usize, logical_len: usize) {
+    fn add(&mut self, key: u64, stored_len: usize, logical_len: u64) {
         self.seen.entry(key).or_insert(stored_len);
         self.logical += logical_len;
         self.records += 1;
@@ -124,7 +124,7 @@ fn run_fastcdc(data: &[u8], min: usize, avg: usize, max: usize, d: &mut Dedup) -
     });
     for c in FastCDC::new(data, min, avg, max) {
         let b = &data[c.offset..c.offset + c.length];
-        d.add(fnv1a(b), b.len(), b.len());
+        d.add(fnv1a(b), b.len(), b.len() as u64);
     }
     Stats {
         records,
@@ -144,7 +144,7 @@ fn run_mincdc(data: &[u8], min: usize, max: usize, d: &mut Dedup) -> Stats {
         n
     });
     for c in SliceChunker::new(data, min, max, cdc) {
-        d.add(fnv1a(&c), c.len(), c.len());
+        d.add(fnv1a(&c), c.len(), c.len() as u64);
     }
     Stats {
         records,
@@ -206,7 +206,7 @@ fn scenario_versioned(min: usize, avg: usize, max: usize) {
     for data in [&v1[..], &v2[..]] {
         for c in FastCDC::new(data, min, avg, max) {
             let b = &data[c.offset..c.offset + c.length];
-            d.add(fnv1a(b), b.len(), b.len());
+            d.add(fnv1a(b), b.len(), b.len() as u64);
         }
     }
     println!(
@@ -222,7 +222,7 @@ fn scenario_versioned(min: usize, avg: usize, max: usize) {
     let mut d = Dedup::new();
     for data in [&v1[..], &v2[..]] {
         for c in SliceChunker::new(data, min, max, cdc) {
-            d.add(fnv1a(&c), c.len(), c.len());
+            d.add(fnv1a(&c), c.len(), c.len() as u64);
         }
     }
     println!(

examples/frontier.rs

@@ -47,7 +47,7 @@ fn run(label: &str, blobs: &[Vec<u8>], total: u64, min: usize, max: usize, cdc:
     let mut best = f64::MAX;
     for _ in 0..3 {
         let t = Instant::now();
-        let mut acc = 0usize;
+        let mut acc = 0u64;
         for b in blobs {
             for s in MothChunker::with_cdc(b, min, max, cdc) {
                 acc ^= s.offset();
@@ -59,7 +59,7 @@ fn run(label: &str, blobs: &[Vec<u8>], total: u64, min: usize, max: usize, cdc:
 
     // Dedup + metadata (single deterministic pass).
     let mut store: HashMap<u64, usize> = HashMap::new();
-    let (mut records, mut chunks) = (0usize, 0usize);
+    let (mut records, mut chunks) = (0usize, 0u64);
     for b in blobs {
         for s in MothChunker::with_cdc(b, min, max, cdc) {
             let key = s.dedup_key();

examples/realbench.rs

@@ -68,11 +68,11 @@ impl Acc {
             secs: 0.0,
         }
     }
-    fn add(&mut self, key: u64, stored: usize, logical: usize) {
+    fn add(&mut self, key: u64, stored: usize, logical: u64) {
         self.seen.entry(key).or_insert(stored);
         self.records += 1;
-        self.logical += logical as u64;
-        self.sizes.push(logical as u32);
+        self.logical += logical;
+        self.sizes.push(u32::try_from(logical).unwrap_or(u32::MAX));
     }
     fn report(&self, name: &str, total_bytes: u64) {
         let gbps = total_bytes as f64 / self.secs.max(1e-9) / 1e9;
@@ -147,15 +147,15 @@ fn main() {
     for b in &blobs {
         for c in FastCDC::new(b, MIN, AVG, fast_max()) {
             let s = &b[c.offset..c.offset + c.length];
-            a.add(fnv1a(s), s.len(), s.len());
+            a.add(fnv1a(s), s.len(), s.len() as u64);
         }
     }
     a.report("fastcdc-v2020", total_bytes);
 
     // mincdc-plain
     let mut a = Acc::new();
     let t = Instant::now();
-    let mut sink = 0usize;
+    let mut sink = 0u64;
     for b in &blobs {
         for c in SliceChunker::new(b, MIN, MC_MAX, cdc) {
             sink ^= c.offset();
@@ -165,7 +165,7 @@ fn main() {
     std::hint::black_box(sink);
     for b in &blobs {
         for c in SliceChunker::new(b, MIN, MC_MAX, cdc) {
-            a.add(fnv1a(&c), c.len(), c.len());
+            a.add(fnv1a(&c), c.len(), c.len() as u64);
         }
     }
     a.report("mincdc-plain", total_bytes);
@@ -174,7 +174,7 @@ fn main() {
     {
         let mut a = Acc::new();
         let t = Instant::now();
-        let mut sink = 0usize;
+        let mut sink = 0u64;
         for b in &blobs {
             for s in MothChunker::with_cdc(b, MIN, MC_MAX, cdc) {
                 sink ^= s.offset();

rustfmt.toml

@@ -1,4 +1,2 @@
-group_imports = "StdExternalCrate"
-imports_granularity = "Module"
 match_block_trailing_comma = true
 use_field_init_shorthand = true