perf(walk): improvements to the walker hot path

Cargo.toml

@@ -39,9 +39,9 @@ regex = "1"
 config-file = "0.2"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-sysinfo = "0.37"
 ctrlc = "3"
 chrono = "0.4"
+rustc-hash = "2"
 
 [target.'cfg(not(target_has_atomic = "64"))'.dependencies]
 portable-atomic = "1.4"

README.md

@@ -117,8 +117,8 @@ Usage: dust -e regex (Only include files matching this regex (eg dust -e "\.png$
 Usage: dust -v regex (Exclude files matching this regex (eg dust -v "\.png$" would ignore png files))
 Usage: dust -L (dereference-links - Treat sym links as directories and go into them)
 Usage: dust -P (Disable the progress indicator)
+Usage: dust -T (Set the number of walker threads. Default is the CPU count. For high-latency storage like NFS or remote mounts, a higher count can speed up the walk by overlapping more concurrent stat calls.)
 Usage: dust -R (For screen readers. Removes bars/symbols. Adds new column: depth level. (May want to use -p for full path too))
-Usage: dust -S (Custom Stack size - Use if you see: 'fatal runtime error: stack overflow' (default allocation: low memory=1048576, high memory=1073741824)"),
 Usage: dust --skip-total (No total row will be displayed)
 Usage: dust -z 40000/30MB/20kib (Exclude output files/directories below size 40000 bytes / 30MB / 20KiB)
 Usage: dust -j (Prints JSON representation of directories, try: dust -j  | jq)

completions/_dust

@@ -54,8 +54,8 @@ kb\:"kilobyte (kB)"
 mb\:"megabyte (MB)"
 gb\:"gigabyte (GB)"
 tb\:"terabyte (TB)"))' \
-'-S+[Specify memory to use as stack size - use if you see\: '\''fatal runtime error\: stack overflow'\'' (default low memory=1048576, high memory=1073741824)]:STACK_SIZE:_default' \
-'--stack-size=[Specify memory to use as stack size - use if you see\: '\''fatal runtime error\: stack overflow'\'' (default low memory=1048576, high memory=1073741824)]:STACK_SIZE:_default' \
+'-S+[Deprecated. The walker no longer recurses so a custom stack size is unnecessary. Accepted for compatibility but the value is ignored]:STACK_SIZE:_default' \
+'--stack-size=[Deprecated. The walker no longer recurses so a custom stack size is unnecessary. Accepted for compatibility but the value is ignored]:STACK_SIZE:_default' \
 '-M+[+/-n matches files modified more/less than n days ago , and n matches files modified exactly n days ago, days are rounded down.That is +n => (−∞, curr−(n+1)), n => \[curr−(n+1), curr−n), and -n => (𝑐𝑢𝑟𝑟−𝑛, +∞)]:MTIME:_default' \
 '--mtime=[+/-n matches files modified more/less than n days ago , and n matches files modified exactly n days ago, days are rounded down.That is +n => (−∞, curr−(n+1)), n => \[curr−(n+1), curr−n), and -n => (𝑐𝑢𝑟𝑟−𝑛, +∞)]:MTIME:_default' \
 '-A+[just like -mtime, but based on file access time]:ATIME:_default' \

completions/_dust.ps1

@@ -42,8 +42,8 @@ Register-ArgumentCompleter -Native -CommandName 'dust' -ScriptBlock {
             [CompletionResult]::new('--terminal-width', '--terminal-width', [CompletionResultType]::ParameterName, 'Specify width of output overriding the auto detection of terminal width')
             [CompletionResult]::new('-o', '-o', [CompletionResultType]::ParameterName, 'Changes output display size. si will print sizes in powers of 1000. b k m g t kb mb gb tb will print the whole tree in that size')
             [CompletionResult]::new('--output-format', '--output-format', [CompletionResultType]::ParameterName, 'Changes output display size. si will print sizes in powers of 1000. b k m g t kb mb gb tb will print the whole tree in that size')
-            [CompletionResult]::new('-S', '-S ', [CompletionResultType]::ParameterName, 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)')
-            [CompletionResult]::new('--stack-size', '--stack-size', [CompletionResultType]::ParameterName, 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)')
+            [CompletionResult]::new('-S', '-S ', [CompletionResultType]::ParameterName, 'Deprecated. The walker no longer recurses so a custom stack size is unnecessary. Accepted for compatibility but the value is ignored')
+            [CompletionResult]::new('--stack-size', '--stack-size', [CompletionResultType]::ParameterName, 'Deprecated. The walker no longer recurses so a custom stack size is unnecessary. Accepted for compatibility but the value is ignored')
             [CompletionResult]::new('-M', '-M ', [CompletionResultType]::ParameterName, '+/-n matches files modified more/less than n days ago , and n matches files modified exactly n days ago, days are rounded down.That is +n => (−∞, curr−(n+1)), n => [curr−(n+1), curr−n), and -n => (𝑐𝑢𝑟𝑟−𝑛, +∞)')
             [CompletionResult]::new('--mtime', '--mtime', [CompletionResultType]::ParameterName, '+/-n matches files modified more/less than n days ago , and n matches files modified exactly n days ago, days are rounded down.That is +n => (−∞, curr−(n+1)), n => [curr−(n+1), curr−n), and -n => (𝑐𝑢𝑟𝑟−𝑛, +∞)')
             [CompletionResult]::new('-A', '-A ', [CompletionResultType]::ParameterName, 'just like -mtime, but based on file access time')

completions/dust.elv

@@ -39,8 +39,8 @@ set edit:completion:arg-completer[dust] = {|@words|
             cand --terminal-width 'Specify width of output overriding the auto detection of terminal width'
             cand -o 'Changes output display size. si will print sizes in powers of 1000. b k m g t kb mb gb tb will print the whole tree in that size'
             cand --output-format 'Changes output display size. si will print sizes in powers of 1000. b k m g t kb mb gb tb will print the whole tree in that size'
-            cand -S 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)'
-            cand --stack-size 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)'
+            cand -S 'Deprecated. The walker no longer recurses so a custom stack size is unnecessary. Accepted for compatibility but the value is ignored'
+            cand --stack-size 'Deprecated. The walker no longer recurses so a custom stack size is unnecessary. Accepted for compatibility but the value is ignored'
             cand -M '+/-n matches files modified more/less than n days ago , and n matches files modified exactly n days ago, days are rounded down.That is +n => (−∞, curr−(n+1)), n => [curr−(n+1), curr−n), and -n => (𝑐𝑢𝑟𝑟−𝑛, +∞)'
             cand --mtime '+/-n matches files modified more/less than n days ago , and n matches files modified exactly n days ago, days are rounded down.That is +n => (−∞, curr−(n+1)), n => [curr−(n+1), curr−n), and -n => (𝑐𝑢𝑟𝑟−𝑛, +∞)'
             cand -A 'just like -mtime, but based on file access time'

completions/dust.fish

@@ -18,7 +18,7 @@ kb\t'kilobyte (kB)'
 mb\t'megabyte (MB)'
 gb\t'gigabyte (GB)'
 tb\t'terabyte (TB)'"
-complete -c dust -s S -l stack-size -d 'Specify memory to use as stack size - use if you see: \'fatal runtime error: stack overflow\' (default low memory=1048576, high memory=1073741824)' -r
+complete -c dust -s S -l stack-size -d 'Deprecated. The walker no longer recurses so a custom stack size is unnecessary. Accepted for compatibility but the value is ignored' -r
 complete -c dust -s M -l mtime -d '+/-n matches files modified more/less than n days ago , and n matches files modified exactly n days ago, days are rounded down.That is +n => (−∞, curr−(n+1)), n => [curr−(n+1), curr−n), and -n => (𝑐𝑢𝑟𝑟−𝑛, +∞)' -r
 complete -c dust -s A -l atime -d 'just like -mtime, but based on file access time' -r
 complete -c dust -s y -l ctime -d 'just like -mtime, but based on file change time' -r

man-page/dust.1

@@ -123,7 +123,7 @@ tb: terabyte (TB)
 .RE
 .TP
 \fB\-S\fR, \fB\-\-stack\-size\fR \fI<STACK_SIZE>\fR
-Specify memory to use as stack size \- use if you see: \*(Aqfatal runtime error: stack overflow\*(Aq (default low memory=1048576, high memory=1073741824)
+Deprecated. The walker no longer recurses so a custom stack size is unnecessary. Accepted for compatibility but the value is ignored
 .TP
 \fB\-j\fR, \fB\-\-output\-json\fR
 Output the directory tree as json to the current directory

src/cli.rs

@@ -144,9 +144,8 @@ pub struct Cli {
     #[arg(short, long, value_enum, value_name("FORMAT"), ignore_case(true))]
     pub output_format: Option<OutputFormat>,
 
-    /// Specify memory to use as stack size - use if you see: 'fatal runtime
-    /// error: stack overflow' (default low memory=1048576, high
-    /// memory=1073741824)
+    /// Deprecated. The walker no longer recurses so a custom stack size is
+    /// unnecessary. Accepted for compatibility but the value is ignored.
     #[arg(short('S'), long)]
     pub stack_size: Option<usize>,
 

src/config.rs

@@ -31,7 +31,6 @@ pub struct Config {
     pub disable_progress: Option<bool>,
     pub depth: Option<usize>,
     pub bars_on_right: Option<bool>,
-    pub stack_size: Option<usize>,
     pub threads: Option<usize>,
     pub output_json: Option<bool>,
     pub print_errors: Option<bool>,
@@ -138,14 +137,6 @@ impl Config {
     pub fn get_bars_on_right(&self, options: &Cli) -> bool {
         Some(true) == self.bars_on_right || options.bars_on_right
     }
-    pub fn get_custom_stack_size(&self, options: &Cli) -> Option<usize> {
-        let from_cmd_line = options.stack_size;
-        if from_cmd_line.is_none() {
-            self.stack_size
-        } else {
-            from_cmd_line
-        }
-    }
     pub fn get_threads(&self, options: &Cli) -> Option<usize> {
         let from_cmd_line = options.threads;
         if from_cmd_line.is_none() {