feat: add --shared-blocks to count cloned and reflinked blocks once

.github/workflows/CICD.yml

@@ -38,6 +38,33 @@ jobs:
         if: success() || failure() # run regardless of fmt outcome
         run: cargo clippy --locked -- -D warnings
 
+  shared-blocks:
+    name: Shared blocks (${{ matrix.fs }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { fs: btrfs, mkfs: mkfs.btrfs }
+          - { fs: xfs, mkfs: mkfs.xfs -m reflink=1 }
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+      - name: Mount a ${{ matrix.fs }} image
+        run: |
+          sudo apt-get -y update
+          sudo apt-get -y install btrfs-progs xfsprogs
+          truncate -s 1G "$RUNNER_TEMP/fs.img"
+          sudo ${{ matrix.mkfs }} "$RUNNER_TEMP/fs.img"
+          sudo mkdir /mnt/reflink
+          sudo mount -o loop "$RUNNER_TEMP/fs.img" /mnt/reflink
+          sudo chown "$(id -u):$(id -g)" /mnt/reflink
+      - name: Test --shared-blocks on reflinks
+        env:
+          DUST_REFLINK_DIR: /mnt/reflink
+        run: cargo test --locked --test test_flags -- --ignored shared_blocks
+
   build:
     name: Build
     runs-on: ${{ matrix.job.os }}

Cargo.toml

@@ -47,6 +47,15 @@ rustc-hash = "2"
 [target.'cfg(not(target_has_atomic = "64"))'.dependencies]
 portable-atomic = "1.4"
 
+[features]
+# Experimental, macOS: make --shared-blocks map the physical extents of files
+# that may share blocks, so partly rewritten clones are deduplicated too.
+# Linux always works this way.
+exact-shared-blocks = []
+
+[target.'cfg(any(target_os = "macos", target_os = "linux"))'.dependencies]
+libc = "0.2"
+
 [target.'cfg(windows)'.dependencies]
 winapi-util = "0.1"
 filesize = "0.2.0"

README.md

@@ -111,6 +111,7 @@ Usage: dust <dir>
 Usage: dust <dir>  <another_dir> <and_more>
 Usage: dust -p (full-path - Show fullpath of the subdirectories)
 Usage: dust -s (apparent-size - shows the length of the file as opposed to the amount of disk space it uses)
+Usage: dust --shared-blocks (Count blocks shared between copy-on-write clones only once. macOS 14+: files that are full APFS clones of each other, e.g. from `cp -c`; a clone that has since been partly rewritten is still counted in full. Linux: blocks shared through reflinks on btrfs, XFS, OCFS2 and bcachefs, e.g. from `cp --reflink`; slower there, as every file on those filesystems is opened)
 Usage: dust -n 30  (Shows 30 directories instead of the default [default is terminal height])
 Usage: dust -d 3  (Shows 3 levels of subdirectories)
 Usage: dust -D (Show only directories (eg dust -D))

build.rs

@@ -8,6 +8,15 @@ use std::path::Path;
 include!("src/cli.rs");
 
 fn main() -> Result<(), Error> {
+    // `shared_extents`: --shared-blocks counts physical extents in a byte-range
+    // map instead of APFS clone ids. Linux has no clone ids, so it always does.
+    println!("cargo::rustc-check-cfg=cfg(shared_extents)");
+    if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("linux")
+        || std::env::var_os("CARGO_FEATURE_EXACT_SHARED_BLOCKS").is_some()
+    {
+        println!("cargo::rustc-cfg=shared_extents");
+    }
+
     let outdir = "completions";
     let app_name = "dust";
     let mut cmd = Cli::command();

completions/_dust

@@ -79,6 +79,7 @@ m\:"last modified time"))' \
 '--limit-filesystem[Only count the files and directories on the same filesystem as the supplied directory]' \
 '-s[Use file length instead of blocks]' \
 '--apparent-size[Use file length instead of blocks]' \
+'--shared-blocks[Count blocks shared between copy-on-write clones only once\: full APFS clones on macOS 14+ (\`cp -c\`), reflinks on Linux (\`cp --reflink\`)]' \
 '-r[Print tree upside down (biggest highest)]' \
 '--reverse[Print tree upside down (biggest highest)]' \
 '-c[No colors will be printed (Useful for commands like\: watch)]' \

completions/_dust.ps1

@@ -63,6 +63,7 @@ Register-ArgumentCompleter -Native -CommandName 'dust' -ScriptBlock {
             [CompletionResult]::new('--limit-filesystem', '--limit-filesystem', [CompletionResultType]::ParameterName, 'Only count the files and directories on the same filesystem as the supplied directory')
             [CompletionResult]::new('-s', '-s', [CompletionResultType]::ParameterName, 'Use file length instead of blocks')
             [CompletionResult]::new('--apparent-size', '--apparent-size', [CompletionResultType]::ParameterName, 'Use file length instead of blocks')
+            [CompletionResult]::new('--shared-blocks', '--shared-blocks', [CompletionResultType]::ParameterName, 'Count blocks shared between copy-on-write clones only once: full APFS clones on macOS 14+ (`cp -c`), reflinks on Linux (`cp --reflink`)')
             [CompletionResult]::new('-r', '-r', [CompletionResultType]::ParameterName, 'Print tree upside down (biggest highest)')
             [CompletionResult]::new('--reverse', '--reverse', [CompletionResultType]::ParameterName, 'Print tree upside down (biggest highest)')
             [CompletionResult]::new('-c', '-c', [CompletionResultType]::ParameterName, 'No colors will be printed (Useful for commands like: watch)')

completions/dust.bash

@@ -23,7 +23,7 @@ _dust() {
 
     case "${cmd}" in
         dust)
-            opts="-d -T -n -p -X -I -L -x -s -r -c -C -b -B -z -R -f -i -v -e -t -w -P -D -F -o -S -j -M -A -y -m -h -V --depth --threads --config --number-of-lines --full-paths --ignore-directory --ignore-all-in-file --dereference-links --limit-filesystem --apparent-size --reverse --no-colors --force-colors --dim --no-percent-bars --bars-on-right --min-size --screen-reader --skip-total --filecount --ignore-hidden --invert-filter --filter --file-types --terminal-width --no-progress --print-errors --only-dir --only-file --output-format --stack-size --output-json --mtime --atime --ctime --files0-from --files-from --collapse --filetime --help --version"
+            opts="-d -T -n -p -X -I -L -x -s -r -c -C -b -B -z -R -f -i -v -e -t -w -P -D -F -o -S -j -M -A -y -m -h -V --depth --threads --config --number-of-lines --full-paths --ignore-directory --ignore-all-in-file --dereference-links --limit-filesystem --apparent-size --shared-blocks --reverse --no-colors --force-colors --dim --no-percent-bars --bars-on-right --min-size --screen-reader --skip-total --filecount --ignore-hidden --invert-filter --filter --file-types --terminal-width --no-progress --print-errors --only-dir --only-file --output-format --stack-size --output-json --mtime --atime --ctime --files0-from --files-from --collapse --filetime --help --version"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0

completions/dust.elv

@@ -60,6 +60,7 @@ set edit:completion:arg-completer[dust] = {|@words|
             cand --limit-filesystem 'Only count the files and directories on the same filesystem as the supplied directory'
             cand -s 'Use file length instead of blocks'
             cand --apparent-size 'Use file length instead of blocks'
+            cand --shared-blocks 'Count blocks shared between copy-on-write clones only once: full APFS clones on macOS 14+ (`cp -c`), reflinks on Linux (`cp --reflink`)'
             cand -r 'Print tree upside down (biggest highest)'
             cand --reverse 'Print tree upside down (biggest highest)'
             cand -c 'No colors will be printed (Useful for commands like: watch)'

completions/dust.fish

@@ -32,6 +32,7 @@ complete -c dust -s p -l full-paths -d 'Subdirectories will not have their path
 complete -c dust -s L -l dereference-links -d 'dereference sym links - Treat sym links as directories and go into them'
 complete -c dust -s x -l limit-filesystem -d 'Only count the files and directories on the same filesystem as the supplied directory'
 complete -c dust -s s -l apparent-size -d 'Use file length instead of blocks'
+complete -c dust -l shared-blocks -d 'Count blocks shared between copy-on-write clones only once: full APFS clones on macOS 14+ (`cp -c`), reflinks on Linux (`cp --reflink`)'
 complete -c dust -s r -l reverse -d 'Print tree upside down (biggest highest)'
 complete -c dust -s c -l no-colors -d 'No colors will be printed (Useful for commands like: watch)'
 complete -c dust -s C -l force-colors -d 'Force colors print'

config/config.toml

@@ -27,6 +27,10 @@ ignore-hidden=true
 # Only count files and directories on the same filesystem as the supplied directory
 limit-filesystem=true
 
+# Count blocks shared between copy-on-write clones only once (APFS clones on
+# macOS 14+, btrfs/XFS reflinks on Linux)
+shared-blocks=true
+
 # print sizes in powers of 1000 (e.g., 1.1G)
 output-format="si"
 

man-page/dust.1

@@ -4,7 +4,7 @@
 .SH NAME
 Dust \- Like du but more intuitive
 .SH SYNOPSIS
-\fBdust\fR [\fB\-d\fR|\fB\-\-depth\fR] [\fB\-T\fR|\fB\-\-threads\fR] [\fB\-\-config\fR] [\fB\-n\fR|\fB\-\-number\-of\-lines\fR] [\fB\-p\fR|\fB\-\-full\-paths\fR] [\fB\-X\fR|\fB\-\-ignore\-directory\fR] [\fB\-I\fR|\fB\-\-ignore\-all\-in\-file\fR] [\fB\-L\fR|\fB\-\-dereference\-links\fR] [\fB\-x\fR|\fB\-\-limit\-filesystem\fR] [\fB\-s\fR|\fB\-\-apparent\-size\fR] [\fB\-r\fR|\fB\-\-reverse\fR] [\fB\-c\fR|\fB\-\-no\-colors\fR] [\fB\-C\fR|\fB\-\-force\-colors\fR] [\fB\-\-dim\fR] [\fB\-b\fR|\fB\-\-no\-percent\-bars\fR] [\fB\-B\fR|\fB\-\-bars\-on\-right\fR] [\fB\-z\fR|\fB\-\-min\-size\fR] [\fB\-R\fR|\fB\-\-screen\-reader\fR] [\fB\-\-skip\-total\fR] [\fB\-f\fR|\fB\-\-filecount\fR] [\fB\-i\fR|\fB\-\-ignore\-hidden\fR] [\fB\-v\fR|\fB\-\-invert\-filter\fR] [\fB\-e\fR|\fB\-\-filter\fR] [\fB\-t\fR|\fB\-\-file\-types\fR] [\fB\-w\fR|\fB\-\-terminal\-width\fR] [\fB\-P\fR|\fB\-\-no\-progress\fR] [\fB\-\-print\-errors\fR] [\fB\-D\fR|\fB\-\-only\-dir\fR] [\fB\-F\fR|\fB\-\-only\-file\fR] [\fB\-o\fR|\fB\-\-output\-format\fR] [\fB\-S\fR|\fB\-\-stack\-size\fR] [\fB\-j\fR|\fB\-\-output\-json\fR] [\fB\-M\fR|\fB\-\-mtime\fR] [\fB\-A\fR|\fB\-\-atime\fR] [\fB\-y\fR|\fB\-\-ctime\fR] [\fB\-\-files0\-from\fR] [\fB\-\-files\-from\fR] [\fB\-\-collapse\fR] [\fB\-m\fR|\fB\-\-filetime\fR] [\fB\-h\fR|\fB\-\-help\fR] [\fB\-V\fR|\fB\-\-version\fR] [\fIPATH\fR] 
+\fBdust\fR [\fB\-d\fR|\fB\-\-depth\fR] [\fB\-T\fR|\fB\-\-threads\fR] [\fB\-\-config\fR] [\fB\-n\fR|\fB\-\-number\-of\-lines\fR] [\fB\-p\fR|\fB\-\-full\-paths\fR] [\fB\-X\fR|\fB\-\-ignore\-directory\fR] [\fB\-I\fR|\fB\-\-ignore\-all\-in\-file\fR] [\fB\-L\fR|\fB\-\-dereference\-links\fR] [\fB\-x\fR|\fB\-\-limit\-filesystem\fR] [\fB\-s\fR|\fB\-\-apparent\-size\fR] [\fB\-\-shared\-blocks\fR] [\fB\-r\fR|\fB\-\-reverse\fR] [\fB\-c\fR|\fB\-\-no\-colors\fR] [\fB\-C\fR|\fB\-\-force\-colors\fR] [\fB\-\-dim\fR] [\fB\-b\fR|\fB\-\-no\-percent\-bars\fR] [\fB\-B\fR|\fB\-\-bars\-on\-right\fR] [\fB\-z\fR|\fB\-\-min\-size\fR] [\fB\-R\fR|\fB\-\-screen\-reader\fR] [\fB\-\-skip\-total\fR] [\fB\-f\fR|\fB\-\-filecount\fR] [\fB\-i\fR|\fB\-\-ignore\-hidden\fR] [\fB\-v\fR|\fB\-\-invert\-filter\fR] [\fB\-e\fR|\fB\-\-filter\fR] [\fB\-t\fR|\fB\-\-file\-types\fR] [\fB\-w\fR|\fB\-\-terminal\-width\fR] [\fB\-P\fR|\fB\-\-no\-progress\fR] [\fB\-\-print\-errors\fR] [\fB\-D\fR|\fB\-\-only\-dir\fR] [\fB\-F\fR|\fB\-\-only\-file\fR] [\fB\-o\fR|\fB\-\-output\-format\fR] [\fB\-S\fR|\fB\-\-stack\-size\fR] [\fB\-j\fR|\fB\-\-output\-json\fR] [\fB\-M\fR|\fB\-\-mtime\fR] [\fB\-A\fR|\fB\-\-atime\fR] [\fB\-y\fR|\fB\-\-ctime\fR] [\fB\-\-files0\-from\fR] [\fB\-\-files\-from\fR] [\fB\-\-collapse\fR] [\fB\-m\fR|\fB\-\-filetime\fR] [\fB\-h\fR|\fB\-\-help\fR] [\fB\-V\fR|\fB\-\-version\fR] [\fIPATH\fR] 
 .SH DESCRIPTION
 Like du but more intuitive
 .SH OPTIONS
@@ -39,6 +39,9 @@ Only count the files and directories on the same filesystem as the supplied dire
 \fB\-s\fR, \fB\-\-apparent\-size\fR
 Use file length instead of blocks
 .TP
+\fB\-\-shared\-blocks\fR
+Count blocks shared between copy\-on\-write clones only once: full APFS clones on macOS 14+ (`cp \-c`), reflinks on Linux (`cp \-\-reflink`)
+.TP
 \fB\-r\fR, \fB\-\-reverse\fR
 Print tree upside down (biggest highest)
 .TP

src/cli.rs

@@ -52,6 +52,11 @@ pub struct Cli {
     #[arg(short('s'), long)]
     pub apparent_size: bool,
 
+    /// Count blocks shared between copy-on-write clones only once: full APFS
+    /// clones on macOS 14+ (`cp -c`), reflinks on Linux (`cp --reflink`)
+    #[arg(long)]
+    pub shared_blocks: bool,
+
     /// Print tree upside down (biggest highest)
     #[arg(short, long)]
     pub reverse: bool,

src/config.rs

@@ -27,6 +27,7 @@ pub struct Config {
     pub screen_reader: Option<bool>,
     pub ignore_hidden: Option<bool>,
     pub limit_filesystem: Option<bool>,
+    pub shared_blocks: Option<bool>,
     pub output_format: Option<String>,
     pub min_size: Option<String>,
     pub only_dir: Option<bool>,
@@ -79,6 +80,9 @@ impl Config {
     pub fn get_limit_filesystem(&self, options: &Cli) -> bool {
         Some(true) == self.limit_filesystem || options.limit_filesystem
     }
+    pub fn get_shared_blocks(&self, options: &Cli) -> bool {
+        Some(true) == self.shared_blocks || options.shared_blocks
+    }
     pub fn get_full_paths(&self, options: &Cli) -> bool {
         Some(true) == self.display_full_paths || options.full_paths
     }