From d3ad1caa15848ae585289434a4e3bba1ddb92a09 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Tue, 29 Apr 2025 15:16:57 +0200 Subject: [PATCH 1/3] Allow threaded zstd usage --- syncoid | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/syncoid b/syncoid index 79ad45f..6ef1cf0 100755 --- a/syncoid +++ b/syncoid @@ -17,6 +17,9 @@ use Capture::Tiny ':all'; my $mbuffer_size = "16M"; my $pvoptions = "-p -t -e -r -b"; +# Get number of CPUs for zstd threading, but leave at least two cores empty +my $ncpus = int(`nproc`); +if ($ncpus > 2) { $ncpus -= 2;} # Blank defaults to use ssh client's default # TODO: Merge into a single "sshflags" option? @@ -1142,10 +1145,10 @@ sub compressargset { decomargs => '-dc', }, 'zstd-fast' => { - rawcmd => 'zstd', - args => '-3', - decomrawcmd => 'zstd', - decomargs => '-dc', + rawcmd => 'zstd', + args => "-T$ncpus -3", + decomrawcmd => 'zstd', + decomargs => "-T$ncpus -dc", }, 'zstdmt-fast' => { rawcmd => 'zstdmt', @@ -1154,10 +1157,10 @@ sub compressargset { decomargs => '-dc', }, 'zstd-slow' => { - rawcmd => 'zstd', - args => '-19', - decomrawcmd => 'zstd', - decomargs => '-dc', + rawcmd => 'zstd', + args => "-T$ncpus -19", + decomrawcmd => 'zstd', + decomargs => "-T$ncpus -dc", }, 'zstdmt-slow' => { rawcmd => 'zstdmt', From 2c009be6cec3c5960e3eb7bee0c00c783c4395f2 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Tue, 29 Apr 2025 16:52:20 +0200 Subject: [PATCH 2/3] add zstd-medium compression parameter --- syncoid | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index 6ef1cf0..15dce0d 100755 --- a/syncoid +++ b/syncoid @@ -1150,6 +1150,12 @@ sub compressargset { decomrawcmd => 'zstd', decomargs => "-T$ncpus -dc", }, + 'zstd-medium' => { + rawcmd => 'zstd', + args => "-T$ncpus -8", + decomrawcmd => 'zstd', + decomargs => "-T$ncpus -dc", + }, 'zstdmt-fast' => { rawcmd => 'zstdmt', args => '-3', @@ -1190,7 +1196,7 @@ sub compressargset { if ($value eq 'default') { $value = $DEFAULT_COMPRESSION; - } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstdmt-fast', 'zstd-slow', 'zstdmt-slow', 'lz4', 'xz', 'lzo', 'default', 'none'))) { + } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-medium', 'zstdmt-fast', 'zstd-slow', 'zstdmt-slow', 'lz4', 'xz', 'lzo', 'default', 'none'))) { writelog('WARN', "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION"); $value = $DEFAULT_COMPRESSION; } @@ -2323,7 +2329,7 @@ syncoid - ZFS snapshot replication tool Options: - --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, zstd-fast, zstdmt-fast, zstd-slow, zstdmt-slow, lz4, xz, lzo (default) & none + --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, zstd-fast, zstd-medium, zstdmt-fast, zstd-slow, zstdmt-slow, lz4, xz, lzo (default) & none --identifier=EXTRA Extra identifier which is included in the snapshot name. Can be used for replicating to multiple targets. --recursive|r Also transfers child datasets --skip-parent Skips syncing of the parent dataset. Does nothing without '--recursive' option. From 0f98814efdac913755cd80d3b0d388aa96f77072 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Tue, 29 Apr 2025 16:52:38 +0200 Subject: [PATCH 3/3] Improve comment about zstd core limit --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 15dce0d..34d254c 100755 --- a/syncoid +++ b/syncoid @@ -17,7 +17,7 @@ use Capture::Tiny ':all'; my $mbuffer_size = "16M"; my $pvoptions = "-p -t -e -r -b"; -# Get number of CPUs for zstd threading, but leave at least two cores empty +# Get number of CPUs for zstd threading, but leave at least two cores empty for system responsiveness my $ncpus = int(`nproc`); if ($ncpus > 2) { $ncpus -= 2;}