From f89372967f8975e6cf0e5bfbf8052186ad4133e0 Mon Sep 17 00:00:00 2001 From: Adam Fulton Date: Mon, 1 Apr 2024 11:53:45 -0500 Subject: [PATCH 1/9] fix(syncoid): regather $snaps on --delete-target-snapshots flag --- syncoid | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/syncoid b/syncoid index 61814d2..ea067b7 100755 --- a/syncoid +++ b/syncoid @@ -865,6 +865,16 @@ sub syncdataset { # those that exist on the source. Remaining are the snapshots # that are only on the target. Then sort to remove the oldest # snapshots first. + + # regather snapshots on source and target + %snaps = getsnaps('source',$sourcehost,$sourcefs,$sourceisroot); + + if ($targetexists) { + my %targetsnaps = getsnaps('target',$targethost,$targetfs,$targetisroot); + my %sourcesnaps = %snaps; + %snaps = (%sourcesnaps, %targetsnaps); + } + my @to_delete = sort { sortsnapshots(\%snaps, $a, $b) } grep {!exists $snaps{'source'}{$_}} keys %{ $snaps{'target'} }; while (@to_delete) { # Create batch of snapshots to remove From d08b2882b7255ba630f5d338936d411a3d56e44c Mon Sep 17 00:00:00 2001 From: Adam Fulton Date: Mon, 1 Apr 2024 13:16:16 -0500 Subject: [PATCH 2/9] finish rebase to master --- syncoid | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/syncoid b/syncoid index ea067b7..cb18897 100755 --- a/syncoid +++ b/syncoid @@ -867,13 +867,13 @@ sub syncdataset { # snapshots first. # regather snapshots on source and target - %snaps = getsnaps('source',$sourcehost,$sourcefs,$sourceisroot); + %snaps = getsnaps('source',$sourcehost,$sourcefs,$sourceisroot,0); - if ($targetexists) { - my %targetsnaps = getsnaps('target',$targethost,$targetfs,$targetisroot); - my %sourcesnaps = %snaps; - %snaps = (%sourcesnaps, %targetsnaps); - } + if ($targetexists) { + my %targetsnaps = getsnaps('target',$targethost,$targetfs,$targetisroot,0); + my %sourcesnaps = %snaps; + %snaps = (%sourcesnaps, %targetsnaps); + } my @to_delete = sort { sortsnapshots(\%snaps, $a, $b) } grep {!exists $snaps{'source'}{$_}} keys %{ $snaps{'target'} }; while (@to_delete) { From 7c8a34eceb40043bc5a09990b668e67235a5e81b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 5 Apr 2024 15:20:28 +0200 Subject: [PATCH 3/9] * proper order of tests * timing fixes for fast NVME pools * skip invasive tests by default --- tests/run-tests.sh | 5 ++++- .../run.sh | 0 .../run.sh | 0 tests/syncoid/{3_force_delete => 003_force_delete}/run.sh | 0 .../run.sh | 0 .../run.sh | 2 ++ .../run.sh | 2 ++ .../run.sh | 0 .../run.sh | 0 .../run.sh | 0 .../syncoid/{10_filter_snaps => 010_filter_snaps}/run.sh | 0 .../run.sh | 8 ++++++-- tests/syncoid/run-tests.sh | 5 ++++- 13 files changed, 18 insertions(+), 4 deletions(-) rename tests/syncoid/{1_bookmark_replication_intermediate => 001_bookmark_replication_intermediate}/run.sh (100%) rename tests/syncoid/{2_bookmark_replication_no_intermediate => 002_bookmark_replication_no_intermediate}/run.sh (100%) rename tests/syncoid/{3_force_delete => 003_force_delete}/run.sh (100%) rename tests/syncoid/{4_bookmark_replication_edge_case => 004_bookmark_replication_edge_case}/run.sh (100%) rename tests/syncoid/{5_reset_resume_state => 005_reset_resume_state}/run.sh (99%) rename tests/syncoid/{6_reset_resume_state2 => 006_reset_resume_state2}/run.sh (99%) rename tests/syncoid/{7_preserve_recordsize => 007_preserve_recordsize}/run.sh (100%) rename tests/syncoid/{8_force_delete_snapshot => 008_force_delete_snapshot}/run.sh (100%) rename tests/syncoid/{9_preserve_properties => 009_preserve_properties}/run.sh (100%) rename tests/syncoid/{10_filter_snaps => 010_filter_snaps}/run.sh (100%) rename tests/syncoid/{815_sync_out-of-order_snapshots => 011_sync_out-of-order_snapshots}/run.sh (91%) diff --git a/tests/run-tests.sh b/tests/run-tests.sh index 418657c..ec14721 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -17,8 +17,11 @@ for test in $(find . -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -g); cd "${test}" echo -n y | bash run.sh > "${LOGFILE}" 2>&1 - if [ $? -eq 0 ]; then + ret=$? + if [ $ret -eq 0 ]; then echo "[PASS]" + elif [ $ret -eq 130 ]; then + echo "[SKIPPED]" else echo "[FAILED] (see ${LOGFILE})" fi diff --git a/tests/syncoid/1_bookmark_replication_intermediate/run.sh b/tests/syncoid/001_bookmark_replication_intermediate/run.sh similarity index 100% rename from tests/syncoid/1_bookmark_replication_intermediate/run.sh rename to tests/syncoid/001_bookmark_replication_intermediate/run.sh diff --git a/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh b/tests/syncoid/002_bookmark_replication_no_intermediate/run.sh similarity index 100% rename from tests/syncoid/2_bookmark_replication_no_intermediate/run.sh rename to tests/syncoid/002_bookmark_replication_no_intermediate/run.sh diff --git a/tests/syncoid/3_force_delete/run.sh b/tests/syncoid/003_force_delete/run.sh similarity index 100% rename from tests/syncoid/3_force_delete/run.sh rename to tests/syncoid/003_force_delete/run.sh diff --git a/tests/syncoid/4_bookmark_replication_edge_case/run.sh b/tests/syncoid/004_bookmark_replication_edge_case/run.sh similarity index 100% rename from tests/syncoid/4_bookmark_replication_edge_case/run.sh rename to tests/syncoid/004_bookmark_replication_edge_case/run.sh diff --git a/tests/syncoid/5_reset_resume_state/run.sh b/tests/syncoid/005_reset_resume_state/run.sh similarity index 99% rename from tests/syncoid/5_reset_resume_state/run.sh rename to tests/syncoid/005_reset_resume_state/run.sh index 43ec78f..4eb4af6 100755 --- a/tests/syncoid/5_reset_resume_state/run.sh +++ b/tests/syncoid/005_reset_resume_state/run.sh @@ -28,6 +28,8 @@ zfs create -o mountpoint="${MOUNT_TARGET}" "${POOL_NAME}"/src dd if=/dev/urandom of="${MOUNT_TARGET}"/big_file bs=1M count=200 +sleep 1 + ../../../syncoid --debug --compress=none --source-bwlimit=2m "${POOL_NAME}"/src "${POOL_NAME}"/dst & syncoid_pid=$! sleep 5 diff --git a/tests/syncoid/6_reset_resume_state2/run.sh b/tests/syncoid/006_reset_resume_state2/run.sh similarity index 99% rename from tests/syncoid/6_reset_resume_state2/run.sh rename to tests/syncoid/006_reset_resume_state2/run.sh index d05696b..c568fd4 100755 --- a/tests/syncoid/6_reset_resume_state2/run.sh +++ b/tests/syncoid/006_reset_resume_state2/run.sh @@ -28,6 +28,8 @@ zfs create -o mountpoint="${MOUNT_TARGET}" "${POOL_NAME}"/src dd if=/dev/urandom of="${MOUNT_TARGET}"/big_file bs=1M count=200 +sleep 1 + zfs snapshot "${POOL_NAME}"/src@big ../../../syncoid --debug --no-sync-snap --compress=none --source-bwlimit=2m "${POOL_NAME}"/src "${POOL_NAME}"/dst & syncoid_pid=$! diff --git a/tests/syncoid/7_preserve_recordsize/run.sh b/tests/syncoid/007_preserve_recordsize/run.sh similarity index 100% rename from tests/syncoid/7_preserve_recordsize/run.sh rename to tests/syncoid/007_preserve_recordsize/run.sh diff --git a/tests/syncoid/8_force_delete_snapshot/run.sh b/tests/syncoid/008_force_delete_snapshot/run.sh similarity index 100% rename from tests/syncoid/8_force_delete_snapshot/run.sh rename to tests/syncoid/008_force_delete_snapshot/run.sh diff --git a/tests/syncoid/9_preserve_properties/run.sh b/tests/syncoid/009_preserve_properties/run.sh similarity index 100% rename from tests/syncoid/9_preserve_properties/run.sh rename to tests/syncoid/009_preserve_properties/run.sh diff --git a/tests/syncoid/10_filter_snaps/run.sh b/tests/syncoid/010_filter_snaps/run.sh similarity index 100% rename from tests/syncoid/10_filter_snaps/run.sh rename to tests/syncoid/010_filter_snaps/run.sh diff --git a/tests/syncoid/815_sync_out-of-order_snapshots/run.sh b/tests/syncoid/011_sync_out-of-order_snapshots/run.sh similarity index 91% rename from tests/syncoid/815_sync_out-of-order_snapshots/run.sh rename to tests/syncoid/011_sync_out-of-order_snapshots/run.sh index af67b36..af87979 100755 --- a/tests/syncoid/815_sync_out-of-order_snapshots/run.sh +++ b/tests/syncoid/011_sync_out-of-order_snapshots/run.sh @@ -7,9 +7,13 @@ set -e . ../../common/lib.sh -POOL_IMAGE="/tmp/jimsalterjrs_sanoid_815.img" +if [ -z "$ALLOW_INVASIVE_TESTS" ]; then + exit 130 +fi +exit 0 +POOL_IMAGE="/tmp/syncoid-test-11.zpool" POOL_SIZE="64M" -POOL_NAME="jimsalterjrs_sanoid_815" +POOL_NAME="syncoid-test-11" truncate -s "${POOL_SIZE}" "${POOL_IMAGE}" diff --git a/tests/syncoid/run-tests.sh b/tests/syncoid/run-tests.sh index 5564667..8307413 100755 --- a/tests/syncoid/run-tests.sh +++ b/tests/syncoid/run-tests.sh @@ -17,8 +17,11 @@ for test in $(find . -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -g); cd "${test}" echo | bash run.sh > "${LOGFILE}" 2>&1 - if [ $? -eq 0 ]; then + ret=$? + if [ $ret -eq 0 ]; then echo "[PASS]" + elif [ $ret -eq 130 ]; then + echo "[SKIPPED]" else echo "[FAILED] (see ${LOGFILE})" fi From 4e86733c1a618b2084046a0950a7bbf2bf6b88e9 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 5 Apr 2024 15:22:13 +0200 Subject: [PATCH 4/9] missed debug statement --- tests/syncoid/011_sync_out-of-order_snapshots/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/syncoid/011_sync_out-of-order_snapshots/run.sh b/tests/syncoid/011_sync_out-of-order_snapshots/run.sh index af87979..bb96ad0 100755 --- a/tests/syncoid/011_sync_out-of-order_snapshots/run.sh +++ b/tests/syncoid/011_sync_out-of-order_snapshots/run.sh @@ -10,7 +10,7 @@ set -e if [ -z "$ALLOW_INVASIVE_TESTS" ]; then exit 130 fi -exit 0 + POOL_IMAGE="/tmp/syncoid-test-11.zpool" POOL_SIZE="64M" POOL_NAME="syncoid-test-11" From d7ed4bdf540de61995e7377103395a1534ee905c Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 5 Apr 2024 15:24:42 +0200 Subject: [PATCH 5/9] support relative paths --- findoid | 3 +++ 1 file changed, 3 insertions(+) diff --git a/findoid b/findoid index 0bb5e5f..2561246 100755 --- a/findoid +++ b/findoid @@ -25,6 +25,9 @@ if ($args{'path'} eq '') { } } +# resolve given path to a canonical one +$args{'path'} = Cwd::realpath($args{'path'}); + my $dataset = getdataset($args{'path'}); my %versions = getversions($args{'path'}, $dataset); From eb4fe8a01cf1916d275bb809247cb0744dc3b33f Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 18 Apr 2024 07:42:47 +0200 Subject: [PATCH 6/9] added missing status information about what is done and provide more details --- syncoid | 71 +++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 14 deletions(-) diff --git a/syncoid b/syncoid index a1cf416..2be2ab0 100755 --- a/syncoid +++ b/syncoid @@ -498,7 +498,6 @@ sub syncdataset { my $ret; if (defined $origin) { - writelog('INFO', "Clone is recreated on target $targetfs based on $origin"); ($ret, $stdout) = syncclone($sourcehost, $sourcefs, $origin, $targethost, $targetfs, $oldestsnap); if ($ret) { writelog('INFO', "clone creation failed, trying ordinary replication as fallback"); @@ -506,12 +505,6 @@ sub syncdataset { return 0; } } else { - if (!defined ($args{'no-stream'}) ) { - writelog('INFO', "Sending oldest full snapshot $sourcefs\@$oldestsnap to new target filesystem:"); - } else { - writelog('INFO', "--no-stream selected; sending newest full snapshot $sourcefs\@$oldestsnap to new target filesystem:"); - } - ($ret, $stdout) = syncfull($sourcehost, $sourcefs, $targethost, $targetfs, $oldestsnap); } @@ -532,8 +525,6 @@ sub syncdataset { # $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); - writelog('INFO', "Updating new target filesystem with incremental $sourcefs\@$oldestsnap ... $newsyncsnap:"); - (my $ret, $stdout) = syncincremental($sourcehost, $sourcefs, $targethost, $targetfs, $oldestsnap, $newsyncsnap, 0); if ($ret != 0) { @@ -898,7 +889,6 @@ sub runsynccmd { my $disp_pvsize = $pvsize == 0 ? 'UNKNOWN' : readablebytes($pvsize); my $sendoptions; if ($sendsource =~ / -t /) { - writelog('INFO', "Resuming interrupted zfs send/receive from $sourcefs to $targetfs (~ $disp_pvsize remaining):"); $sendoptions = getoptionsline(\@sendoptions, ('P','V','e','v')); } elsif ($sendsource =~ /#/) { $sendoptions = getoptionsline(\@sendoptions, ('L','V','c','e','w')); @@ -934,12 +924,13 @@ sub runsynccmd { my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $targetfsescaped 2>&1"; my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); - writelog('INFO', "Sync size: ~$disp_pvsize"); + writelog('DEBUG', "sync size: ~$disp_pvsize"); writelog('DEBUG', "$synccmd"); # make sure target is (still) not currently in receive. if (iszfsbusy($targethost,$targetfs,$targetisroot)) { - writelog('WARN', "Cannot sync now: $targetfs is already target of a zfs receive process."); + my $targetname = buildnicename($targethost, $targetfs); + writelog('WARN', "Cannot sync now: $targetname is already target of a zfs receive process."); return (1, ''); } @@ -971,6 +962,16 @@ sub syncfull { my $sendsource = "$sourcefsescaped\@$snapescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$snapname",0,$sourceisroot); + my $srcname = buildnicename($sourcehost, $sourcefs, $snapname); + my $targetname = buildnicename($targethost, $targetfs); + my $disp_pvsize = $pvsize == 0 ? 'UNKNOWN' : readablebytes($pvsize); + + if (!defined ($args{'no-stream'}) ) { + writelog('INFO', "Sending oldest full snapshot $srcname to new target filesystem $targetname (~ $disp_pvsize):"); + } else { + writelog('INFO', "--no-stream selected; sending newest full snapshot $srcname to new target filesystem $targetname: (~ $disp_pvsize)"); + } + return runsynccmd($sourcehost, $sourcefs, $sendsource, $targethost, $targetfs, $pvsize); } # end syncfull() @@ -1011,7 +1012,6 @@ sub syncincremental { foreach my $i (0..(scalar(@intsnaps) - 2)) { my $snapa = $intsnaps[$i]; my $snapb = $intsnaps[$i + 1]; - writelog('INFO', "Performing an incremental sync between '$snapa' and '$snapb'"); syncincremental($sourcehost, $sourcefs, $targethost, $targetfs, $snapa, $snapb, 1) == 0 or return $?; } @@ -1026,6 +1026,12 @@ sub syncincremental { my $sendsource = "$streamarg $sourcefsescaped\@$fromsnapescaped $sourcefsescaped\@$tosnapescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$fromsnap","$sourcefs\@$tosnap",$sourceisroot); + my $srcname = buildnicename($sourcehost, $sourcefs, $fromsnap); + my $targetname = buildnicename($targethost, $targetfs); + my $disp_pvsize = $pvsize == 0 ? 'UNKNOWN' : readablebytes($pvsize); + + writelog('INFO', "Sending incremental $srcname ... $tosnap to $targetname (~ $disp_pvsize):"); + return runsynccmd($sourcehost, $sourcefs, $sendsource, $targethost, $targetfs, $pvsize); } # end syncincremental() @@ -1038,6 +1044,12 @@ sub syncclone { my $sendsource = "-i $originescaped $sourcefsescaped\@$tosnapescaped"; my $pvsize = getsendsize($sourcehost,$origin,"$sourcefs\@$tosnap",$sourceisroot); + my $srcname = buildnicename($sourcehost, $origin); + my $targetname = buildnicename($targethost, $targetfs); + my $disp_pvsize = $pvsize == 0 ? 'UNKNOWN' : readablebytes($pvsize); + + writelog('INFO', "Clone is recreated on target $targetname based on $srcname (~ $disp_pvsize):"); + return runsynccmd($sourcehost, $sourcefs, $sendsource, $targethost, $targetfs, $pvsize); } # end syncclone() @@ -1047,6 +1059,12 @@ sub syncresume { my $sendsource = "-t $receivetoken"; my $pvsize = getsendsize($sourcehost,"","",$sourceisroot,$receivetoken); + my $srcname = buildnicename($sourcehost, $sourcefs); + my $targetname = buildnicename($targethost, $targetfs); + my $disp_pvsize = $pvsize == 0 ? 'UNKNOWN' : readablebytes($pvsize); + + writelog('INFO', "Resuming interrupted zfs send/receive from $srcname to $targetname (~ $disp_pvsize remaining):"); + return runsynccmd($sourcehost, $sourcefs, $sendsource, $targethost, $targetfs, $pvsize); } # end syncresume() @@ -1058,6 +1076,11 @@ sub syncbookmark { my $tosnapescaped = escapeshellparam($tosnap); my $sendsource = "-i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$tosnapescaped"; + my $srcname = buildnicename($sourcehost, $sourcefs, '', $bookmark); + my $targetname = buildnicename($targethost, $targetfs); + + writelog('INFO', "Sending incremental $srcname ... $tosnap to $targetname:"); + return runsynccmd($sourcehost, $sourcefs, $sendsource, $targethost, $targetfs, 0); } # end syncbookmark @@ -1507,7 +1530,7 @@ sub getnewestsnapshot { my $snaps = shift; foreach my $snap (sort { sortsnapshots($snaps, $b, $a) } keys %{ $snaps{'source'} }) { # return on first snap found - it's the newest - writelog('INFO', "NEWEST SNAPSHOT: $snap"); + writelog('DEBUG', "NEWEST SNAPSHOT: $snap"); return $snap; } # must not have had any snapshots on source - looks like we'd better create one! @@ -2233,6 +2256,26 @@ sub snapisincluded { return 1; } +sub buildnicename { + my ($host,$fs,$snapname,$bookmarkname) = @_; + + my $name; + if ($host) { + $host =~ s/-S \/tmp\/syncoid[a-zA-Z0-9-@]+ //g; + $name = "$host:$fs"; + } else { + $name = "$fs"; + } + + if ($snapname) { + $name = "$name\@$snapname"; + } elsif ($bookmarkname) { + $name = "$name#$bookmarkname"; + } + + return $name; +} + __END__ =head1 NAME From 8b7d29d5a030d8620ceefc7822b5226d9729c71a Mon Sep 17 00:00:00 2001 From: 0xFelix Date: Sat, 20 Apr 2024 18:41:43 +0200 Subject: [PATCH 7/9] syncoid: Add zstdmt compress options Add the zstdmt-fast and zstdmt-slow compress options to allow use of multithreading when using zstd compression. Signed-off-by: 0xFelix --- syncoid | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index b57aa43..9f9eb89 100755 --- a/syncoid +++ b/syncoid @@ -1114,12 +1114,24 @@ sub compressargset { decomrawcmd => 'zstd', decomargs => '-dc', }, + 'zstdmt-fast' => { + rawcmd => 'zstdmt', + args => '-3', + decomrawcmd => 'zstdmt', + decomargs => '-dc', + }, 'zstd-slow' => { rawcmd => 'zstd', args => '-19', decomrawcmd => 'zstd', decomargs => '-dc', }, + 'zstdmt-slow' => { + rawcmd => 'zstdmt', + args => '-19', + decomrawcmd => 'zstdmt', + decomargs => '-dc', + }, 'xz' => { rawcmd => 'xz', args => '', @@ -1142,7 +1154,7 @@ sub compressargset { if ($value eq 'default') { $value = $DEFAULT_COMPRESSION; - } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lz4', 'xz', 'lzo', 'default', 'none'))) { + } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstdmt-fast', 'zstd-slow', 'zstdmt-slow', 'lz4', 'xz', 'lzo', 'default', 'none'))) { writelog('WARN', "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION"); $value = $DEFAULT_COMPRESSION; } @@ -2255,7 +2267,7 @@ syncoid - ZFS snapshot replication tool Options: - --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, zstd-fast, zstd-slow, lz4, xz, lzo (default) & none + --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, zstd-fast, zstdmt-fast, zstd-slow, zstdmt-slow, lz4, xz, lzo (default) & none --identifier=EXTRA Extra identifier which is included in the snapshot name. Can be used for replicating to multiple targets. --recursive|r Also transfers child datasets --skip-parent Skips syncing of the parent dataset. Does nothing without '--recursive' option. From 6f74c7c4b39a7ab35d671e8df6f26919cc347e12 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 23 Apr 2024 23:38:47 +0200 Subject: [PATCH 8/9] * improve performance (especially for monitor commands) by caching the dataset list * list snapshots only when needed --- sanoid | 145 +++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 120 insertions(+), 25 deletions(-) diff --git a/sanoid b/sanoid index 295957b..533f9ea 100755 --- a/sanoid +++ b/sanoid @@ -35,17 +35,6 @@ if (keys %args < 4) { $args{'verbose'} = 1; } - -my $cacheTTL = 900; # 15 minutes - -# Allow a much older snapshot cache file than default if _only_ "--monitor-*" action commands are given -# (ignore "--verbose", "--configdir" etc) -if (($args{'monitor-snapshots'} || $args{'monitor-health'} || $args{'monitor-capacity'}) && ! ($args{'cron'} || $args{'force-update'} || $args{'take-snapshots'} || $args{'prune-snapshots'} || $args{'force-prune'})) { - # The command combination above must not assert true for any command that takes or prunes snapshots - $cacheTTL = 18000; # 5 hours - if ($args{'debug'}) { print "DEBUG: command combo means that the cache file (provided it exists) will be allowed to be older than default.\n"; } -} - # for compatibility reasons, older versions used hardcoded command paths $ENV{'PATH'} = $ENV{'PATH'} . ":/bin:/sbin"; @@ -57,25 +46,70 @@ my $zpool = 'zpool'; my $conf_file = "$args{'configdir'}/sanoid.conf"; my $default_conf_file = "$args{'configdir'}/sanoid.defaults.conf"; -# parse config file -my %config = init($conf_file,$default_conf_file); - my $cache_dir = $args{'cache-dir'}; my $run_dir = $args{'run-dir'}; make_path($cache_dir); make_path($run_dir); -# if we call getsnaps(%config,1) it will forcibly update the cache, TTL or no TTL -my $forcecacheupdate = 0; +my $cacheTTL = 1200; # 20 minutes + +# Allow a much older snapshot cache file than default if _only_ "--monitor-*" action commands are given +# (ignore "--verbose", "--configdir" etc) +if ( + ( + $args{'monitor-snapshots'} + || $args{'monitor-health'} + || $args{'monitor-capacity'} + ) && ! ( + $args{'cron'} + || $args{'force-update'} + || $args{'take-snapshots'} + || $args{'prune-snapshots'} + || $args{'force-prune'} + ) +) { + # The command combination above must not assert true for any command that takes or prunes snapshots + $cacheTTL = 18000; # 5 hours + if ($args{'debug'}) { print "DEBUG: command combo means that the cache file (provided it exists) will be allowed to be older than default.\n"; } +} + +# snapshot cache my $cache = "$cache_dir/snapshots.txt"; -my %snaps = getsnaps( \%config, $cacheTTL, $forcecacheupdate ); + +# configured dataset cache +my $cachedatasetspath = "$cache_dir/datasets.txt"; +my @cachedatasets; + +# parse config file +my %config = init($conf_file,$default_conf_file); + my %pruned; my %capacitycache; -my %snapsbytype = getsnapsbytype( \%config, \%snaps ); +my %snaps; +my %snapsbytype; +my %snapsbypath; -my %snapsbypath = getsnapsbypath( \%config, \%snaps ); +# get snapshot list only if needed +if ($args{'monitor-snapshots'} + || $args{'monitor-health'} + || $args{'cron'} + || $args{'take-snapshots'} + || $args{'prune-snapshots'} + || $args{'force-update'} + || $args{'debug'} +) { + my $forcecacheupdate = 0; + if ($args{'force-update'}) { + $forcecacheupdate = 1; + } + + %snaps = getsnaps( \%config, $cacheTTL, $forcecacheupdate); + + %snapsbytype = getsnapsbytype( \%config, \%snaps ); + %snapsbypath = getsnapsbypath( \%config, \%snaps ); +} # let's make it a little easier to be consistent passing these hashes in the same order to each sub my @params = ( \%config, \%snaps, \%snapsbytype, \%snapsbypath ); @@ -84,7 +118,6 @@ if ($args{'debug'}) { $args{'verbose'}=1; blabber (@params); } if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); } if ($args{'monitor-health'}) { monitor_health(@params); } if ($args{'monitor-capacity'}) { monitor_capacity(@params); } -if ($args{'force-update'}) { my $snaps = getsnaps( \%config, $cacheTTL, 1 ); } if ($args{'cron'}) { if ($args{'quiet'}) { $args{'verbose'} = 0; } @@ -275,7 +308,6 @@ sub prune_snapshots { my ($config, $snaps, $snapsbytype, $snapsbypath) = @_; my %datestamp = get_date(); - my $forcecacheupdate = 0; foreach my $section (keys %config) { if ($section =~ /^template/) { next; } @@ -826,7 +858,7 @@ sub getsnaps { if (checklock('sanoid_cacheupdate')) { writelock('sanoid_cacheupdate'); if ($args{'verbose'}) { - if ($args{'force-update'}) { + if ($forcecacheupdate) { print "INFO: cache forcibly expired - updating from zfs list.\n"; } else { print "INFO: cache expired - updating from zfs list.\n"; @@ -901,6 +933,20 @@ sub init { die "FATAL: you're using sanoid.defaults.conf v$defaults_version, this version of sanoid requires a minimum sanoid.defaults.conf v$MINIMUM_DEFAULTS_VERSION"; } + my @updatedatasets; + + # load dataset cache if valid + if (!$args{'force-update'} && -f $cachedatasetspath) { + my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, $mtime, $ctime, $blksize, $blocks) = stat($cachedatasetspath); + + if ((time() - $mtime) <= $cacheTTL) { + if ($args{'debug'}) { print "DEBUG: dataset cache not expired (" . (time() - $mtime) . " seconds old with TTL of $cacheTTL): pulling dataset list from cache.\n"; } + open FH, "< $cachedatasetspath"; + @cachedatasets = ; + close FH; + } + } + foreach my $section (keys %ini) { # first up - die with honor if unknown parameters are set in any modules or templates by the user. @@ -990,6 +1036,10 @@ sub init { $config{$section}{'path'} = $section; } + if (! @cachedatasets) { + push (@updatedatasets, "$config{$section}{'path'}\n"); + } + # how 'bout some recursion? =) if ($config{$section}{'zfs_recursion'} && $config{$section}{'zfs_recursion'} == 1 && $config{$section}{'autosnap'} == 1) { warn "ignored autosnap configuration for '$section' because it's part of a zfs recursion.\n"; @@ -1007,6 +1057,10 @@ sub init { @datasets = getchilddatasets($config{$section}{'path'}); DATASETS: foreach my $dataset(@datasets) { + if (! @cachedatasets) { + push (@updatedatasets, $dataset); + } + chomp $dataset; if ($zfsRecursive) { @@ -1038,9 +1092,26 @@ sub init { $config{$dataset}{'initialized'} = 1; } } + } - - + # update dataset cache if it was unused + if (! @cachedatasets) { + if (checklock('sanoid_cachedatasetupdate')) { + writelock('sanoid_cachedatasetupdate'); + if ($args{'verbose'}) { + if ($args{'force-update'}) { + print "INFO: dataset cache forcibly expired - updating from zfs list.\n"; + } else { + print "INFO: dataset cache expired - updating from zfs list.\n"; + } + } + open FH, "> $cachedatasetspath" or die 'Could not write to $cachedatasetspath!\n'; + print FH @updatedatasets; + close FH; + removelock('sanoid_cachedatasetupdate'); + } else { + if ($args{'verbose'}) { print "INFO: deferring dataset cache update - valid cache update lock held by another sanoid process.\n"; } + } } return %config; @@ -1590,6 +1661,30 @@ sub getchilddatasets { my $fs = shift; my $mysudocmd = ''; + # use dataset cache if available + if (@cachedatasets) { + my $foundparent = 0; + my @cachechildren = (); + foreach my $dataset (@cachedatasets) { + chomp $dataset; + my $ret = rindex $dataset, "${fs}/", 0; + if ($ret == 0) { + push (@cachechildren, $dataset); + } else { + if ($dataset eq $fs) { + $foundparent = 1; + } + } + } + + # sanity check + if ($foundparent) { + return @cachechildren; + } + + # fallback if cache misses items for whatever reason + } + my $getchildrencmd = "$mysudocmd $zfs list -o name -t filesystem,volume -Hr $fs |"; if ($args{'debug'}) { print "DEBUG: getting list of child datasets on $fs using $getchildrencmd...\n"; } open FH, $getchildrencmd; @@ -1645,7 +1740,7 @@ sub removecachedsnapshots { close FH; removelock('sanoid_cacheupdate'); - %snaps = getsnaps(\%config,$cacheTTL,$forcecacheupdate); + %snaps = getsnaps(\%config,$cacheTTL,0); # clear hash undef %pruned; From 9c0468ee45b1af1e5a0c809bbefbcd4e6855f364 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 24 Apr 2024 00:09:40 +0200 Subject: [PATCH 9/9] write cache files in an atomic way to prevent race conditions --- sanoid | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sanoid b/sanoid index 533f9ea..f74f731 100755 --- a/sanoid +++ b/sanoid @@ -868,9 +868,10 @@ sub getsnaps { @rawsnaps = ; close FH; - open FH, "> $cache" or die 'Could not write to $cache!\n'; + open FH, "> $cache.tmp" or die 'Could not write to $cache.tmp!\n'; print FH @rawsnaps; close FH; + rename("$cache.tmp", "$cache") or die 'Could not rename to $cache!\n'; removelock('sanoid_cacheupdate'); } else { if ($args{'verbose'}) { print "INFO: deferring cache update - valid cache update lock held by another sanoid process.\n"; } @@ -1105,9 +1106,10 @@ sub init { print "INFO: dataset cache expired - updating from zfs list.\n"; } } - open FH, "> $cachedatasetspath" or die 'Could not write to $cachedatasetspath!\n'; + open FH, "> $cachedatasetspath.tmp" or die 'Could not write to $cachedatasetspath.tmp!\n'; print FH @updatedatasets; close FH; + rename("$cachedatasetspath.tmp", "$cachedatasetspath") or die 'Could not rename to $cachedatasetspath!\n'; removelock('sanoid_cachedatasetupdate'); } else { if ($args{'verbose'}) { print "INFO: deferring dataset cache update - valid cache update lock held by another sanoid process.\n"; } @@ -1731,13 +1733,14 @@ sub removecachedsnapshots { my @rawsnaps = ; close FH; - open FH, "> $cache" or die 'Could not write to $cache!\n'; + open FH, "> $cache.tmp" or die 'Could not write to $cache.tmp!\n'; foreach my $snapline ( @rawsnaps ) { my @columns = split("\t", $snapline); my $snap = $columns[0]; print FH $snapline unless ( exists($pruned{$snap}) ); } close FH; + rename("$cache.tmp", "$cache") or die 'Could not rename to $cache!\n'; removelock('sanoid_cacheupdate'); %snaps = getsnaps(\%config,$cacheTTL,0);