From 2d89434ac3181cd7e8d1b2f9d5fe676d85eadd17 Mon Sep 17 00:00:00 2001 From: Mathieu Arnold Date: Fri, 13 Mar 2020 14:20:25 +0100 Subject: [PATCH 1/5] Add target snapshot deletion. --- README.md | 3 +++ syncoid | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1cb2f17..51ac568 100644 --- a/README.md +++ b/README.md @@ -319,6 +319,9 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --preserve-recordsize This argument tells syncoid to set the recordsize on the target before writing any data to it matching the one set on the replication src. This only applies to initial sends. ++ --push-snap-removal + + With this argument snapshots that are removed in the source will also be removed. Use this if you only want to handle snapshots on the source. + --no-clone-rollback diff --git a/syncoid b/syncoid index e5046f3..3796ce1 100755 --- a/syncoid +++ b/syncoid @@ -25,7 +25,8 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn "source-bwlimit=s", "target-bwlimit=s", "sshconfig=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", "no-clone-handling", "no-privilege-elevation", "force-delete", "no-rollback", "create-bookmark", - "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", "mbuffer-size=s" => \$mbuffer_size) + "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", "mbuffer-size=s" => \$mbuffer_size, + "push-snap-removal",) or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -888,6 +889,20 @@ sub syncdataset { } } + if (defined $args{'push-snap-removal'}) { + foreach my $snap ( sort { $snaps{'target'}{$a}{'creation'}<=>$snaps{'target'}{$b}{'creation'} } keys %{ $snaps{'target'} }) { + if (!exists $snaps{'source'}{$snap}) { + if ($targethost ne '') { + if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd destroy $targetfsescaped\@$snap\n"; } + system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd destroy $targetfsescaped\@$snap")); + } else { + if ($debug) { print "$targetsudocmd $zfscmd destroy $targetfsescaped\@$snap\n"; } + system ("$targetsudocmd $zfscmd destroy $targetfsescaped\@$snap"); + } + } + } + } + } # end syncdataset() sub compressargset { @@ -1999,6 +2014,7 @@ Options: --create-bookmark Creates a zfs bookmark for the newest snapshot on the source after replication succeeds (only works with --no-sync-snap) --preserve-recordsize Preserves the recordsize on initial sends to the target --no-rollback Does not rollback snapshots on target (it probably requires a readonly target) + --push-snap-removal Remove snapshots on the target that do not exist on the source any more --exclude=REGEX Exclude specific datasets which match the given regular expression. Can be specified multiple times --sendoptions=OPTIONS Use advanced options for zfs send (the arguments are filtered as needed), e.g. syncoid --sendoptions="Lc e" sets zfs send -L -c -e ... --recvoptions=OPTIONS Use advanced options for zfs receive (the arguments are filtered as needed), e.g. syncoid --recvoptions="ux recordsize o compression=lz4" sets zfs receive -u -x recordsize -o compression=lz4 ... From 63dd819ec55d181283e184e898a88bc3751e7e89 Mon Sep 17 00:00:00 2001 From: Mathieu Arnold Date: Mon, 11 May 2020 16:37:40 +0200 Subject: [PATCH 2/5] Rename option. --- README.md | 5 +++-- syncoid | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 51ac568..f6028b0 100644 --- a/README.md +++ b/README.md @@ -319,9 +319,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --preserve-recordsize This argument tells syncoid to set the recordsize on the target before writing any data to it matching the one set on the replication src. This only applies to initial sends. -+ --push-snap-removal - With this argument snapshots that are removed in the source will also be removed. Use this if you only want to handle snapshots on the source. ++ --delete-target-snapshots + + With this argument snapshots which are missing on the source will be destroyed on the target. Use this if you only want to handle snapshots on the source. + --no-clone-rollback diff --git a/syncoid b/syncoid index 3796ce1..19c8656 100755 --- a/syncoid +++ b/syncoid @@ -26,7 +26,7 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", "no-clone-handling", "no-privilege-elevation", "force-delete", "no-rollback", "create-bookmark", "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", "mbuffer-size=s" => \$mbuffer_size, - "push-snap-removal",) + "delete-target-snapshots") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -889,7 +889,7 @@ sub syncdataset { } } - if (defined $args{'push-snap-removal'}) { + if (defined $args{'delete-target-snapshots'}) { foreach my $snap ( sort { $snaps{'target'}{$a}{'creation'}<=>$snaps{'target'}{$b}{'creation'} } keys %{ $snaps{'target'} }) { if (!exists $snaps{'source'}{$snap}) { if ($targethost ne '') { @@ -2014,7 +2014,7 @@ Options: --create-bookmark Creates a zfs bookmark for the newest snapshot on the source after replication succeeds (only works with --no-sync-snap) --preserve-recordsize Preserves the recordsize on initial sends to the target --no-rollback Does not rollback snapshots on target (it probably requires a readonly target) - --push-snap-removal Remove snapshots on the target that do not exist on the source any more + --delete-target-snapshots With this argument snapshots which are missing on the source will be destroyed on the target. Use this if you only want to handle snapshots on the source. --exclude=REGEX Exclude specific datasets which match the given regular expression. Can be specified multiple times --sendoptions=OPTIONS Use advanced options for zfs send (the arguments are filtered as needed), e.g. syncoid --sendoptions="Lc e" sets zfs send -L -c -e ... --recvoptions=OPTIONS Use advanced options for zfs receive (the arguments are filtered as needed), e.g. syncoid --recvoptions="ux recordsize o compression=lz4" sets zfs receive -u -x recordsize -o compression=lz4 ... From ecd14005395059e5dbbbdd8f7530d479d1e5843b Mon Sep 17 00:00:00 2001 From: Mathieu Arnold Date: Mon, 17 Aug 2020 16:36:14 +0200 Subject: [PATCH 3/5] Handle output/errors of those zfs destroy commands. If there was an obsolete remote syncoid_hostname_* snapshot that did not get removed at the correct time, for some reason, like, maybe, network problems, it would have been cleaned up in pruneoldsyncsnaps just before this code, and we would get a strange error message saying: could not find any snapshots to destroy; check snapshot names. Also, when using --quiet, do not output anything, as failing to remove an obsolete snapshot is not really a big problem. --- syncoid | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/syncoid b/syncoid index 19c8656..4393e4f 100755 --- a/syncoid +++ b/syncoid @@ -890,15 +890,18 @@ sub syncdataset { } if (defined $args{'delete-target-snapshots'}) { - foreach my $snap ( sort { $snaps{'target'}{$a}{'creation'}<=>$snaps{'target'}{$b}{'creation'} } keys %{ $snaps{'target'} }) { - if (!exists $snaps{'source'}{$snap}) { - if ($targethost ne '') { - if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd destroy $targetfsescaped\@$snap\n"; } - system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd destroy $targetfsescaped\@$snap")); - } else { - if ($debug) { print "$targetsudocmd $zfscmd destroy $targetfsescaped\@$snap\n"; } - system ("$targetsudocmd $zfscmd destroy $targetfsescaped\@$snap"); - } + my $snaps = join ',', grep {!exists $snaps{'source'}{$_}} keys %{ $snaps{'target'} }; + if ($snaps ne '') { + my $command; + if ($targethost ne '') { + $command = "$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd destroy $targetfsescaped\@$snaps"); + } else { + $command = "$targetsudocmd $zfscmd destroy $targetfsescaped\@$snaps"; + } + if ($debug) { print "$command\n"; } + my ($stdout, $stderr, $result) = capture { system $command; }; + if ($result != 0 && !$quiet) { + warn "$command failed: $stderr"; } } } From f711e6bf28e8b10f9e0f91f81f93ff6149c8492d Mon Sep 17 00:00:00 2001 From: Mathieu Arnold Date: Mon, 17 May 2021 13:32:22 +0200 Subject: [PATCH 4/5] Add a note about when snapshot deletion is done. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f6028b0..5ad2e0c 100644 --- a/README.md +++ b/README.md @@ -323,6 +323,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --delete-target-snapshots With this argument snapshots which are missing on the source will be destroyed on the target. Use this if you only want to handle snapshots on the source. + Note that snapshot deletion is only done after a successful synchronization. If no new snapshots are found, no synchronization is done and no deletion either. + --no-clone-rollback From 2f706a4ae1cab80417e76cc071aa1c561706791d Mon Sep 17 00:00:00 2001 From: Mathieu Arnold Date: Sat, 8 Apr 2023 09:58:40 +0200 Subject: [PATCH 5/5] Batch snapshot deletion. This is to prevent a problem with a large amount of snapshots which exceed the allowed shell command length. --- syncoid | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index 4393e4f..1736afe 100755 --- a/syncoid +++ b/syncoid @@ -890,8 +890,14 @@ sub syncdataset { } if (defined $args{'delete-target-snapshots'}) { - my $snaps = join ',', grep {!exists $snaps{'source'}{$_}} keys %{ $snaps{'target'} }; - if ($snaps ne '') { + # Find the snapshots that exist on the target, filter with + # those that exist on the source. Remaining are the snapshots + # that are only on the target. Then sort by creation date, as + # to remove the oldest snapshots first. + my @to_delete = sort { $snaps{'target'}{$a}{'creation'}<=>$snaps{'target'}{$b}{'creation'} } grep {!exists $snaps{'source'}{$_}} keys %{ $snaps{'target'} }; + while (@to_delete) { + # Create batch of snapshots to remove + my $snaps = join ',', splice(@to_delete, 0, 50); my $command; if ($targethost ne '') { $command = "$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd destroy $targetfsescaped\@$snaps");