From 3bae6baf3d68efcc451d2db366c4692c854b3677 Mon Sep 17 00:00:00 2001 From: rbike <46862457+rbike@users.noreply.github.com> Date: Tue, 2 Mar 2021 16:03:16 +0100 Subject: [PATCH 1/3] syncoid hold Added hold feature to syncoid. Parameter "--use-hold" sets hold for newsyncsmap and removes hold from matchingsnap both on source and target. Hold name is "syncoid" + identiifer + host name --- syncoid | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/syncoid b/syncoid index b771d8f..22d1c65 100755 --- a/syncoid +++ b/syncoid @@ -3,6 +3,9 @@ # this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. +# +# 2021-03-ß2: redbike +# support f0r zfs holds added $::VERSION = '2.0.3'; @@ -25,7 +28,7 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", "no-clone-handling", "no-privilege-elevation", "force-delete", "no-clone-rollback", "no-rollback", - "create-bookmark", "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", + "create-bookmark", "use-hold", "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", "mbuffer-size=s" => \$mbuffer_size) or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -355,6 +358,7 @@ sub syncdataset { } my $newsyncsnap; + my $matchingsnap; # skip snapshot checking/creation in case of resumed receive if (!defined($receivetoken)) { @@ -602,7 +606,7 @@ sub syncdataset { my $bookmark = 0; my $bookmarkcreation = 0; - my $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, \%snaps); + $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, \%snaps); if (! $matchingsnap) { # no matching snapshots, check for bookmarks as fallback my %bookmarks = getbookmarks($sourcehost,$sourcefs,$sourceisroot); @@ -827,7 +831,36 @@ sub syncdataset { #setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly); } } - +# if "--use-hold" parameter is used set hold on newsync snapshot and remove hold on matching snapshot both on source and target + # hold name: "syncoid" + identifier + hostname -> in case of replication to multiple targets separate holds can be set for each target by assinging different identifiers to each target. Only if all targets have been replicated all syncoid holds are removed from the matching snapshot and it can be removed + if (defined $args{'use-hold'}) { + my $holdcmd; + my $holdreleasecmd; + my $hostid = hostname(); + my $matchingsnapescaped = escapeshellparam($matchingsnap); + my $holdname = "syncoid\_$identifier$hostid"; + if ($sourcehost ne '') { + $holdcmd = "$sshcmd $sourcehost " . escapeshellparam("$sourcesudocmd $zfscmd hold $holdname $sourcefsescaped\@$newsyncsnapescaped"); + $holdreleasecmd = "$sshcmd $sourcehost " . escapeshellparam("$sourcesudocmd $zfscmd release $holdname $sourcefsescaped\@$matchingsnapescaped"); + } else { + $holdcmd = "$sourcesudocmd $zfscmd hold $holdname $sourcefsescaped\@$newsyncsnapescaped"; + $holdreleasecmd = "$sourcesudocmd $zfscmd release $holdname $sourcefsescaped\@$matchingsnapescaped"; + }; + if ($debug) { print "DEBUG: Set new hold on source: $holdcmd\n"; } + system($holdcmd) == 0 or warn "WARNING: $holdcmd failed: $?"; + if ($debug) { print "DEBUG: Release old hold on source: $holdreleasecmd\n"; } + system($holdreleasecmd) == 0 or warn "WARNING: $holdreleasecmd failed: $?"; + if ($targethost ne '') { + $holdcmd = "$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd hold $holdname $targetfsescaped\@$newsyncsnapescaped"); + $holdreleasecmd = "$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd release $holdname $targetfsescaped\@$matchingsnapescaped"); + } else { + $holdcmd = "$targetsudocmd $zfscmd hold $holdname $targetfsescaped\@$newsyncsnapescaped"; $holdreleasecmd = "$targetsudocmd $zfscmd release $holdname $targetfsescaped\@$matchingsnapescaped"; + }; + if ($debug) { print "DEBUG: Set new hold on target: $holdcmd\n"; } + system($holdcmd) == 0 or warn "WARNING: $holdcmd failed: $?"; + if ($debug) { print "DEBUG: Release old hold on target: $holdreleasecmd\n"; } + system($holdreleasecmd) == 0 or warn "WARNING: $holdreleasecmd failed: $?"; + } if (defined $args{'no-sync-snap'}) { if (defined $args{'create-bookmark'}) { my $bookmarkcmd; @@ -1325,7 +1358,7 @@ sub buildsynccmd { sub pruneoldsyncsnaps { my ($rhost,$fs,$newsyncsnap,$isroot,@snaps) = @_; - my $fsescaped = escapeshellparam($fs); + my $fsescaped = escapeshellparam($fs); if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } From 2858fbe907680f04f0cd449bf676a2b33e13343b Mon Sep 17 00:00:00 2001 From: rbike <46862457+rbike@users.noreply.github.com> Date: Wed, 3 Mar 2021 13:49:13 +0100 Subject: [PATCH 2/3] Docs update + error handling updated in-program docs and readme, check if matchingsnap exists before releasing hold --- README.md | 5 ++++- syncoid | 23 +++++++++++++++-------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index b4558c0..3048b7f 100644 --- a/README.md +++ b/README.md @@ -272,7 +272,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --identifier= - Adds the given identifier to the snapshot name after "syncoid_" prefix and before the hostname. This enables the use case of reliable replication to multiple targets from the same host. The following chars are allowed: a-z, A-Z, 0-9, _, -, : and . . + Adds the given identifier to the snapshot and hold name after "syncoid_" prefix and before the hostname. This enables the use case of reliable replication to multiple targets from the same host. The following chars are allowed: a-z, A-Z, 0-9, _, -, : and . . + -r --recursive @@ -314,6 +314,9 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This argument tells syncoid to create a zfs bookmark for the newest snapshot after it got replicated successfully. The bookmark name will be equal to the snapshot name. Only works in combination with the --no-sync-snap option. This can be very useful for irregular replication where the last matching snapshot on the source was already deleted but the bookmark remains so a replication is still possible. ++ --use-hold + This argument tells syncoid to add a hold to the newest snapshot on the source and target after replication succeeds and to remove the hold after the next succesful replication. Setting a hold prevents the snapshots from being destroyed. The hold name incldues the identifier if set. This allows for separate holds in case of replication to multiple targets. + + --preserve-recordsize This argument tells syncoid to set the recordsize on the target before writing any data to it matching the one set on the replication src. This only applies to initial sends. diff --git a/syncoid b/syncoid index 22d1c65..84510fb 100755 --- a/syncoid +++ b/syncoid @@ -4,8 +4,8 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. # -# 2021-03-ß2: redbike -# support f0r zfs holds added +# 2021-03-03: redbike +# support for zfs holds added $::VERSION = '2.0.3'; @@ -845,21 +845,27 @@ sub syncdataset { } else { $holdcmd = "$sourcesudocmd $zfscmd hold $holdname $sourcefsescaped\@$newsyncsnapescaped"; $holdreleasecmd = "$sourcesudocmd $zfscmd release $holdname $sourcefsescaped\@$matchingsnapescaped"; - }; + } if ($debug) { print "DEBUG: Set new hold on source: $holdcmd\n"; } system($holdcmd) == 0 or warn "WARNING: $holdcmd failed: $?"; - if ($debug) { print "DEBUG: Release old hold on source: $holdreleasecmd\n"; } - system($holdreleasecmd) == 0 or warn "WARNING: $holdreleasecmd failed: $?"; + # Do hold release only if matchingsnap exists + if ($matchingsnap) { + if ($debug) { print "DEBUG: Release old hold on source: $holdreleasecmd\n"; } + system($holdreleasecmd) == 0 or warn "WARNING: $holdreleasecmd failed: $?"; + } if ($targethost ne '') { $holdcmd = "$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd hold $holdname $targetfsescaped\@$newsyncsnapescaped"); $holdreleasecmd = "$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd release $holdname $targetfsescaped\@$matchingsnapescaped"); } else { $holdcmd = "$targetsudocmd $zfscmd hold $holdname $targetfsescaped\@$newsyncsnapescaped"; $holdreleasecmd = "$targetsudocmd $zfscmd release $holdname $targetfsescaped\@$matchingsnapescaped"; - }; + } if ($debug) { print "DEBUG: Set new hold on target: $holdcmd\n"; } system($holdcmd) == 0 or warn "WARNING: $holdcmd failed: $?"; - if ($debug) { print "DEBUG: Release old hold on target: $holdreleasecmd\n"; } - system($holdreleasecmd) == 0 or warn "WARNING: $holdreleasecmd failed: $?"; + # Do hold release only if matchingsnap exists + if ($matchingsnap) { + if ($debug) { print "DEBUG: Release old hold on target: $holdreleasecmd\n"; } + system($holdreleasecmd) == 0 or warn "WARNING: $holdreleasecmd failed: $?"; + } } if (defined $args{'no-sync-snap'}) { if (defined $args{'create-bookmark'}) { @@ -2005,6 +2011,7 @@ Options: --no-sync-snap Does not create new snapshot, only transfers existing --keep-sync-snap Don't destroy created sync snapshots --create-bookmark Creates a zfs bookmark for the newest snapshot on the source after replication succeeds (only works with --no-sync-snap) + --use-hold Adds a hold to the newest snapshot on the source and target after replication succeeds and removes the hold after the next succesful replication. The hold name incldues the identifier if set. This allows for separate holds in case of multiple targets --preserve-recordsize Preserves the recordsize on initial sends to the target --no-clone-rollback Does not rollback clones on target --no-rollback Does not rollback clones or snapshots on target (it probably requires a readonly target) From 21eee41fdb20a6bc3b2068c18a3061723f52435a Mon Sep 17 00:00:00 2001 From: rbike <46862457+rbike@users.noreply.github.com> Date: Thu, 1 Apr 2021 09:23:48 +0200 Subject: [PATCH 3/3] Final revision comments from @phreaker0 implemented --- syncoid | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/syncoid b/syncoid index 84510fb..a3708c6 100755 --- a/syncoid +++ b/syncoid @@ -3,9 +3,6 @@ # this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -# -# 2021-03-03: redbike -# support for zfs holds added $::VERSION = '2.0.3'; @@ -831,7 +828,7 @@ sub syncdataset { #setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly); } } -# if "--use-hold" parameter is used set hold on newsync snapshot and remove hold on matching snapshot both on source and target + # if "--use-hold" parameter is used set hold on newsync snapshot and remove hold on matching snapshot both on source and target # hold name: "syncoid" + identifier + hostname -> in case of replication to multiple targets separate holds can be set for each target by assinging different identifiers to each target. Only if all targets have been replicated all syncoid holds are removed from the matching snapshot and it can be removed if (defined $args{'use-hold'}) { my $holdcmd; @@ -1364,7 +1361,7 @@ sub buildsynccmd { sub pruneoldsyncsnaps { my ($rhost,$fs,$newsyncsnap,$isroot,@snaps) = @_; - my $fsescaped = escapeshellparam($fs); + my $fsescaped = escapeshellparam($fs); if ($rhost ne '') { $rhost = "$sshcmd $rhost"; }