mirror of https://github.com/jimsalterjrs/sanoid
Merge pull request #247 from phreaker0/bookmarks
implemented support for zfs bookmarks as fallback
This commit is contained in:
Jim Salter
GitHub
commit
bb79e7b5ec
194
syncoid
194
syncoid
|
|
@ -481,11 +481,49 @@ sub syncdataset {
|
||||||
|
|
||||||
my $targetsize = getzfsvalue($targethost,$targetfs,$targetisroot,'-p used');
|
my $targetsize = getzfsvalue($targethost,$targetfs,$targetisroot,'-p used');
|
||||||
|
|
||||||
my $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, $targetsize, \%snaps);
|
my $bookmark = 0;
|
||||||
|
my $bookmarkcreation = 0;
|
||||||
|
|
||||||
|
my $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, \%snaps);
|
||||||
if (! $matchingsnap) {
|
if (! $matchingsnap) {
|
||||||
# no matching snapshot; we whined piteously already, but let's go ahead and return false
|
# no matching snapshots, check for bookmarks as fallback
|
||||||
# now in case more child datasets need replication.
|
my %bookmarks = getbookmarks($sourcehost,$sourcefs,$sourceisroot);
|
||||||
return 0;
|
|
||||||
|
# check for matching guid of source bookmark and target snapshot (oldest first)
|
||||||
|
foreach my $snap ( sort { $snaps{'target'}{$b}{'creation'}<=>$snaps{'target'}{$a}{'creation'} } keys %{ $snaps{'target'} }) {
|
||||||
|
my $guid = $snaps{'target'}{$snap}{'guid'};
|
||||||
|
|
||||||
|
if (defined $bookmarks{$guid}) {
|
||||||
|
# found a match
|
||||||
|
$bookmark = $bookmarks{$guid}{'name'};
|
||||||
|
$bookmarkcreation = $bookmarks{$guid}{'creation'};
|
||||||
|
$matchingsnap = $snap;
|
||||||
|
last;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (! $bookmark) {
|
||||||
|
# if we got this far, we failed to find a matching snapshot/bookmark.
|
||||||
|
if ($exitcode < 2) { $exitcode = 2; }
|
||||||
|
|
||||||
|
print "\n";
|
||||||
|
print "CRITICAL ERROR: Target $targetfs exists but has no snapshots matching with $sourcefs!\n";
|
||||||
|
print " Replication to target would require destroying existing\n";
|
||||||
|
print " target. Cowardly refusing to destroy your existing target.\n\n";
|
||||||
|
|
||||||
|
# experience tells me we need a mollyguard for people who try to
|
||||||
|
# zfs create targetpool/targetsnap ; syncoid sourcepool/sourcesnap targetpool/targetsnap ...
|
||||||
|
|
||||||
|
if ( $targetsize < (64*1024*1024) ) {
|
||||||
|
print " NOTE: Target $targetfs dataset is < 64MB used - did you mistakenly run\n";
|
||||||
|
print " \`zfs create $args{'target'}\` on the target? ZFS initial\n";
|
||||||
|
print " replication must be to a NON EXISTENT DATASET, which will\n";
|
||||||
|
print " then be CREATED BY the initial replication process.\n\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
# return false now in case more child datasets need replication.
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
# make sure target is (still) not currently in receive.
|
# make sure target is (still) not currently in receive.
|
||||||
|
|
@ -510,20 +548,77 @@ sub syncdataset {
|
||||||
system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped");
|
system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped");
|
||||||
}
|
}
|
||||||
|
|
||||||
my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped";
|
my $nextsnapshot = 0;
|
||||||
my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
|
|
||||||
my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot);
|
|
||||||
my $disp_pvsize = readablebytes($pvsize);
|
|
||||||
if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }
|
|
||||||
my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
|
|
||||||
|
|
||||||
if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; }
|
if ($bookmark) {
|
||||||
if ($debug) { print "DEBUG: $synccmd\n"; }
|
my $bookmarkescaped = escapeshellparam($bookmark);
|
||||||
system("$synccmd") == 0 or do {
|
|
||||||
warn "CRITICAL ERROR: $synccmd failed: $?";
|
if (!defined $args{'no-stream'}) {
|
||||||
if ($exitcode < 2) { $exitcode = 2; }
|
# if intermediate snapshots are needed we need to find the next oldest snapshot,
|
||||||
return 0;
|
# do an replication to it and replicate as always from oldest to newest
|
||||||
};
|
# because bookmark sends doesn't support intermediates directly
|
||||||
|
foreach my $snap ( sort { $snaps{'source'}{$a}{'creation'}<=>$snaps{'source'}{$b}{'creation'} } keys %{ $snaps{'source'} }) {
|
||||||
|
if ($snaps{'source'}{$snap}{'creation'} >= $bookmarkcreation) {
|
||||||
|
$nextsnapshot = $snap;
|
||||||
|
last;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# bookmark stream size can't be determined
|
||||||
|
my $pvsize = 0;
|
||||||
|
my $disp_pvsize = "UNKNOWN";
|
||||||
|
|
||||||
|
if ($nextsnapshot) {
|
||||||
|
my $nextsnapshotescaped = escapeshellparam($nextsnapshot);
|
||||||
|
|
||||||
|
my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$nextsnapshotescaped";
|
||||||
|
my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
|
||||||
|
my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
|
||||||
|
|
||||||
|
if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $nextsnapshot (~ $disp_pvsize):\n"; }
|
||||||
|
if ($debug) { print "DEBUG: $synccmd\n"; }
|
||||||
|
system("$synccmd") == 0 or do {
|
||||||
|
warn "CRITICAL ERROR: $synccmd failed: $?";
|
||||||
|
if ($exitcode < 2) { $exitcode = 2; }
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
$matchingsnap = $nextsnapshot;
|
||||||
|
$matchingsnapescaped = escapeshellparam($matchingsnap);
|
||||||
|
} else {
|
||||||
|
my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$newsyncsnapescaped";
|
||||||
|
my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
|
||||||
|
my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
|
||||||
|
|
||||||
|
if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $newsyncsnap (~ $disp_pvsize):\n"; }
|
||||||
|
if ($debug) { print "DEBUG: $synccmd\n"; }
|
||||||
|
system("$synccmd") == 0 or do {
|
||||||
|
warn "CRITICAL ERROR: $synccmd failed: $?";
|
||||||
|
if ($exitcode < 2) { $exitcode = 2; }
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# do a normal replication if bookmarks aren't used or if previous
|
||||||
|
# bookmark replication was only done to the next oldest snapshot
|
||||||
|
if (!$bookmark || $nextsnapshot) {
|
||||||
|
my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped";
|
||||||
|
my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
|
||||||
|
my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot);
|
||||||
|
my $disp_pvsize = readablebytes($pvsize);
|
||||||
|
if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }
|
||||||
|
my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
|
||||||
|
|
||||||
|
if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; }
|
||||||
|
if ($debug) { print "DEBUG: $synccmd\n"; }
|
||||||
|
system("$synccmd") == 0 or do {
|
||||||
|
warn "CRITICAL ERROR: $synccmd failed: $?";
|
||||||
|
if ($exitcode < 2) { $exitcode = 2; }
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
# restore original readonly value to target after sync complete
|
# restore original readonly value to target after sync complete
|
||||||
# dyking this functionality out for the time being due to buggy mount/unmount behavior
|
# dyking this functionality out for the time being due to buggy mount/unmount behavior
|
||||||
|
|
@ -1022,32 +1117,15 @@ sub pruneoldsyncsnaps {
|
||||||
}
|
}
|
||||||
|
|
||||||
sub getmatchingsnapshot {
|
sub getmatchingsnapshot {
|
||||||
my ($sourcefs, $targetfs, $targetsize, $snaps) = @_;
|
my ($sourcefs, $targetfs, $snaps) = @_;
|
||||||
foreach my $snap ( sort { $snaps{'source'}{$b}{'creation'}<=>$snaps{'source'}{$a}{'creation'} } keys %{ $snaps{'source'} }) {
|
foreach my $snap ( sort { $snaps{'source'}{$b}{'creation'}<=>$snaps{'source'}{$a}{'creation'} } keys %{ $snaps{'source'} }) {
|
||||||
if (defined $snaps{'target'}{$snap}{'guid'}) {
|
if (defined $snaps{'target'}{$snap}) {
|
||||||
if ($snaps{'source'}{$snap}{'guid'} == $snaps{'target'}{$snap}{'guid'}) {
|
if ($snaps{'source'}{$snap}{'guid'} == $snaps{'target'}{$snap}{'guid'}) {
|
||||||
return $snap;
|
return $snap;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
# if we got this far, we failed to find a matching snapshot.
|
|
||||||
if ($exitcode < 2) { $exitcode = 2; }
|
|
||||||
|
|
||||||
print "\n";
|
|
||||||
print "CRITICAL ERROR: Target $targetfs exists but has no snapshots matching with $sourcefs!\n";
|
|
||||||
print " Replication to target would require destroying existing\n";
|
|
||||||
print " target. Cowardly refusing to destroy your existing target.\n\n";
|
|
||||||
|
|
||||||
# experience tells me we need a mollyguard for people who try to
|
|
||||||
# zfs create targetpool/targetsnap ; syncoid sourcepool/sourcesnap targetpool/targetsnap ...
|
|
||||||
|
|
||||||
if ( $targetsize < (64*1024*1024) ) {
|
|
||||||
print " NOTE: Target $targetfs dataset is < 64MB used - did you mistakenly run\n";
|
|
||||||
print " \`zfs create $args{'target'}\` on the target? ZFS initial\n";
|
|
||||||
print " replication must be to a NON EXISTENT DATASET, which will\n";
|
|
||||||
print " then be CREATED BY the initial replication process.\n\n";
|
|
||||||
}
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1176,6 +1254,50 @@ sub getsnaps() {
|
||||||
return %snaps;
|
return %snaps;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sub getbookmarks() {
|
||||||
|
my ($rhost,$fs,$isroot,%bookmarks) = @_;
|
||||||
|
my $mysudocmd;
|
||||||
|
my $fsescaped = escapeshellparam($fs);
|
||||||
|
if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; }
|
||||||
|
|
||||||
|
if ($rhost ne '') {
|
||||||
|
$rhost = "$sshcmd $rhost";
|
||||||
|
# double escaping needed
|
||||||
|
$fsescaped = escapeshellparam($fsescaped);
|
||||||
|
}
|
||||||
|
|
||||||
|
my $getbookmarkcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t bookmark guid,creation $fsescaped |";
|
||||||
|
if ($debug) { print "DEBUG: getting list of bookmarks on $fs using $getbookmarkcmd...\n"; }
|
||||||
|
open FH, $getbookmarkcmd;
|
||||||
|
my @rawbookmarks = <FH>;
|
||||||
|
close FH;
|
||||||
|
|
||||||
|
# this is a little obnoxious. get guid,creation returns guid,creation on two separate lines
|
||||||
|
# as though each were an entirely separate get command.
|
||||||
|
|
||||||
|
my $lastguid;
|
||||||
|
|
||||||
|
foreach my $line (@rawbookmarks) {
|
||||||
|
# only import bookmark guids, creation from the specified filesystem
|
||||||
|
if ($line =~ /\Q$fs\E\#.*guid/) {
|
||||||
|
chomp $line;
|
||||||
|
$lastguid = $line;
|
||||||
|
$lastguid =~ s/^.*\tguid\t*(\d*).*/$1/;
|
||||||
|
my $bookmark = $line;
|
||||||
|
$bookmark =~ s/^.*\#(.*)\tguid.*$/$1/;
|
||||||
|
$bookmarks{$lastguid}{'name'}=$bookmark;
|
||||||
|
} elsif ($line =~ /\Q$fs\E\#.*creation/) {
|
||||||
|
chomp $line;
|
||||||
|
my $creation = $line;
|
||||||
|
$creation =~ s/^.*\tcreation\t*(\d*).*/$1/;
|
||||||
|
my $bookmark = $line;
|
||||||
|
$bookmark =~ s/^.*\#(.*)\tcreation.*$/$1/;
|
||||||
|
$bookmarks{$lastguid}{'creation'}=$creation;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return %bookmarks;
|
||||||
|
}
|
||||||
|
|
||||||
sub getsendsize {
|
sub getsendsize {
|
||||||
my ($sourcehost,$snap1,$snap2,$isroot,$receivetoken) = @_;
|
my ($sourcehost,$snap1,$snap2,$isroot,$receivetoken) = @_;
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,56 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# test replication with fallback to bookmarks and all intermediate snapshots
|
||||||
|
|
||||||
|
set -x
|
||||||
|
set -e
|
||||||
|
|
||||||
|
. ../../common/lib.sh
|
||||||
|
|
||||||
|
POOL_IMAGE="/tmp/syncoid-test-1.zpool"
|
||||||
|
POOL_SIZE="200M"
|
||||||
|
POOL_NAME="syncoid-test-1"
|
||||||
|
TARGET_CHECKSUM="a23564d5bb8a2babc3ac8936fd82825ad9fff9c82d4924f5924398106bbda9f0 -"
|
||||||
|
|
||||||
|
truncate -s "${POOL_SIZE}" "${POOL_IMAGE}"
|
||||||
|
|
||||||
|
zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}"
|
||||||
|
|
||||||
|
function cleanUp {
|
||||||
|
zpool export "${POOL_NAME}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# export pool in any case
|
||||||
|
trap cleanUp EXIT
|
||||||
|
|
||||||
|
zfs create "${POOL_NAME}"/src
|
||||||
|
zfs snapshot "${POOL_NAME}"/src@snap1
|
||||||
|
zfs bookmark "${POOL_NAME}"/src@snap1 "${POOL_NAME}"/src#snap1
|
||||||
|
# initial replication
|
||||||
|
../../../syncoid --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst
|
||||||
|
# destroy last common snapshot on source
|
||||||
|
zfs destroy "${POOL_NAME}"/src@snap1
|
||||||
|
|
||||||
|
# create intermediate snapshots
|
||||||
|
# sleep is needed so creation time can be used for proper sorting
|
||||||
|
sleep 1
|
||||||
|
zfs snapshot "${POOL_NAME}"/src@snap2
|
||||||
|
sleep 1
|
||||||
|
zfs snapshot "${POOL_NAME}"/src@snap3
|
||||||
|
sleep 1
|
||||||
|
zfs snapshot "${POOL_NAME}"/src@snap4
|
||||||
|
sleep 1
|
||||||
|
zfs snapshot "${POOL_NAME}"/src@snap5
|
||||||
|
|
||||||
|
# replicate which should fallback to bookmarks
|
||||||
|
../../../syncoid --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1
|
||||||
|
|
||||||
|
# verify
|
||||||
|
output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name)
|
||||||
|
checksum=$(echo "${output}" | grep -v syncoid_ | sha256sum)
|
||||||
|
|
||||||
|
if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
exit 0
|
||||||
|
|
@ -0,0 +1,56 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# test replication with fallback to bookmarks and all intermediate snapshots
|
||||||
|
|
||||||
|
set -x
|
||||||
|
set -e
|
||||||
|
|
||||||
|
. ../../common/lib.sh
|
||||||
|
|
||||||
|
POOL_IMAGE="/tmp/syncoid-test-2.zpool"
|
||||||
|
POOL_SIZE="200M"
|
||||||
|
POOL_NAME="syncoid-test-2"
|
||||||
|
TARGET_CHECKSUM="2460d4d4417793d2c7a5c72cbea4a8a584c0064bf48d8b6daa8ba55076cba66d -"
|
||||||
|
|
||||||
|
truncate -s "${POOL_SIZE}" "${POOL_IMAGE}"
|
||||||
|
|
||||||
|
zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}"
|
||||||
|
|
||||||
|
function cleanUp {
|
||||||
|
zpool export "${POOL_NAME}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# export pool in any case
|
||||||
|
trap cleanUp EXIT
|
||||||
|
|
||||||
|
zfs create "${POOL_NAME}"/src
|
||||||
|
zfs snapshot "${POOL_NAME}"/src@snap1
|
||||||
|
zfs bookmark "${POOL_NAME}"/src@snap1 "${POOL_NAME}"/src#snap1
|
||||||
|
# initial replication
|
||||||
|
../../../syncoid --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst
|
||||||
|
# destroy last common snapshot on source
|
||||||
|
zfs destroy "${POOL_NAME}"/src@snap1
|
||||||
|
|
||||||
|
# create intermediate snapshots
|
||||||
|
# sleep is needed so creation time can be used for proper sorting
|
||||||
|
sleep 1
|
||||||
|
zfs snapshot "${POOL_NAME}"/src@snap2
|
||||||
|
sleep 1
|
||||||
|
zfs snapshot "${POOL_NAME}"/src@snap3
|
||||||
|
sleep 1
|
||||||
|
zfs snapshot "${POOL_NAME}"/src@snap4
|
||||||
|
sleep 1
|
||||||
|
zfs snapshot "${POOL_NAME}"/src@snap5
|
||||||
|
|
||||||
|
# replicate which should fallback to bookmarks
|
||||||
|
../../../syncoid --no-stream --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1
|
||||||
|
|
||||||
|
# verify
|
||||||
|
output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name)
|
||||||
|
checksum=$(echo "${output}" | sha256sum)
|
||||||
|
|
||||||
|
if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
exit 0
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# run's all the available tests
|
||||||
|
|
||||||
|
for test in */; do
|
||||||
|
if [ ! -x "${test}/run.sh" ]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
testName="${test%/}"
|
||||||
|
|
||||||
|
LOGFILE=/tmp/syncoid_test_run_"${testName}".log
|
||||||
|
|
||||||
|
pushd . > /dev/null
|
||||||
|
|
||||||
|
echo -n "Running test ${testName} ... "
|
||||||
|
cd "${test}"
|
||||||
|
echo | bash run.sh > "${LOGFILE}" 2>&1
|
||||||
|
|
||||||
|
if [ $? -eq 0 ]; then
|
||||||
|
echo "[PASS]"
|
||||||
|
else
|
||||||
|
echo "[FAILED] (see ${LOGFILE})"
|
||||||
|
fi
|
||||||
|
|
||||||
|
popd > /dev/null
|
||||||
|
done
|
||||||
Loading…
Reference in New Issue