mirror of https://github.com/jimsalterjrs/sanoid
Merge 89b98ce113 into 45b1ce9e5d
This commit is contained in:
Felix Matouschek
GitHub
commit
2936dfd528
77
syncoid
77
syncoid
|
|
@ -410,6 +410,7 @@ sub syncdataset {
|
|||
|
||||
my $newsyncsnap;
|
||||
my $matchingsnap;
|
||||
my $usebookmark = 0;
|
||||
|
||||
# skip snapshot checking/creation in case of resumed receive
|
||||
if (!defined($receivetoken)) {
|
||||
|
|
@ -585,24 +586,27 @@ sub syncdataset {
|
|||
my $targetsize = getzfsvalue($targethost,$targetfs,$targetisroot,'-p used');
|
||||
|
||||
my %bookmark = ();
|
||||
my $bookmarksnap = 0;
|
||||
|
||||
$matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, \%snaps);
|
||||
|
||||
# find most recent matching bookmark
|
||||
my %bookmarks = getbookmarks($sourcehost,$sourcefs,$sourceisroot);
|
||||
|
||||
# check for matching guid of source bookmark and target snapshot (oldest first)
|
||||
foreach my $snap ( sort { sortsnapshots(\%snaps, $b, $a, 'target') } keys %{ $snaps{'target'} }) {
|
||||
my $guid = $snaps{'target'}{$snap}{'guid'};
|
||||
|
||||
if (defined $bookmarks{$guid}) {
|
||||
# found a match
|
||||
%bookmark = %{ $bookmarks{$guid} };
|
||||
$bookmarksnap = $snap;
|
||||
last;
|
||||
}
|
||||
}
|
||||
|
||||
if (! $matchingsnap) {
|
||||
# no matching snapshots, check for bookmarks as fallback
|
||||
my %bookmarks = getbookmarks($sourcehost,$sourcefs,$sourceisroot);
|
||||
|
||||
# check for matching guid of source bookmark and target snapshot (oldest first)
|
||||
foreach my $snap ( sort { sortsnapshots(\%snaps, $b, $a) } keys %{ $snaps{'target'} }) {
|
||||
my $guid = $snaps{'target'}{$snap}{'guid'};
|
||||
|
||||
if (defined $bookmarks{$guid}) {
|
||||
# found a match
|
||||
%bookmark = %{ $bookmarks{$guid} };
|
||||
$matchingsnap = $snap;
|
||||
last;
|
||||
}
|
||||
}
|
||||
|
||||
if (! %bookmark) {
|
||||
# force delete is not possible for the root dataset
|
||||
if ($args{'force-delete'} && index($targetfs, '/') != -1) {
|
||||
|
|
@ -657,6 +661,18 @@ sub syncdataset {
|
|||
|
||||
# return false now in case more child datasets need replication.
|
||||
return 0;
|
||||
} else {
|
||||
# use bookmark as fallback
|
||||
$matchingsnap = $bookmarksnap;
|
||||
$usebookmark = 1;
|
||||
}
|
||||
} elsif (%bookmark && $bookmark{'guid'} ne $snaps{'source'}{$matchingsnap}{'guid'}) {
|
||||
my $comparetxg = defined $snaps{'source'}{$matchingsnap}{'createtxg'} && defined $bookmark{'createtxg'};
|
||||
if (($comparetxg && $bookmark{'createtxg'} > $snaps{'source'}{$matchingsnap}{'createtxg'}) ||
|
||||
$bookmark{'creation'} > substr($snaps{'source'}{$matchingsnap}{'creation'}, 0, -3)) {
|
||||
writelog('DEBUG', "using bookmark $bookmark{'name'} because it was created after latest matching snapshot $matchingsnap");
|
||||
$matchingsnap = $bookmarksnap;
|
||||
$usebookmark = 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -676,18 +692,16 @@ sub syncdataset {
|
|||
|
||||
my $nextsnapshot = 0;
|
||||
|
||||
if (%bookmark) {
|
||||
if ($usebookmark) {
|
||||
|
||||
if (!defined $args{'no-stream'}) {
|
||||
# if intermediate snapshots are needed we need to find the next oldest snapshot,
|
||||
# do an replication to it and replicate as always from oldest to newest
|
||||
# because bookmark sends doesn't support intermediates directly
|
||||
foreach my $snap ( sort { sortsnapshots(\%snaps, $a, $b) } keys %{ $snaps{'source'} }) {
|
||||
my $comparisonkey = 'creation';
|
||||
if (defined $snaps{'source'}{$snap}{'createtxg'} && defined $bookmark{'createtxg'}) {
|
||||
$comparisonkey = 'createtxg';
|
||||
}
|
||||
if ($snaps{'source'}{$snap}{$comparisonkey} >= $bookmark{$comparisonkey}) {
|
||||
foreach my $snap ( sort { sortsnapshots(\%snaps, $a, $b, 'source') } keys %{ $snaps{'source'} }) {
|
||||
my $comparetxg = defined $snaps{'source'}{$snap}{'createtxg'} && defined $bookmark{'createtxg'};
|
||||
if (($comparetxg && $snaps{'source'}{$snap}{'createtxg'} >= $bookmark{'createtxg'}) ||
|
||||
substr($snaps{'source'}{$snap}{'creation'}, 0, -3) >= $bookmark{'creation'}) {
|
||||
$nextsnapshot = $snap;
|
||||
last;
|
||||
}
|
||||
|
|
@ -736,7 +750,7 @@ sub syncdataset {
|
|||
|
||||
# do a normal replication if bookmarks aren't used or if previous
|
||||
# bookmark replication was only done to the next oldest snapshot
|
||||
if (!%bookmark || $nextsnapshot) {
|
||||
if (!$usebookmark || $nextsnapshot) {
|
||||
if ($matchingsnap eq $newsyncsnap) {
|
||||
# edge case: bookmark replication used the latest snapshot
|
||||
return 0;
|
||||
|
|
@ -801,7 +815,7 @@ sub syncdataset {
|
|||
|
||||
# if "--use-hold" parameter is used set hold on newsync snapshot and remove hold on matching snapshot both on source and target
|
||||
# hold name: "syncoid" + identifier + hostname -> in case of replication to multiple targets separate holds can be set for each target by assinging different identifiers to each target. Only if all targets have been replicated all syncoid holds are removed from the matching snapshot and it can be removed
|
||||
if (defined $args{'use-hold'}) {
|
||||
if (defined $args{'use-hold'} && !$usebookmark) {
|
||||
my $holdcmd;
|
||||
my $holdreleasecmd;
|
||||
my $hostid = hostname();
|
||||
|
|
@ -865,7 +879,7 @@ sub syncdataset {
|
|||
# those that exist on the source. Remaining are the snapshots
|
||||
# that are only on the target. Then sort to remove the oldest
|
||||
# snapshots first.
|
||||
my @to_delete = sort { sortsnapshots(\%snaps, $a, $b) } grep {!exists $snaps{'source'}{$_}} keys %{ $snaps{'target'} };
|
||||
my @to_delete = sort { sortsnapshots(\%snaps, $a, $b, 'source') } grep {!exists $snaps{'source'}{$_}} keys %{ $snaps{'target'} };
|
||||
while (@to_delete) {
|
||||
# Create batch of snapshots to remove
|
||||
my $snaps = join ',', splice(@to_delete, 0, 50);
|
||||
|
|
@ -1486,16 +1500,16 @@ sub readablebytes {
|
|||
}
|
||||
|
||||
sub sortsnapshots {
|
||||
my ($snaps, $left, $right) = @_;
|
||||
if (defined $snaps->{'source'}{$left}{'createtxg'} && defined $snaps->{'source'}{$right}{'createtxg'}) {
|
||||
return $snaps->{'source'}{$left}{'createtxg'} <=> $snaps->{'source'}{$right}{'createtxg'};
|
||||
my ($snaps, $left, $right, $idx) = @_;
|
||||
if (defined $snaps->{$idx}{$left}{'createtxg'} && defined $snaps->{$idx}{$right}{'createtxg'}) {
|
||||
return $snaps->{$idx}{$left}{'createtxg'} <=> $snaps->{$idx}{$right}{'createtxg'};
|
||||
}
|
||||
return $snaps->{'source'}{$left}{'creation'} <=> $snaps->{'source'}{$right}{'creation'};
|
||||
return $snaps->{$idx}{$left}{'creation'} <=> $snaps->{$idx}{$right}{'creation'};
|
||||
}
|
||||
|
||||
sub getoldestsnapshot {
|
||||
my $snaps = shift;
|
||||
foreach my $snap (sort { sortsnapshots($snaps, $a, $b) } keys %{ $snaps{'source'} }) {
|
||||
foreach my $snap (sort { sortsnapshots($snaps, $a, $b, 'source') } keys %{ $snaps{'source'} }) {
|
||||
# return on first snap found - it's the oldest
|
||||
return $snap;
|
||||
}
|
||||
|
|
@ -1509,7 +1523,7 @@ sub getoldestsnapshot {
|
|||
|
||||
sub getnewestsnapshot {
|
||||
my $snaps = shift;
|
||||
foreach my $snap (sort { sortsnapshots($snaps, $b, $a) } keys %{ $snaps{'source'} }) {
|
||||
foreach my $snap (sort { sortsnapshots($snaps, $b, $a, 'source') } keys %{ $snaps{'source'} }) {
|
||||
# return on first snap found - it's the newest
|
||||
writelog('INFO', "NEWEST SNAPSHOT: $snap");
|
||||
return $snap;
|
||||
|
|
@ -1688,7 +1702,7 @@ sub pruneoldsyncsnaps {
|
|||
|
||||
sub getmatchingsnapshot {
|
||||
my ($sourcefs, $targetfs, $snaps) = @_;
|
||||
foreach my $snap ( sort { sortsnapshots($snaps, $b, $a) } keys %{ $snaps{'source'} }) {
|
||||
foreach my $snap ( sort { sortsnapshots($snaps, $b, $a, 'source') } keys %{ $snaps{'source'} }) {
|
||||
if (defined $snaps{'target'}{$snap}) {
|
||||
if ($snaps{'source'}{$snap}{'guid'} == $snaps{'target'}{$snap}{'guid'}) {
|
||||
return $snap;
|
||||
|
|
@ -1964,6 +1978,7 @@ sub getbookmarks() {
|
|||
|
||||
for my $bookmark (keys %bookmark_data) {
|
||||
my $guid = $bookmark_data{$bookmark}{'guid'};
|
||||
$bookmarks{$guid}{'guid'} = $guid;
|
||||
$bookmarks{$guid}{'name'} = $bookmark;
|
||||
$bookmarks{$guid}{'creation'} = $bookmark_data{$bookmark}{'creation'};
|
||||
$bookmarks{$guid}{'createtxg'} = $bookmark_data{$bookmark}{'createtxg'};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
#!/bin/bash
|
||||
|
||||
# test using bookmark created after last snapshot
|
||||
|
||||
set -x
|
||||
set -e
|
||||
|
||||
. ../../common/lib.sh
|
||||
|
||||
POOL_IMAGE="/tmp/syncoid-test-11.zpool"
|
||||
MOUNT_TARGET="/tmp/syncoid-test-11.mount"
|
||||
POOL_SIZE="1000M"
|
||||
POOL_NAME="syncoid-test-11"
|
||||
TARGET_CHECKSUM="9791444505ef5ab4ac8c943cdcbbb99b98fefc0ee658ac048505cc647e25a1f6 -"
|
||||
|
||||
truncate -s "${POOL_SIZE}" "${POOL_IMAGE}"
|
||||
|
||||
zpool create -m "${MOUNT_TARGET}" -f "${POOL_NAME}" "${POOL_IMAGE}"
|
||||
|
||||
function cleanUp {
|
||||
zpool export "${POOL_NAME}"
|
||||
}
|
||||
|
||||
# export pool in any case
|
||||
trap cleanUp EXIT
|
||||
|
||||
zfs create "${POOL_NAME}"/a
|
||||
zfs snapshot "${POOL_NAME}"/a@s0
|
||||
|
||||
# This fully replicates a to b
|
||||
../../../syncoid --debug --no-sync-snap --no-rollback --create-bookmark "${POOL_NAME}"/a "${POOL_NAME}"/b
|
||||
|
||||
echo "Test 1" > "${MOUNT_TARGET}"/a/file1
|
||||
zfs snapshot "${POOL_NAME}"/a@s1
|
||||
|
||||
# This incrementally replicates from a@s0 to a@s1
|
||||
../../../syncoid --debug --no-sync-snap --no-rollback --create-bookmark "${POOL_NAME}"/a "${POOL_NAME}"/b
|
||||
|
||||
echo "Test 2" > "${MOUNT_TARGET}"/a/file2
|
||||
zfs snapshot "${POOL_NAME}"/a@s2
|
||||
|
||||
# Destroy latest common snap between a and b
|
||||
zfs destroy "${POOL_NAME}"/a@s1
|
||||
|
||||
# This uses a#s1 as base snap although common but older snap a@s0 exists
|
||||
../../../syncoid --debug --no-sync-snap --no-rollback --create-bookmark "${POOL_NAME}"/a "${POOL_NAME}"/b
|
||||
|
||||
echo "Test 3" > "${MOUNT_TARGET}"/a/file3
|
||||
zfs snapshot "${POOL_NAME}"/a@s3
|
||||
|
||||
# This uses a@s2 as base snap again
|
||||
../../../syncoid --debug --no-sync-snap --no-rollback --create-bookmark "${POOL_NAME}"/a "${POOL_NAME}"/b
|
||||
|
||||
# verify
|
||||
output=$(zfs list -t snapshot -r -H -o name "${POOL_NAME}")
|
||||
checksum=$(echo "${output}" | shasum -a 256)
|
||||
|
||||
if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
Loading…
Reference in New Issue