cyteen
/
sanoid
mirror of https://github.com/jimsalterjrs/sanoid
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into silence-excludes

This commit is contained in:
Christoph Klaffl 2019-01-25 16:09:57 +01:00
parent e4e90659a1 9ec0c0bfb1
commit 6f5e8463c2
No known key found for this signature in database
GPG Key ID: FC1C525C2A47CC28
9 changed files with 73 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
sanoid
View File

@ -377,9 +377,9 @@ sub take_snapshots {
if ($config{$section}{'process_children_only'}) { next; }
my $path = $config{$section}{'path'};
my @types = ('yearly','monthly','weekly','daily','hourly','frequently');
foreach my $type (keys %{ $config{$section} }){
unless ($type =~ /ly$/) { next; }
foreach my $type (@types) {
if ($config{$section}{$type} > 0) {
my $newestage; # in seconds
@ -513,11 +513,15 @@ sub take_snapshots {
my $dataset = (split '@', $snap)[0];
my $snapname = (split '@', $snap)[1];
my $presnapshotfailure = 0;
if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) {
my $ret = 0;
if ($config{$dataset}{'pre_snapshot_script'}) {
$ENV{'SANOID_TARGET'} = $dataset;
$ENV{'SANOID_SNAPNAME'} = $snapname;
if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; }
my $ret = runscript('pre_snapshot_script',$dataset);
if (!$args{'readonly'}) {
$ret = runscript('pre_snapshot_script',$dataset);
}
delete $ENV{'SANOID_TARGET'};
delete $ENV{'SANOID_SNAPNAME'};
@ -533,12 +537,15 @@ sub take_snapshots {
system($zfs, "snapshot", "$snap") == 0
or warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?";
}
if ($config{$dataset}{'post_snapshot_script'} and !$args{'readonly'}) {
if ($config{$dataset}{'post_snapshot_script'}) {
if (!$presnapshotfailure or $config{$dataset}{'force_post_snapshot_script'}) {
$ENV{'SANOID_TARGET'} = $dataset;
$ENV{'SANOID_SNAPNAME'} = $snapname;
if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; }
runscript('post_snapshot_script',$dataset);
if (!$args{'readonly'}) {
runscript('post_snapshot_script',$dataset);
}
delete $ENV{'SANOID_TARGET'};
delete $ENV{'SANOID_SNAPNAME'};

49
syncoid
View File

@ -14,16 +14,30 @@ use Pod::Usage;
use Time::Local;
use Sys::Hostname;
my $mbuffer_size = "16M";
# Blank defaults to use ssh client's default
# TODO: Merge into a single "sshflags" option?
my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => '');
GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r",
GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "sendoptions=s", "recvoptions=s",
"source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
"debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s",
"no-clone-handling", "no-privilege-elevation", "force-delete", "no-clone-rollback", "no-rollback") or pod2usage(2);
"no-clone-handling", "no-privilege-elevation", "force-delete", "no-clone-rollback", "no-rollback",
"mbuffer-size=s" => \$mbuffer_size) or pod2usage(2);
my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
my $sendoptions = '';
if (length $args{'sendoptions'}) {
$sendoptions = $args{'sendoptions'}
}
my $recvoptions = '';
if (length $args{'recvoptions'}) {
$recvoptions = $args{'recvoptions'}
}
# TODO Expand to accept multiple sources?
if (scalar(@ARGV) != 2) {
print("Source or target not found!\n");
@ -56,7 +70,7 @@ my $pscmd = '/bin/ps';
my $pvcmd = '/usr/bin/pv';
my $mbuffercmd = '/usr/bin/mbuffer';
my $sudocmd = '/usr/bin/sudo';
my $mbufferoptions = '-q -s 128k -m 16M 2>/dev/null';
my $mbufferoptions = "-q -s 128k -m $mbuffer_size 2>/dev/null";
# currently using ls to check for file existence because we aren't depending on perl
# being present on remote machines.
my $lscmd = '/bin/ls';
@ -364,13 +378,13 @@ sub syncdataset {
if (defined $args{'no-stream'}) { $oldestsnap = getnewestsnapshot(\%snaps); }
my $oldestsnapescaped = escapeshellparam($oldestsnap);
my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped";
my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped";
my $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions $sourcefsescaped\@$oldestsnapescaped";
my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped";
my $pvsize;
if (defined $origin) {
my $originescaped = escapeshellparam($origin);
$sendcmd = "$sourcesudocmd $zfscmd send -i $originescaped $sourcefsescaped\@$oldestsnapescaped";
$sendcmd = "$sourcesudocmd $zfscmd send $sendoptions -i $originescaped $sourcefsescaped\@$oldestsnapescaped";
my $streamargBackup = $args{'streamarg'};
$args{'streamarg'} = "-i";
$pvsize = getsendsize($sourcehost,$origin,"$sourcefs\@$oldestsnap",$sourceisroot);
@ -418,7 +432,7 @@ sub syncdataset {
# $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly');
# setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on');
$sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnapescaped $sourcefsescaped\@$newsyncsnapescaped";
$sendcmd = "$sourcesudocmd $zfscmd send $sendoptions $args{'streamarg'} $sourcefsescaped\@$oldestsnapescaped $sourcefsescaped\@$newsyncsnapescaped";
$pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap","$sourcefs\@$newsyncsnap",$sourceisroot);
$disp_pvsize = readablebytes($pvsize);
if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }
@ -455,8 +469,8 @@ sub syncdataset {
# and because this will ony resume the receive to the next
# snapshot, do a normal sync after that
if (defined($receivetoken)) {
my $sendcmd = "$sourcesudocmd $zfscmd send -t $receivetoken";
my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped";
my $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions -t $receivetoken";
my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped";
my $pvsize = getsendsize($sourcehost,"","",$sourceisroot,$receivetoken);
my $disp_pvsize = readablebytes($pvsize);
if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }
@ -607,8 +621,8 @@ sub syncdataset {
if ($nextsnapshot) {
my $nextsnapshotescaped = escapeshellparam($nextsnapshot);
my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$nextsnapshotescaped";
my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped";
my $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$nextsnapshotescaped";
my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped";
my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $nextsnapshot (~ $disp_pvsize):\n"; }
@ -622,8 +636,8 @@ sub syncdataset {
$matchingsnap = $nextsnapshot;
$matchingsnapescaped = escapeshellparam($matchingsnap);
} else {
my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$newsyncsnapescaped";
my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped";
my $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$newsyncsnapescaped";
my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped";
my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $newsyncsnap (~ $disp_pvsize):\n"; }
@ -639,8 +653,8 @@ sub syncdataset {
# do a normal replication if bookmarks aren't used or if previous
# bookmark replication was only done to the next oldest snapshot
if (!$bookmark || $nextsnapshot) {
my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped";
my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped";
my $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped";
my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped";
my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot);
my $disp_pvsize = readablebytes($pvsize);
if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }
@ -1383,7 +1397,7 @@ sub getsendsize {
$snaps = "-t $receivetoken";
}
my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send -nP $snaps";
my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send $sendoptions -nP $snaps";
if ($debug) { print "DEBUG: getting estimated transfer size from source $sourcehost using \"$getsendsizecmd 2>&1 |\"...\n"; }
open FH, "$getsendsizecmd 2>&1 |";
@ -1488,11 +1502,14 @@ Options:
--skip-parent Skips syncing of the parent dataset. Does nothing without '--recursive' option.
--source-bwlimit=<limit k|m|g|t> Bandwidth limit on the source transfer
--target-bwlimit=<limit k|m|g|t> Bandwidth limit on the target transfer
--mbuffer-size=VALUE Specify the mbuffer size (default: 16M), please refer to mbuffer(1) manual page.
--no-stream Replicates using newest snapshot instead of intermediates
--no-sync-snap Does not create new snapshot, only transfers existing
--no-clone-rollback Does not rollback clones on target
--no-rollback Does not rollback clones or snapshots on target (it probably requires a readonly target)
--exclude=REGEX Exclude specific datasets which match the given regular expression. Can be specified multiple times
--sendoptions=OPTIONS DANGER: Inject OPTIONS into zfs send, e.g. syncoid --sendoptions="-Lce" sets zfs send -Lce ...
--recvoptions=OPTIONS DANGER: Inject OPTIONS into zfs received, e.g. syncoid --recvoptions="-x property" sets zfs receive -x property ...
--sshkey=FILE Specifies a ssh public key to use to connect
--sshport=PORT Connects to remote on a particular port
--sshcipher|c=CIPHER Passes CIPHER to ssh to use a particular cipher set

2
tests/1_one_year/run.sh
View File

@ -39,7 +39,7 @@ function cleanUp {
trap cleanUp EXIT
while [ $timestamp -le $END ]; do
date --utc --set @$timestamp; date; "${SANOID}" --cron --verbose
setdate $timestamp; date; "${SANOID}" --cron --verbose
timestamp=$((timestamp+3600))
done

2
tests/2_dst_handling/run.sh
View File

@ -42,7 +42,7 @@ function cleanUp {
trap cleanUp EXIT
while [ $timestamp -le $END ]; do
date --utc --set @$timestamp; date; "${SANOID}" --cron --verbose
setdate $timestamp; date; "${SANOID}" --cron --verbose
timestamp=$((timestamp+900))
done

20
tests/common/lib.sh
View File

@ -1,5 +1,7 @@
#!/bin/bash
unamestr="$(uname)"
function setup {
export LANG=C
export LANGUAGE=C
@ -58,7 +60,11 @@ function saveSnapshotList {
zfs list -t snapshot -o name -Hr "${POOL_NAME}" | sort > "${RESULT}"
# clear the seconds for comparing
sed -i 's/\(autosnap_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9]:[0-9][0-9]:\)[0-9][0-9]_/\100_/g' "${RESULT}"
if [ "$unamestr" == 'FreeBSD' ]; then
sed -i '' 's/\(autosnap_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9]:[0-9][0-9]:\)[0-9][0-9]_/\100_/g' "${RESULT}"
else
sed -i 's/\(autosnap_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9]:[0-9][0-9]:\)[0-9][0-9]_/\100_/g' "${RESULT}"
fi
}
function verifySnapshotList {
@ -90,7 +96,7 @@ function verifySnapshotList {
message="${message}monthly snapshot count is wrong: ${monthly_count}\n"
fi
checksum=$(sha256sum "${RESULT}" | cut -d' ' -f1)
checksum=$(shasum -a 256 "${RESULT}" | cut -d' ' -f1)
if [ "${checksum}" != "${CHECKSUM}" ]; then
failed=1
message="${message}result checksum mismatch\n"
@ -105,3 +111,13 @@ function verifySnapshotList {
exit 1
}
function setdate {
TIMESTAMP="$1"
if [ "$unamestr" == 'FreeBSD' ]; then
date -u -f '%s' "${TIMESTAMP}"
else
date --utc --set "@${TIMESTAMP}"
fi
}

2
tests/run-tests.sh
View File

@ -15,7 +15,7 @@ for test in */; do
echo -n "Running test ${testName} ... "
cd "${test}"
echo | bash run.sh > "${LOGFILE}" 2>&1
echo -n y | bash run.sh > "${LOGFILE}" 2>&1
if [ $? -eq 0 ]; then
echo "[PASS]"

4
tests/syncoid/1_bookmark_replication_intermediate/run.sh
View File

@ -46,8 +46,8 @@ zfs snapshot "${POOL_NAME}"/src@snap5
../../../syncoid --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1
# verify
output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name)
checksum=$(echo "${output}" | grep -v syncoid_ | sha256sum)
output=$(zfs list -t snapshot -r -H -o name "${POOL_NAME}")
checksum=$(echo "${output}" | grep -v syncoid_ | shasum -a 256)
if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then
exit 1

4
tests/syncoid/2_bookmark_replication_no_intermediate/run.sh
View File

@ -46,8 +46,8 @@ zfs snapshot "${POOL_NAME}"/src@snap5
../../../syncoid --no-stream --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1
# verify
output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name)
checksum=$(echo "${output}" | sha256sum)
output=$(zfs list -t snapshot -r -H -o name "${POOL_NAME}")
checksum=$(echo "${output}" | shasum -a 256)
if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then
exit 1

4
tests/syncoid/3_force_delete/run.sh
View File

@ -37,8 +37,8 @@ sleep 1
../../../syncoid -r --force-delete --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1
# verify
output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name | sed 's/@syncoid_.*$'/@syncoid_/)
checksum=$(echo "${output}" | sha256sum)
output=$(zfs list -t snapshot -r -H -o name "${POOL_NAME}" | sed 's/@syncoid_.*$'/@syncoid_/)
checksum=$(echo "${output}" | shasum -a 256)
if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then
exit 1