From d8613d13797db640f9cf7432f29677056bddeb43 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 7 Nov 2017 08:03:53 +0100 Subject: [PATCH 001/148] implemented frequent snapshots with configurable period Fixes #75 --- sanoid | 21 +++++++++++++++++---- sanoid.conf | 2 ++ sanoid.defaults.conf | 7 +++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/sanoid b/sanoid index d6e58ce..e8756db 100755 --- a/sanoid +++ b/sanoid @@ -121,12 +121,13 @@ sub monitor_snapshots { my $path = $config{$section}{'path'}; push @paths, $path; - my @types = ('yearly','monthly','daily','hourly'); + my @types = ('yearly','monthly','daily','hourly','frequently'); foreach my $type (@types) { my $smallerperiod = 0; # we need to set the period length in seconds first - if ($type eq 'hourly') { $smallerperiod = 60; } + if ($type eq 'frequently') { $smallerperiod = 1; } + elsif ($type eq 'hourly') { $smallerperiod = 60; } elsif ($type eq 'daily') { $smallerperiod = 60*60; } elsif ($type eq 'monthly') { $smallerperiod = 60*60*24; } elsif ($type eq 'yearly') { $smallerperiod = 60*60*24; } @@ -200,7 +201,8 @@ sub prune_snapshots { unless ($type =~ /ly$/) { next; } # we need to set the period length in seconds first - if ($type eq 'hourly') { $period = 60*60; } + if ($type eq 'frequently') { $period = 60 * $config{$section}{'frequent_period'}; } + elsif ($type eq 'hourly') { $period = 60*60; } elsif ($type eq 'daily') { $period = 60*60*24; } elsif ($type eq 'monthly') { $period = 60*60*24*31; } elsif ($type eq 'yearly') { $period = 60*60*24*365.25; } @@ -291,7 +293,18 @@ sub take_snapshots { my @preferredtime; my $lastpreferred; - if ($type eq 'hourly') { + if ($type eq 'frequently') { + my $frequentslice = int($datestamp{'min'} / $config{$section}{'frequent_period'}); + + push @preferredtime,0; # try to hit 0 seconds + push @preferredtime,$frequentslice * $config{$section}{'frequent_period'}; + push @preferredtime,$datestamp{'hour'}; + push @preferredtime,$datestamp{'mday'}; + push @preferredtime,($datestamp{'mon'}-1); # january is month 0 + push @preferredtime,$datestamp{'year'}; + $lastpreferred = timelocal(@preferredtime); + if ($lastpreferred > time()) { $lastpreferred -= 60 * $config{$section}{'frequent_period'}; } # preferred time is later this frequent period - so look at last frequent period + } elsif ($type eq 'hourly') { push @preferredtime,0; # try to hit 0 seconds push @preferredtime,$config{$section}{'hourly_min'}; push @preferredtime,$datestamp{'hour'}; diff --git a/sanoid.conf b/sanoid.conf index 9b1f19d..b999634 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -40,6 +40,7 @@ daily = 60 [template_production] + frequently = 0 hourly = 36 daily = 30 monthly = 3 @@ -49,6 +50,7 @@ [template_backup] autoprune = yes + frequently = 0 hourly = 30 daily = 90 monthly = 12 diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 35c804d..b5e4e63 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -16,12 +16,17 @@ recursive = use_template = process_children_only = +# The period in minutes for frequent snapshots, +# should be in the range of 1-30 and divide an hour without remainder +frequent_period = 15 + # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately # prune any of those type snapshots already present. # # Otherwise, if autoprune is set, we will prune any snapshots of that type which are older # than (setting * periodicity) - so if daily = 90, we'll prune any dailies older than 90 days. autoprune = yes +frequently = 0 hourly = 48 daily = 90 monthly = 6 @@ -62,6 +67,8 @@ yearly_min = 0 monitor = yes monitor_dont_warn = no monitor_dont_crit = no +frequently_warn = 2000 +frequently_crit = 8000 hourly_warn = 90 hourly_crit = 360 daily_warn = 28 From c9adcdab1e7e2a7eaa04bdbc15a24881278cefdb Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 8 Nov 2017 20:25:07 +0100 Subject: [PATCH 002/148] hardcoded new defaults --- sanoid | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/sanoid b/sanoid index e8756db..6428cdf 100755 --- a/sanoid +++ b/sanoid @@ -553,6 +553,18 @@ sub getsnaps { #################################################################################### #################################################################################### +sub verify_option_existence { + my ($hash, $key, $default) = @_; + + if (! defined (%$hash{$key})) { + $hash->{$key} = $default; + } +} + +#################################################################################### +#################################################################################### +#################################################################################### + sub init { my ($conf_file, $default_conf_file) = @_; my %config; @@ -568,6 +580,12 @@ sub init { my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON"); my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF"); + # hardcoded defaults which may be missing from older default configuration file + verify_option_existence($defaults{'template_default'}, 'frequent_period', 15); + verify_option_existence($defaults{'template_default'}, 'frequently', 0); + verify_option_existence($defaults{'template_default'}, 'frequently_warn', 2000); + verify_option_existence($defaults{'template_default'}, 'frequently_crit', 8000); + foreach my $section (keys %ini) { # first up - die with honor if unknown parameters are set in any modules or templates by the user. From 8bd98f18008494e27c47c6d16cc1fbbaec9a33a6 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 8 Nov 2017 21:40:30 +0100 Subject: [PATCH 003/148] added more documentation for frequent snapshots --- README.md | 1 + sanoid.defaults.conf | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9c4e6ba..863b697 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ And its /etc/sanoid/sanoid.conf might look something like this: ############################# [template_production] + frequently = 0 hourly = 36 daily = 30 monthly = 3 diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index b5e4e63..e7c22a8 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -16,8 +16,16 @@ recursive = use_template = process_children_only = -# The period in minutes for frequent snapshots, -# should be in the range of 1-30 and divide an hour without remainder +# for snapshots shorter than one hour, the period duration must be defined +# in minutes. Because they are executed within a full hour, the selected +# value should divide 60 minutes without remainder so taken snapshots +# are apart in equal intervals. Values larger than 59 aren't practical +# as only one snapshot will be taken on each full hour in this case. +# examples: +# frequent_period = 15 -> four snapshot each hour 15 minutes apart +# frequent_period = 5 -> twelve snapshots each hour 5 minutes apart +# frequent_period = 45 -> two snapshots each hour with different time gaps +# between them: 45 minutes and 15 minutes in this case frequent_period = 15 # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately From bc2f8bd4e9b150294bad5c3150a682a3e857e3e0 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 8 Nov 2017 21:48:37 +0100 Subject: [PATCH 004/148] Revert "hardcoded new defaults" This reverts commit c9adcdab1e7e2a7eaa04bdbc15a24881278cefdb. --- sanoid | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/sanoid b/sanoid index 6428cdf..e8756db 100755 --- a/sanoid +++ b/sanoid @@ -553,18 +553,6 @@ sub getsnaps { #################################################################################### #################################################################################### -sub verify_option_existence { - my ($hash, $key, $default) = @_; - - if (! defined (%$hash{$key})) { - $hash->{$key} = $default; - } -} - -#################################################################################### -#################################################################################### -#################################################################################### - sub init { my ($conf_file, $default_conf_file) = @_; my %config; @@ -580,12 +568,6 @@ sub init { my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON"); my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF"); - # hardcoded defaults which may be missing from older default configuration file - verify_option_existence($defaults{'template_default'}, 'frequent_period', 15); - verify_option_existence($defaults{'template_default'}, 'frequently', 0); - verify_option_existence($defaults{'template_default'}, 'frequently_warn', 2000); - verify_option_existence($defaults{'template_default'}, 'frequently_crit', 8000); - foreach my $section (keys %ini) { # first up - die with honor if unknown parameters are set in any modules or templates by the user. From 293d83bdaa6155e765432a9a145a786d72afed9e Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 8 Nov 2017 22:08:02 +0100 Subject: [PATCH 005/148] versioning and compatibility check for default configuration file --- sanoid | 12 ++++++++++++ sanoid.defaults.conf | 2 ++ 2 files changed, 14 insertions(+) diff --git a/sanoid b/sanoid index e8756db..452f535 100755 --- a/sanoid +++ b/sanoid @@ -5,6 +5,7 @@ # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. $::VERSION = '1.4.17'; +my $MINIMUM_DEFAULTS_VERSION = 2; use strict; use warnings; @@ -568,6 +569,17 @@ sub init { my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON"); my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF"); + # check if default configuration file is up to date + my $defaults_version = 1; + if (defined $defaults{'version'}{'version'}) { + $defaults_version = $defaults{'version'}{'version'}; + delete $defaults{'version'}; + } + + if ($defaults_version < $MINIMUM_DEFAULTS_VERSION) { + die "FATAL: you're using sanoid.defaults.conf v$defaults_version, this version of sanoid requires a minimum sanoid.defaults.conf v$MINIMUM_DEFAULTS_VERSION"; + } + foreach my $section (keys %ini) { # first up - die with honor if unknown parameters are set in any modules or templates by the user. diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index e7c22a8..06fc714 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -5,6 +5,8 @@ # # # you have been warned. # ################################################################################### +[version] +version = 2 [template_default] From 00b920682913641732981c233acd9262671719af Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 9 Nov 2017 09:44:40 +0100 Subject: [PATCH 006/148] implemented weekly period --- sanoid | 26 +++++++++++++++++++++++--- sanoid.defaults.conf | 11 +++++++++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/sanoid b/sanoid index d6e58ce..9b6c815 100755 --- a/sanoid +++ b/sanoid @@ -121,15 +121,16 @@ sub monitor_snapshots { my $path = $config{$section}{'path'}; push @paths, $path; - my @types = ('yearly','monthly','daily','hourly'); + my @types = ('yearly','monthly', 'weekly', 'daily','hourly'); foreach my $type (@types) { my $smallerperiod = 0; # we need to set the period length in seconds first if ($type eq 'hourly') { $smallerperiod = 60; } elsif ($type eq 'daily') { $smallerperiod = 60*60; } - elsif ($type eq 'monthly') { $smallerperiod = 60*60*24; } - elsif ($type eq 'yearly') { $smallerperiod = 60*60*24; } + elsif ($type eq 'weekly') { $smallerperiod = 60*60*24; } + elsif ($type eq 'monthly') { $smallerperiod = 60*60*24*7; } + elsif ($type eq 'yearly') { $smallerperiod = 60*60*24*31; } my $typewarn = $type . '_warn'; my $typecrit = $type . '_crit'; @@ -202,6 +203,7 @@ sub prune_snapshots { # we need to set the period length in seconds first if ($type eq 'hourly') { $period = 60*60; } elsif ($type eq 'daily') { $period = 60*60*24; } + elsif ($type eq 'weekly') { $period = 60*60*24*7; } elsif ($type eq 'monthly') { $period = 60*60*24*31; } elsif ($type eq 'yearly') { $period = 60*60*24*365.25; } @@ -309,6 +311,24 @@ sub take_snapshots { push @preferredtime,$datestamp{'year'}; $lastpreferred = timelocal(@preferredtime); if ($lastpreferred > time()) { $lastpreferred -= 60*60*24; } # preferred time is later today - so look at yesterday's + } elsif ($type eq 'weekly') { + # calculate offset in seconds for the desired weekday + my $offset; + if ($config{$section}{'weekly_wday'} < $datestamp{'wday'}) { + $offset += 6; + } + $offset += $config{$section}{'weekly_wday'} - $datestamp{'wday'}; + $offset *= 60*60*24; # full day + + push @preferredtime,0; # try to hit 0 seconds + push @preferredtime,$config{$section}{'weekly_min'}; + push @preferredtime,$config{$section}{'weekly_hour'}; + push @preferredtime,$datestamp{'mday'}; + push @preferredtime,($datestamp{'mon'}-1); # january is month 0 + push @preferredtime,$datestamp{'year'}; + $lastpreferred = timelocal(@preferredtime); + $lastpreferred -= $offset; + if ($lastpreferred > time()) { $lastpreferred -= 60*60*24*7; } # preferred time is later today - so look at yesterday's } elsif ($type eq 'monthly') { push @preferredtime,0; # try to hit 0 seconds push @preferredtime,$config{$section}{'monthly_min'}; diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 35c804d..7187104 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -24,6 +24,7 @@ process_children_only = autoprune = yes hourly = 48 daily = 90 +weekly = 0 monthly = 6 yearly = 0 min_percent_free = 10 @@ -40,6 +41,10 @@ hourly_min = 0 # daily - at 23:59 (most people expect a daily to contain everything done DURING that day) daily_hour = 23 daily_min = 59 +# weekly -at 23:30 each Monday +weekly_wday = 1 +weekly_hour = 23 +weekly_min = 30 # monthly - immediately at the beginning of the month (ie 00:00 of day 1) monthly_mday = 1 monthly_hour = 0 @@ -66,7 +71,9 @@ hourly_warn = 90 hourly_crit = 360 daily_warn = 28 daily_crit = 32 -monthly_warn = 32 -monthly_crit = 35 +weekly_warn = 7 +weekly_crit = 10 +monthly_warn = 5 +monthly_crit = 6 yearly_warn = 0 yearly_crit = 0 From 4649704046cee190c8482e00e79b9b8d408829e4 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 9 Nov 2017 17:41:52 +0100 Subject: [PATCH 007/148] codestyle fix and disable monitoring of weekly snapshots --- sanoid | 2 +- sanoid.defaults.conf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sanoid b/sanoid index 9b6c815..05c4def 100755 --- a/sanoid +++ b/sanoid @@ -121,7 +121,7 @@ sub monitor_snapshots { my $path = $config{$section}{'path'}; push @paths, $path; - my @types = ('yearly','monthly', 'weekly', 'daily','hourly'); + my @types = ('yearly','monthly','weekly','daily','hourly'); foreach my $type (@types) { my $smallerperiod = 0; diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 7187104..ff79ebb 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -71,8 +71,8 @@ hourly_warn = 90 hourly_crit = 360 daily_warn = 28 daily_crit = 32 -weekly_warn = 7 -weekly_crit = 10 +weekly_warn = 0 +weekly_crit = 0 monthly_warn = 5 monthly_crit = 6 yearly_warn = 0 From ac16b2128e6863a8af619d0bf9282bf92f6dfbc9 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 9 Nov 2017 17:44:04 +0100 Subject: [PATCH 008/148] disable monitoring of frequent snapshots --- sanoid.defaults.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 06fc714..a521401 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -77,8 +77,8 @@ yearly_min = 0 monitor = yes monitor_dont_warn = no monitor_dont_crit = no -frequently_warn = 2000 -frequently_crit = 8000 +frequently_warn = 0 +frequently_crit = 0 hourly_warn = 90 hourly_crit = 360 daily_warn = 28 From 4a3e93372c3502a7a4d9e2202d147ab00fd23559 Mon Sep 17 00:00:00 2001 From: Jason Lewis Date: Mon, 20 Nov 2017 15:16:43 +1100 Subject: [PATCH 009/148] check for emtpy lockfile --- sanoid | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index d6e58ce..889696f 100755 --- a/sanoid +++ b/sanoid @@ -930,13 +930,22 @@ sub checklock { # no lockfile return 1; } + # make sure lockfile contains something + if ( -z $lockfile) { + # zero size lockfile, something is wrong + die "ERROR: something is wrong! $lockfile is empty\n"; + } # lockfile exists. read pid and mutex from it. see if it's our pid. if not, see if # there's still a process running with that pid and with the same mutex. - open FH, "< $lockfile"; + open FH, "< $lockfile" or die "ERROR: unable to open $lockfile"; my @lock = ; close FH; + # if we didn't get exactly 2 items from the lock file there is a problem + if (scalar(@lock) != 2) { + die "ERROR: $lockfile is invalid.\n" + } my $lockmutex = pop(@lock); my $lockpid = pop(@lock); @@ -948,7 +957,6 @@ sub checklock { # we own the lockfile. no need to check any further. return 2; } - open PL, "$pscmd -p $lockpid -o args= |"; my @processlist = ; close PL; From 31da53140fda7c5bfc79d063e3c6da8d8f49300c Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 19:00:44 +0100 Subject: [PATCH 010/148] implemented a simple test which will take snapshots over a whole year and checks the resulting snapshot list --- tests/1_one_year/run.sh | 55 ++++++++++++++++++ tests/1_one_year/sanoid.conf | 10 ++++ tests/common/lib.sh | 106 +++++++++++++++++++++++++++++++++++ tests/run-tests.sh | 27 +++++++++ 4 files changed, 198 insertions(+) create mode 100755 tests/1_one_year/run.sh create mode 100644 tests/1_one_year/sanoid.conf create mode 100644 tests/common/lib.sh create mode 100755 tests/run-tests.sh diff --git a/tests/1_one_year/run.sh b/tests/1_one_year/run.sh new file mode 100755 index 0000000..7cec813 --- /dev/null +++ b/tests/1_one_year/run.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -x + +# this test will take hourly, daily and monthly snapshots +# for the whole year of 2017 in the timezone Europe/Vienna +# sanoid is run hourly and no snapshots are pruned + +. ../common/lib.sh + +POOL_NAME="sanoid-test-1" +POOL_TARGET="" # root +RESULT="/tmp/sanoid_test_result" +RESULT_CHECKSUM="aa15e5595b0ed959313289ecb70323dad9903328ac46e881da5c4b0f871dd7cf" + +# UTC timestamp of start and end +START="1483225200" +END="1514761199" + +# prepare +setup +checkEnvironment +disableTimeSync + +# set timezone +ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime + +timestamp=$START + +mkdir -p "${POOL_TARGET}" +truncate -s 5120M "${POOL_TARGET}"/zpool.img + +zpool create -f "${POOL_NAME}" "${POOL_TARGET}"/zpool.img + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +while [ $timestamp -le $END ]; do + date --utc --set @$timestamp; date; "${SANOID}" --cron --verbose + timestamp=$((timestamp+3600)) +done + +saveSnapshotList "${POOL_NAME}" "${RESULT}" + +# hourly daily monthly +verifySnapshotList "${RESULT}" 8759 366 12 "${RESULT_CHECKSUM}" + +# hourly count should be 8760 but one hour get's lost because of DST + +# daily count should be 365 but one additional daily is taken +# because the DST change leads to a day with 25 hours +# which will trigger an additional daily snapshot diff --git a/tests/1_one_year/sanoid.conf b/tests/1_one_year/sanoid.conf new file mode 100644 index 0000000..f5692f0 --- /dev/null +++ b/tests/1_one_year/sanoid.conf @@ -0,0 +1,10 @@ +[sanoid-test-1] + use_template = production + +[template_production] + hourly = 36 + daily = 30 + monthly = 3 + yearly = 0 + autosnap = yes + autoprune = no diff --git a/tests/common/lib.sh b/tests/common/lib.sh new file mode 100644 index 0000000..2c15e9b --- /dev/null +++ b/tests/common/lib.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +function setup { + export LANG=C + export LANGUAGE=C + export LC_ALL=C + + export SANOID="../../sanoid" + + # make sure that there is no cache file + rm -f /var/cache/sanoidsnapshots.txt + + # install needed sanoid configuration files + [ -f sanoid.conf ] && cp sanoid.conf /etc/sanoid/sanoid.conf + cp ../../sanoid.defaults.conf /etc/sanoid/sanoid.defaults.conf +} + +function checkEnvironment { + ASK=1 + + which systemd-detect-virt > /dev/null + if [ $? -eq 0 ]; then + systemd-detect-virt --vm > /dev/null + if [ $? -eq 0 ]; then + # we are in a vm + ASK=0 + fi + fi + + if [ $ASK -eq 1 ]; then + set +x + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "you should be running this test in a" + echo "dedicated vm, as it will mess with your system!" + echo "Are you sure you wan't to continue? (y)" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + set -x + + read -n 1 c + if [ "$c" != "y" ]; then + exit 1 + fi + fi +} + +function disableTimeSync { + # disable ntp sync + which timedatectl > /dev/null + if [ $? -eq 0 ]; then + timedatectl set-ntp 0 + fi +} + +function saveSnapshotList { + POOL_NAME="$1" + RESULT="$2" + + zfs list -t snapshot -o name -Hr "${POOL_NAME}" | sort > "${RESULT}" + + # clear the seconds for comparing + sed -i 's/\(autosnap_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9]:[0-9][0-9]:\)[0-9][0-9]_/\100_/g' "${RESULT}" +} + +function verifySnapshotList { + RESULT="$1" + HOURLY_COUNT=$2 + DAILY_COUNT=$3 + MONTHLY_COUNT=$4 + CHECKSUM="$5" + + failed=0 + message="" + + hourly_count=$(grep -c "autosnap_.*_hourly" < "${RESULT}") + daily_count=$(grep -c "autosnap_.*_daily" < "${RESULT}") + monthly_count=$(grep -c "autosnap_.*_monthly" < "${RESULT}") + + if [ "${hourly_count}" -ne "${HOURLY_COUNT}" ]; then + failed=1 + message="${message}hourly snapshot count is wrong: ${hourly_count}\n" + fi + + if [ "${daily_count}" -ne "${DAILY_COUNT}" ]; then + failed=1 + message="${message}daily snapshot count is wrong: ${daily_count}\n" + fi + + if [ "${monthly_count}" -ne "${MONTHLY_COUNT}" ]; then + failed=1 + message="${message}monthly snapshot count is wrong: ${monthly_count}\n" + fi + + checksum=$(sha256sum "${RESULT}" | cut -d' ' -f1) + if [ "${checksum}" != "${CHECKSUM}" ]; then + failed=1 + message="${message}result checksum mismatch\n" + fi + + if [ "${failed}" -eq 0 ]; then + exit 0 + fi + + echo "TEST FAILED:" >&2 + echo -n -e "${message}" >&2 + +} diff --git a/tests/run-tests.sh b/tests/run-tests.sh new file mode 100755 index 0000000..a8469e9 --- /dev/null +++ b/tests/run-tests.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# run's all the available tests + +for test in */; do + if [ ! -x "${test}/run.sh" ]; then + continue + fi + + testName="${test%/}" + + LOGFILE=/tmp/sanoid_test_run_"${testName}".log + + pushd . > /dev/null + + echo -n "Running test ${testName} ... " + cd "${test}" + echo | bash run.sh > "${LOGFILE}" 2>&1 + + if [ $? -eq 0 ]; then + echo "[PASS]" + else + echo "[FAILED] (see ${LOGFILE})" + fi + + popd > /dev/null +done From 9a6cdb85438eb730348a35e23fd4fdf7b41b60f3 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 20:15:29 +0100 Subject: [PATCH 011/148] handle DST (daylight saving times) properly for hourly and daily snapshots, fixes #155 --- sanoid | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index d6e58ce..030fea2 100755 --- a/sanoid +++ b/sanoid @@ -268,6 +268,19 @@ sub take_snapshots { my @newsnaps; + # get utc timestamp of the current day for DST check + my $daystartUtc = timelocal(0, 0, 0, $datestamp{'mday'}, ($datestamp{'mon'}-1), $datestamp{'year'}); + my ($isdst) = (localtime($daystartUtc))[8]; + my $dstOffset = 0; + + if ($isdst ne $datestamp{'isdst'}) { + # current dst is different then at the beginning og the day + if ($isdst) { + # DST ended in the current day + $dstOffset = 60*60; + } + } + if ($args{'verbose'}) { print "INFO: taking snapshots...\n"; } foreach my $section (keys %config) { if ($section =~ /^template/) { next; } @@ -291,6 +304,9 @@ sub take_snapshots { my @preferredtime; my $lastpreferred; + # to avoid duplicates with DST + my $dateSuffix = ""; + if ($type eq 'hourly') { push @preferredtime,0; # try to hit 0 seconds push @preferredtime,$config{$section}{'hourly_min'}; @@ -299,6 +315,13 @@ sub take_snapshots { push @preferredtime,($datestamp{'mon'}-1); # january is month 0 push @preferredtime,$datestamp{'year'}; $lastpreferred = timelocal(@preferredtime); + + if ($dstOffset ne 0) { + # timelocal doesn't take DST into account + $lastpreferred += $dstOffset; + # DST ended, avoid duplicates + $dateSuffix = "_y"; + } if ($lastpreferred > time()) { $lastpreferred -= 60*60; } # preferred time is later this hour - so look at last hour's } elsif ($type eq 'daily') { push @preferredtime,0; # try to hit 0 seconds @@ -308,7 +331,29 @@ sub take_snapshots { push @preferredtime,($datestamp{'mon'}-1); # january is month 0 push @preferredtime,$datestamp{'year'}; $lastpreferred = timelocal(@preferredtime); - if ($lastpreferred > time()) { $lastpreferred -= 60*60*24; } # preferred time is later today - so look at yesterday's + + # timelocal doesn't take DST into account + $lastpreferred += $dstOffset; + + # check if the planned time has different DST flag than the current + my ($isdst) = (localtime($lastpreferred))[8]; + if ($isdst ne $datestamp{'isdst'}) { + if (!$isdst) { + # correct DST difference + $lastpreferred -= 60*60; + } + } + + if ($lastpreferred > time()) { + $lastpreferred -= 60*60*24; + + if ($dstOffset ne 0) { + # because we are going back one day + # the DST difference has to be accounted + # for in reverse now + $lastpreferred -= 2*$dstOffset; + } + } # preferred time is later today - so look at yesterday's } elsif ($type eq 'monthly') { push @preferredtime,0; # try to hit 0 seconds push @preferredtime,$config{$section}{'monthly_min'}; @@ -336,7 +381,7 @@ sub take_snapshots { # update to most current possible datestamp %datestamp = get_date(); # print "we should have had a $type snapshot of $path $maxage seconds ago; most recent is $newestage seconds old.\n"; - push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}_$type"); + push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}_${dateSuffix}_$type"); } } } From c1f7cd4241cefcafc1951b1e005b4c0a566ac062 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 20:23:52 +0100 Subject: [PATCH 012/148] fixed snapshot suffix --- sanoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 030fea2..1c470c3 100755 --- a/sanoid +++ b/sanoid @@ -381,7 +381,7 @@ sub take_snapshots { # update to most current possible datestamp %datestamp = get_date(); # print "we should have had a $type snapshot of $path $maxage seconds ago; most recent is $newestage seconds old.\n"; - push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}_${dateSuffix}_$type"); + push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}${dateSuffix}_$type"); } } } From e61ccf1c9dcccb1e156ed684413e0c0a5671664a Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 20:24:54 +0100 Subject: [PATCH 013/148] added test for checking for correct DST handling behaviour --- tests/2_dst_handling/run.sh | 54 ++++++++++++++++++++++++++++++++ tests/2_dst_handling/sanoid.conf | 10 ++++++ 2 files changed, 64 insertions(+) create mode 100755 tests/2_dst_handling/run.sh create mode 100644 tests/2_dst_handling/sanoid.conf diff --git a/tests/2_dst_handling/run.sh b/tests/2_dst_handling/run.sh new file mode 100755 index 0000000..eba21ed --- /dev/null +++ b/tests/2_dst_handling/run.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -x + +# this test will check the behaviour arround a date where DST ends +# with hourly, daily and monthly snapshots checked in a 15 minute interval + +# Daylight saving time 2017 in Europe/Vienna began at 02:00 on Sunday, 26 March +# and ended at 03:00 on Sunday, 29 October. All times are in +# Central European Time. + +. ../common/lib.sh + +POOL_NAME="sanoid-test-2" +POOL_TARGET="" # root +RESULT="/tmp/sanoid_test_result" +RESULT_CHECKSUM="a916d9cd46f4b80f285d069f3497d02671bbb1bfd12b43ef93531cbdaf89d55c" + +# UTC timestamp of start and end +START="1509141600" +END="1509400800" + +# prepare +setup +checkEnvironment +disableTimeSync + +# set timezone +ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime + +timestamp=$START + +mkdir -p "${POOL_TARGET}" +truncate -s 512M "${POOL_TARGET}"/zpool2.img + +zpool create -f "${POOL_NAME}" "${POOL_TARGET}"/zpool2.img + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +while [ $timestamp -le $END ]; do + date --utc --set @$timestamp; date; "${SANOID}" --cron --verbose + timestamp=$((timestamp+900)) +done + +saveSnapshotList "${POOL_NAME}" "${RESULT}" + +# hourly daily monthly +verifySnapshotList "${RESULT}" 73 3 1 "${RESULT_CHECKSUM}" + +# one more hour because of DST diff --git a/tests/2_dst_handling/sanoid.conf b/tests/2_dst_handling/sanoid.conf new file mode 100644 index 0000000..7ded3f8 --- /dev/null +++ b/tests/2_dst_handling/sanoid.conf @@ -0,0 +1,10 @@ +[sanoid-test-2] + use_template = production + +[template_production] + hourly = 36 + daily = 30 + monthly = 3 + yearly = 0 + autosnap = yes + autoprune = no From 8d4484a2d1641789c1e667c03e69773e96cde6ea Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 23:24:34 +0100 Subject: [PATCH 014/148] exit with error code upon failure --- debian/changelog | 4 ++++ tests/common/lib.sh | 1 + 2 files changed, 5 insertions(+) diff --git a/debian/changelog b/debian/changelog index ab530b0..beb6584 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,7 @@ +sanoid (1.4.17-SNAPSHOT) unstable; urgency=medium + +-- Jim Salter Wed, 9 Aug 2017 12:28:49 -0400 + sanoid (1.4.16) unstable; urgency=medium * merged @hrast01's extended fix to support -o option1=val,option2=val passthrough to SSH. merged @JakobR's diff --git a/tests/common/lib.sh b/tests/common/lib.sh index 2c15e9b..78f128b 100644 --- a/tests/common/lib.sh +++ b/tests/common/lib.sh @@ -103,4 +103,5 @@ function verifySnapshotList { echo "TEST FAILED:" >&2 echo -n -e "${message}" >&2 + exit 1 } From 53894b2855016e6398609995ee383c173fc2ff99 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 23:36:13 +0100 Subject: [PATCH 015/148] indentation fix --- sanoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 1c470c3..12d746d 100755 --- a/sanoid +++ b/sanoid @@ -317,7 +317,7 @@ sub take_snapshots { $lastpreferred = timelocal(@preferredtime); if ($dstOffset ne 0) { - # timelocal doesn't take DST into account + # timelocal doesn't take DST into account $lastpreferred += $dstOffset; # DST ended, avoid duplicates $dateSuffix = "_y"; From ceb1397ef084941b92bc2348d85b39020cac49a7 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 23:40:23 +0100 Subject: [PATCH 016/148] Revert "exit with error code upon failure" This reverts commit 8d4484a2d1641789c1e667c03e69773e96cde6ea. --- debian/changelog | 4 ---- tests/common/lib.sh | 1 - 2 files changed, 5 deletions(-) diff --git a/debian/changelog b/debian/changelog index beb6584..ab530b0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,3 @@ -sanoid (1.4.17-SNAPSHOT) unstable; urgency=medium - --- Jim Salter Wed, 9 Aug 2017 12:28:49 -0400 - sanoid (1.4.16) unstable; urgency=medium * merged @hrast01's extended fix to support -o option1=val,option2=val passthrough to SSH. merged @JakobR's diff --git a/tests/common/lib.sh b/tests/common/lib.sh index 78f128b..2c15e9b 100644 --- a/tests/common/lib.sh +++ b/tests/common/lib.sh @@ -103,5 +103,4 @@ function verifySnapshotList { echo "TEST FAILED:" >&2 echo -n -e "${message}" >&2 - exit 1 } From 371f8ff318ad749bcb9374b8acc723ac77da841d Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 23:41:15 +0100 Subject: [PATCH 017/148] exit with error code upon failure --- tests/common/lib.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/common/lib.sh b/tests/common/lib.sh index 2c15e9b..78f128b 100644 --- a/tests/common/lib.sh +++ b/tests/common/lib.sh @@ -103,4 +103,5 @@ function verifySnapshotList { echo "TEST FAILED:" >&2 echo -n -e "${message}" >&2 + exit 1 } From e260f9095f7ac6d6666c22e9b8b312a0ac7ff6e6 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 10 Dec 2017 16:05:08 +0100 Subject: [PATCH 018/148] escape filesystem names as needed to avoid interpreting special characters like whitespace and stop interpreting metacharacters in fs names for some regular expressions, fixes #40 --- syncoid | 191 +++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 135 insertions(+), 56 deletions(-) diff --git a/syncoid b/syncoid index d927cba..3d58863 100755 --- a/syncoid +++ b/syncoid @@ -97,7 +97,7 @@ if (! $args{'recursive'}) { if ($debug) { print "DEBUG: recursive sync of $sourcefs.\n"; } my @datasets = getchilddatasets($sourcehost, $sourcefs, $sourceisroot); foreach my $dataset(@datasets) { - $dataset =~ s/$sourcefs//; + $dataset =~ s/\Q$sourcefs\E//; chomp $dataset; my $childsourcefs = $sourcefs . $dataset; my $childtargetfs = $targetfs . $dataset; @@ -126,11 +126,16 @@ exit 0; sub getchilddatasets { my ($rhost,$fs,$isroot,%snaps) = @_; my $mysudocmd; + my $fsescaped = escapeshellparam($fs); if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } - my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name -t filesystem,volume -Hr $fs |"; + my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name -t filesystem,volume -Hr $fsescaped |"; if ($debug) { print "DEBUG: getting list of child datasets on $fs using $getchildrencmd...\n"; } open FH, $getchildrencmd; my @children = ; @@ -143,6 +148,9 @@ sub syncdataset { my ($sourcehost, $sourcefs, $targethost, $targetfs) = @_; + my $sourcefsescaped = escapeshellparam($sourcefs); + my $targetfsescaped = escapeshellparam($targetfs); + if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; } # make sure target is not currently in receive. @@ -209,8 +217,8 @@ sub syncdataset { # if --no-stream is specified, our full needs to be the newest snapshot, not the oldest. if (defined $args{'no-stream'}) { $oldestsnap = getnewestsnapshot(\%snaps); } - my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefs\@$oldestsnap"; - my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfs"; + my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnap"; + my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); my $disp_pvsize = readablebytes($pvsize); @@ -245,7 +253,7 @@ sub syncdataset { # $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); - $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefs\@$oldestsnap $sourcefs\@$newsyncsnap"; + $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnap $sourcefsescaped\@$newsyncsnap"; $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap","$sourcefs\@$newsyncsnap",$sourceisroot); $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -305,15 +313,15 @@ sub syncdataset { # rollback target to matchingsnap if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; } if ($targethost ne '') { - if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap\n"; } - system ("$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap"); + if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap\n"; } + system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap")); } else { - if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap\n"; } - system ("$targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap"); + if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap\n"; } + system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap"); } - my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefs\@$matchingsnap $sourcefs\@$newsyncsnap"; - my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfs"; + my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnap $sourcefsescaped\@$newsyncsnap"; + my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -590,7 +598,7 @@ sub iszfsbusy { foreach my $process (@processes) { # if ($debug) { print "DEBUG: checking process $process...\n"; } - if ($process =~ /zfs *(receive|recv).*$fs/) { + if ($process =~ /zfs *(receive|recv).*\Q$fs\E/) { # there's already a zfs receive process for our target filesystem - return true if ($debug) { print "DEBUG: process $process matches target $fs!\n"; } return 1; @@ -603,24 +611,40 @@ sub iszfsbusy { sub setzfsvalue { my ($rhost,$fs,$isroot,$property,$value) = @_; - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + + my $fsescaped = escapeshellparam($fs); + + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } + if ($debug) { print "DEBUG: setting $property to $value on $fs...\n"; } my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } - if ($debug) { print "$rhost $mysudocmd $zfscmd set $property=$value $fs\n"; } - system("$rhost $mysudocmd $zfscmd set $property=$value $fs") == 0 - or warn "WARNING: $rhost $mysudocmd $zfscmd set $property=$value $fs died: $?, proceeding anyway.\n"; + if ($debug) { print "$rhost $mysudocmd $zfscmd set $property=$value $fsescaped\n"; } + system("$rhost $mysudocmd $zfscmd set $property=$value $fsescaped") == 0 + or warn "WARNING: $rhost $mysudocmd $zfscmd set $property=$value $fsescaped died: $?, proceeding anyway.\n"; return; } sub getzfsvalue { my ($rhost,$fs,$isroot,$property) = @_; - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + + my $fsescaped = escapeshellparam($fs); + + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } + if ($debug) { print "DEBUG: getting current value of $property on $fs...\n"; } my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } - if ($debug) { print "$rhost $mysudocmd $zfscmd get -H $property $fs\n"; } - open FH, "$rhost $mysudocmd $zfscmd get -H $property $fs |"; + if ($debug) { print "$rhost $mysudocmd $zfscmd get -H $property $fsescaped\n"; } + open FH, "$rhost $mysudocmd $zfscmd get -H $property $fsescaped |"; my $value = ; close FH; my @values = split(/\s/,$value); @@ -706,17 +730,24 @@ sub buildsynccmd { if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; } if ($avail{'compress'}) { $synccmd .= " $args{'compresscmd'} |"; } if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; } - $synccmd .= " $sshcmd $targethost '"; - if ($avail{'targetmbuffer'}) { $synccmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; } - if ($avail{'compress'}) { $synccmd .= " $args{'decompresscmd'} |"; } - $synccmd .= " $recvcmd'"; + $synccmd .= " $sshcmd $targethost "; + + my $remotecmd = ""; + if ($avail{'targetmbuffer'}) { $remotecmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; } + if ($avail{'compress'}) { $remotecmd .= " $args{'decompresscmd'} |"; } + $remotecmd .= " $recvcmd"; + + $synccmd .= escapeshellparam($remotecmd); } elsif ($targethost eq '') { # remote source, local target. #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $mbuffercmd | $pvcmd | $recvcmd"; - $synccmd = "$sshcmd $sourcehost '$sendcmd"; - if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; } - if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } - $synccmd .= "' | "; + + my $remotecmd = $sendcmd; + if ($avail{'compress'}) { $remotecmd .= " | $args{'compresscmd'}"; } + if ($avail{'sourcembuffer'}) { $remotecmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } + + $synccmd = "$sshcmd $sourcehost " . escapeshellparam($remotecmd); + $synccmd .= " | "; if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } @@ -724,25 +755,37 @@ sub buildsynccmd { } else { #remote source, remote target... weird, but whatever, I'm not here to judge you. #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'"; - $synccmd = "$sshcmd $sourcehost '$sendcmd"; - if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; } - if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } - $synccmd .= "' | "; + + my $remotecmd = $sendcmd; + if ($avail{'compress'}) { $remotecmd .= " | $args{'compresscmd'}"; } + if ($avail{'sourcembuffer'}) { $remotecmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } + + $synccmd = "$sshcmd $sourcehost " . escapeshellparam($remotecmd); + $synccmd .= " | "; + if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } if ($avail{'compress'}) { $synccmd .= "$args{'compresscmd'} | "; } if ($avail{'localmbuffer'}) { $synccmd .= "$mbuffercmd $mbufferoptions | "; } - $synccmd .= "$sshcmd $targethost '"; - if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } - if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } - $synccmd .= "$recvcmd'"; + $synccmd .= "$sshcmd $targethost "; + + $remotecmd = ""; + if ($avail{'targetmbuffer'}) { $remotecmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; } + if ($avail{'compress'}) { $remotecmd .= " $args{'decompresscmd'} |"; } + $remotecmd .= " $recvcmd"; + + $synccmd .= escapeshellparam($remotecmd); } return $synccmd; } sub pruneoldsyncsnaps { my ($rhost,$fs,$newsyncsnap,$isroot,@snaps) = @_; + + my $fsescaped = escapeshellparam($fs); + if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + my $hostid = hostname(); my $mysudocmd; @@ -752,7 +795,7 @@ sub pruneoldsyncsnaps { # only prune snaps beginning with syncoid and our own hostname foreach my $snap(@snaps) { - if ($snap =~ /^syncoid_$hostid/) { + if ($snap =~ /^syncoid_\Q$hostid\E/) { # no matter what, we categorically refuse to # prune the new sync snap we created for this run if ($snap ne $newsyncsnap) { @@ -768,12 +811,14 @@ sub pruneoldsyncsnaps { my $prunecmd; foreach my $snap(@prunesnaps) { $counter ++; - $prunecmd .= "$mysudocmd $zfscmd destroy $fs\@$snap; "; + $prunecmd .= "$mysudocmd $zfscmd destroy $fsescaped\@$snap; "; if ($counter > $maxsnapspercmd) { $prunecmd =~ s/\; $//; - if ($rhost ne '') { $prunecmd = '"' . $prunecmd . '"'; } if ($debug) { print "DEBUG: pruning up to $maxsnapspercmd obsolete sync snapshots...\n"; } if ($debug) { print "DEBUG: $rhost $prunecmd\n"; } + if ($rhost ne '') { + $prunecmd = escapeshellparam($prunecmd); + } system("$rhost $prunecmd") == 0 or warn "CRITICAL ERROR: $rhost $prunecmd failed: $?"; $prunecmd = ''; @@ -784,9 +829,11 @@ sub pruneoldsyncsnaps { # the loop, commit 'em now if ($counter) { $prunecmd =~ s/\; $//; - if ($rhost ne '') { $prunecmd = '"' . $prunecmd . '"'; } if ($debug) { print "DEBUG: pruning up to $maxsnapspercmd obsolete sync snapshots...\n"; } if ($debug) { print "DEBUG: $rhost $prunecmd\n"; } + if ($rhost ne '') { + $prunecmd = escapeshellparam($prunecmd); + } system("$rhost $prunecmd") == 0 or warn "WARNING: $rhost $prunecmd failed: $?"; } @@ -824,13 +871,18 @@ sub getmatchingsnapshot { sub newsyncsnap { my ($rhost,$fs,$isroot) = @_; - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + my $fsescaped = escapeshellparam($fs); + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } my $hostid = hostname(); my %date = getdate(); my $snapname = "syncoid\_$hostid\_$date{'stamp'}"; - my $snapcmd = "$rhost $mysudocmd $zfscmd snapshot $fs\@$snapname\n"; + my $snapcmd = "$rhost $mysudocmd $zfscmd snapshot $fsescaped\@$snapname\n"; system($snapcmd) == 0 or die "CRITICAL ERROR: $snapcmd failed: $?"; return $snapname; @@ -838,16 +890,21 @@ sub newsyncsnap { sub targetexists { my ($rhost,$fs,$isroot) = @_; - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + my $fsescaped = escapeshellparam($fs); + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } - my $checktargetcmd = "$rhost $mysudocmd $zfscmd get -H name $fs"; + my $checktargetcmd = "$rhost $mysudocmd $zfscmd get -H name $fsescaped"; if ($debug) { print "DEBUG: checking to see if target filesystem exists using \"$checktargetcmd 2>&1 |\"...\n"; } open FH, "$checktargetcmd 2>&1 |"; my $targetexists = ; close FH; my $exit = $?; - $targetexists = ( $targetexists =~ /^$fs/ && $exit == 0 ); + $targetexists = ( $targetexists =~ /^\Q$fs\E/ && $exit == 0 ); return $targetexists; } @@ -888,11 +945,16 @@ sub dumphash() { sub getsnaps() { my ($type,$rhost,$fs,$isroot,%snaps) = @_; my $mysudocmd; + my $fsescaped = escapeshellparam($fs); if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } - my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t snapshot guid,creation $fs |"; + my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t snapshot guid,creation $fsescaped |"; if ($debug) { print "DEBUG: getting list of snapshots on $fs using $getsnapcmd...\n"; } open FH, $getsnapcmd; my @rawsnaps = ; @@ -903,24 +965,24 @@ sub getsnaps() { foreach my $line (@rawsnaps) { # only import snap guids from the specified filesystem - if ($line =~ /$fs\@.*guid/) { + if ($line =~ /\Q$fs\E\@.*guid/) { chomp $line; my $guid = $line; $guid =~ s/^.*\sguid\s*(\d*).*/$1/; my $snap = $line; - $snap =~ s/^\S*\@(\S*)\s*guid.*$/$1/; + $snap =~ s/^.*\@(\S*)\s*guid.*$/$1/; $snaps{$type}{$snap}{'guid'}=$guid; } } foreach my $line (@rawsnaps) { # only import snap creations from the specified filesystem - if ($line =~ /$fs\@.*creation/) { + if ($line =~ /\Q$fs\E\@.*creation/) { chomp $line; my $creation = $line; $creation =~ s/^.*\screation\s*(\d*).*/$1/; my $snap = $line; - $snap =~ s/^\S*\@(\S*)\s*creation.*$/$1/; + $snap =~ s/^.*\@(\S*)\s*creation.*$/$1/; $snaps{$type}{$snap}{'creation'}=$creation; } } @@ -932,21 +994,30 @@ sub getsnaps() { sub getsendsize { my ($sourcehost,$snap1,$snap2,$isroot) = @_; + my $snap1escaped = escapeshellparam($snap1); + my $snap2escaped = escapeshellparam($snap2); + my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } + my $sourcessh; + if ($sourcehost ne '') { + $sourcessh = "$sshcmd $sourcehost"; + $snap1escaped = escapeshellparam($snap1escaped); + $snap2escaped = escapeshellparam($snap2escaped); + } else { + $sourcessh = ''; + } + my $snaps; if ($snap2) { # if we got a $snap2 argument, we want an incremental send estimate from $snap1 to $snap2. - $snaps = "$args{'streamarg'} $snap1 $snap2"; + $snaps = "$args{'streamarg'} $snap1escaped $snap2escaped"; } else { # if we didn't get a $snap2 arg, we want a full send estimate for $snap1. - $snaps = "$snap1"; + $snaps = "$snap1escaped"; } - my $sourcessh; - if ($sourcehost ne '') { $sourcessh = "$sshcmd $sourcehost"; } else { $sourcessh = ''; } - my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send -nP $snaps"; if ($debug) { print "DEBUG: getting estimated transfer size from source $sourcehost using \"$getsendsizecmd 2>&1 |\"...\n"; } @@ -987,3 +1058,11 @@ sub getdate { $date{'stamp'} = "$date{'year'}-$date{'mon'}-$date{'mday'}:$date{'hour'}:$date{'min'}:$date{'sec'}"; return %date; } + +sub escapeshellparam { + my ($par) = @_; + # "escape" all single quotes + $par =~ s/'/'"'"'/g; + # single-quote entire string + return "'$par'"; + } From adca6230b735b973d34988068c0a3643aff9a760 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 10 Dec 2017 16:10:49 +0100 Subject: [PATCH 019/148] missed one regular expression --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 3d58863..ed5cce9 100755 --- a/syncoid +++ b/syncoid @@ -919,7 +919,7 @@ sub getssh { if ($fs =~ /\@/) { $rhost = $fs; $fs =~ s/^\S*\@\S*://; - $rhost =~ s/:$fs$//; + $rhost =~ s/:\Q$fs\E$//; my $remoteuser = $rhost; $remoteuser =~ s/\@.*$//; if ($remoteuser eq 'root') { $isroot = 1; } else { $isroot = 0; } From 3027831095d51e4e53f28bb82e337d8c11606cac Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 10 Dec 2017 16:32:49 +0100 Subject: [PATCH 020/148] escape all remaining snapshot names which could have special characters --- syncoid | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/syncoid b/syncoid index ed5cce9..797b5c0 100755 --- a/syncoid +++ b/syncoid @@ -216,8 +216,9 @@ sub syncdataset { # if --no-stream is specified, our full needs to be the newest snapshot, not the oldest. if (defined $args{'no-stream'}) { $oldestsnap = getnewestsnapshot(\%snaps); } + my $oldestsnapescaped = escapeshellparam($oldestsnap); - my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnap"; + my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped"; my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); @@ -253,7 +254,7 @@ sub syncdataset { # $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); - $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnap $sourcefsescaped\@$newsyncsnap"; + $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnapescaped $sourcefsescaped\@$newsyncsnap"; $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap","$sourcefs\@$newsyncsnap",$sourceisroot); $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -310,17 +311,18 @@ sub syncdataset { # barf some text but don't touch the filesystem if (!$quiet) { print "INFO: no snapshots on source newer than $newsyncsnap on target. Nothing to do, not syncing.\n"; } } else { + my $matchingsnapescaped = escapeshellparam($matchingsnap); # rollback target to matchingsnap if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; } if ($targethost ne '') { - if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap\n"; } - system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap")); + if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped\n"; } + system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped")); } else { - if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap\n"; } - system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap"); + if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped\n"; } + system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped"); } - my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnap $sourcefsescaped\@$newsyncsnap"; + my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnap"; my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); From 85cc99c9e6b4c507a1559903ecc7718685b0a77c Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 10 Dec 2017 16:56:56 +0100 Subject: [PATCH 021/148] accidentally removed helptext --- syncoid | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/syncoid b/syncoid index 86810d4..8825dcb 100755 --- a/syncoid +++ b/syncoid @@ -1005,3 +1005,41 @@ sub escapeshellparam { # single-quote entire string return "'$par'"; } + +__END__ + +=head1 NAME + +syncoid - ZFS snapshot replication tool + +=head1 SYNOPSIS + + syncoid [options]... SOURCE TARGET + or syncoid [options]... SOURCE [USER@]HOST:TARGET + or syncoid [options]... [USER@]HOST:SOURCE [TARGET] + or syncoid [options]... [USER@]HOST:SOURCE [USER@]HOST:TARGET + + SOURCE Source ZFS dataset. Can be either local or remote + TARGET Target ZFS dataset. Can be either local or remote + +Options: + + --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none + --recursive|r Also transfers child datasets + --source-bwlimit= Bandwidth limit on the source transfer + --target-bwlimit= Bandwidth limit on the target transfer + --no-stream Replicates using newest snapshot instead of intermediates + --no-sync-snap Does not create new snapshot, only transfers existing + + --sshkey=FILE Specifies a ssh public key to use to connect + --sshport=PORT Connects to remote on a particular port + --sshcipher|c=CIPHER Passes CIPHER to ssh to use a particular cipher set + --sshoption|o=OPTION Passes OPTION to ssh for remote usage. Can be specified multiple times + + --help Prints this helptext + --verbose Prints the version number + --debug Prints out a lot of additional information during a syncoid run + --monitor-version Currently does nothing + --quiet Suppresses non-error output + --dumpsnaps Dumps a list of snapshots during the run + --no-command-checks Do not check command existence before attempting transfer. Not recommended From 2a3d91e4465ea35ce68548915aeaf4eb1ec75745 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 13 Dec 2017 00:44:48 +0100 Subject: [PATCH 022/148] fixed weekly snapshot interval --- sanoid | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sanoid b/sanoid index 05c4def..792497a 100755 --- a/sanoid +++ b/sanoid @@ -313,9 +313,9 @@ sub take_snapshots { if ($lastpreferred > time()) { $lastpreferred -= 60*60*24; } # preferred time is later today - so look at yesterday's } elsif ($type eq 'weekly') { # calculate offset in seconds for the desired weekday - my $offset; + my $offset = 0; if ($config{$section}{'weekly_wday'} < $datestamp{'wday'}) { - $offset += 6; + $offset += 7; } $offset += $config{$section}{'weekly_wday'} - $datestamp{'wday'}; $offset *= 60*60*24; # full day @@ -327,7 +327,7 @@ sub take_snapshots { push @preferredtime,($datestamp{'mon'}-1); # january is month 0 push @preferredtime,$datestamp{'year'}; $lastpreferred = timelocal(@preferredtime); - $lastpreferred -= $offset; + $lastpreferred += $offset; if ($lastpreferred > time()) { $lastpreferred -= 60*60*24*7; } # preferred time is later today - so look at yesterday's } elsif ($type eq 'monthly') { push @preferredtime,0; # try to hit 0 seconds From 742db32e686520dc22ea11a8f831ed15618fdf9c Mon Sep 17 00:00:00 2001 From: Martin van Wingerden Date: Fri, 29 Dec 2017 20:02:01 +0100 Subject: [PATCH 023/148] Made two INFO prints mutable Added a quiet check for two non-controllable INFO prints Fixes: #182 Signed-off-by: Martin van Wingerden --- sanoid | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index d6e58ce..b995924 100755 --- a/sanoid +++ b/sanoid @@ -235,7 +235,7 @@ sub prune_snapshots { foreach my $snap( @prunesnaps ){ if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; } if (iszfsbusy($path)) { - print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; + if ($args{'verbose'}) { print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; } } else { if (! $args{'readonly'}) { system($zfs, "destroy",$snap) == 0 or warn "could not remove $snap : $?"; } } @@ -244,7 +244,7 @@ sub prune_snapshots { $forcecacheupdate = 1; %snaps = getsnaps(%config,$cacheTTL,$forcecacheupdate); } else { - print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n"; + if ($args{'verbose'}) { print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n"; } } } } From 3d9b3b04badbce3074c279eb34aab97977df47b1 Mon Sep 17 00:00:00 2001 From: Gionatan Danti Date: Thu, 4 Jan 2018 12:20:37 +0100 Subject: [PATCH 024/148] Added "force-prune" option to skip busy dataset check --- sanoid | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index d6e58ce..029d846 100755 --- a/sanoid +++ b/sanoid @@ -18,7 +18,7 @@ use Time::Local; # to parse dates in reverse my %args = ("configdir" => "/etc/sanoid"); GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet", "monitor-health", "force-update", "configdir=s", - "monitor-snapshots", "take-snapshots", "prune-snapshots" + "monitor-snapshots", "take-snapshots", "prune-snapshots", "force-prune" ) or pod2usage(2); # If only config directory (or nothing) has been specified, default to --cron --verbose @@ -234,7 +234,7 @@ sub prune_snapshots { writelock('sanoid_pruning'); foreach my $snap( @prunesnaps ){ if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; } - if (iszfsbusy($path)) { + if (iszfsbusy($path) && !$args{'force-prune'}) { print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; } else { if (! $args{'readonly'}) { system($zfs, "destroy",$snap) == 0 or warn "could not remove $snap : $?"; } @@ -1082,6 +1082,7 @@ Options: --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format --take-snapshots Creates snapshots as specified in sanoid.conf --prune-snapshots Purges expired snapshots as specified in sanoid.conf + --force-prune Purges expired snapshots even if a send/recv is in progress --help Prints this helptext --version Prints the version number From 8a2a673c58ddbc1b222ec65a75c4db77b9e8b4e4 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Sat, 6 Jan 2018 11:01:44 +0000 Subject: [PATCH 025/148] fixed loud 'NEWEST SNAPSHOT' message --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 7337f5b..d039d15 100755 --- a/syncoid +++ b/syncoid @@ -597,7 +597,7 @@ sub getnewestsnapshot { my $snaps = shift; foreach my $snap ( sort { $snaps{'source'}{$b}{'creation'}<=>$snaps{'source'}{$a}{'creation'} } keys %{ $snaps{'source'} }) { # return on first snap found - it's the newest - print "NEWEST SNAPSHOT: $snap\n"; + if (!$quiet) { print "NEWEST SNAPSHOT: $snap\n"; } return $snap; } # must not have had any snapshots on source - looks like we'd better create one! From e902df1ef2347aa08aeb20478a2b988695dd5d20 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Sat, 6 Jan 2018 11:03:02 +0000 Subject: [PATCH 026/148] also made warnings quiet --- syncoid | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/syncoid b/syncoid index d039d15..aec99cd 100755 --- a/syncoid +++ b/syncoid @@ -452,13 +452,13 @@ sub checkcommands { if ($avail{'sourcecompress'} eq '') { if ($compressargs{'rawcmd'} ne '') { - print "WARN: $compressargs{'rawcmd'} not available on source $s- sync will continue without compression.\n"; + if (!$quiet) { print "WARN: $compressargs{'rawcmd'} not available on source $s- sync will continue without compression.\n"; } } $avail{'compress'} = 0; } if ($avail{'targetcompress'} eq '') { if ($compressargs{'rawcmd'} ne '') { - print "WARN: $compressargs{'rawcmd'} not available on target $t - sync will continue without compression.\n"; + if (!$quiet) { print "WARN: $compressargs{'rawcmd'} not available on target $t - sync will continue without compression.\n"; } } $avail{'compress'} = 0; } @@ -471,7 +471,7 @@ sub checkcommands { # corner case - if source AND target are BOTH remote, we have to check for local compress too if ($sourcehost ne '' && $targethost ne '' && $avail{'localcompress'} eq '') { if ($compressargs{'rawcmd'} ne '') { - print "WARN: $compressargs{'rawcmd'} not available on local machine - sync will continue without compression.\n"; + if (!$quiet) { print "WARN: $compressargs{'rawcmd'} not available on local machine - sync will continue without compression.\n"; } } $avail{'compress'} = 0; } @@ -479,7 +479,7 @@ sub checkcommands { if ($debug) { print "DEBUG: checking availability of $mbuffercmd on source...\n"; } $avail{'sourcembuffer'} = `$sourcessh $lscmd $mbuffercmd 2>/dev/null`; if ($avail{'sourcembuffer'} eq '') { - print "WARN: $mbuffercmd not available on source $s - sync will continue without source buffering.\n"; + if (!$quiet) { print "WARN: $mbuffercmd not available on source $s - sync will continue without source buffering.\n"; } $avail{'sourcembuffer'} = 0; } else { $avail{'sourcembuffer'} = 1; @@ -488,7 +488,7 @@ sub checkcommands { if ($debug) { print "DEBUG: checking availability of $mbuffercmd on target...\n"; } $avail{'targetmbuffer'} = `$targetssh $lscmd $mbuffercmd 2>/dev/null`; if ($avail{'targetmbuffer'} eq '') { - print "WARN: $mbuffercmd not available on target $t - sync will continue without target buffering.\n"; + if (!$quiet) { print "WARN: $mbuffercmd not available on target $t - sync will continue without target buffering.\n"; } $avail{'targetmbuffer'} = 0; } else { $avail{'targetmbuffer'} = 1; @@ -500,14 +500,14 @@ sub checkcommands { $avail{'localmbuffer'} = `$lscmd $mbuffercmd 2>/dev/null`; if ($avail{'localmbuffer'} eq '') { $avail{'localmbuffer'} = 0; - print "WARN: $mbuffercmd not available on local machine - sync will continue without local buffering.\n"; + if (!$quiet) { print "WARN: $mbuffercmd not available on local machine - sync will continue without local buffering.\n"; } } } if ($debug) { print "DEBUG: checking availability of $pvcmd on local machine...\n"; } $avail{'localpv'} = `$lscmd $pvcmd 2>/dev/null`; if ($avail{'localpv'} eq '') { - print "WARN: $pvcmd not available on local machine - sync will continue without progress bar.\n"; + if (!$quiet) { print "WARN: $pvcmd not available on local machine - sync will continue without progress bar.\n"; } $avail{'localpv'} = 0; } else { $avail{'localpv'} = 1; From d5f4b5abba08a597108538cadbd09961afa80363 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 13 Feb 2018 18:47:55 +0100 Subject: [PATCH 027/148] support resumable zfs send/receive --- syncoid | 127 +++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 98 insertions(+), 29 deletions(-) diff --git a/syncoid b/syncoid index 8825dcb..8a31f37 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "resume") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -161,35 +161,55 @@ sub syncdataset { # does the target filesystem exist yet? my $targetexists = targetexists($targethost,$targetfs,$targetisroot); - # build hashes of the snaps on the source and target filesystems. + my $receiveextraargs = ""; + my $receivetoken; + if (defined $args{'resume'}) { + # save state of interrupted receive stream + $receiveextraargs = "-s"; - %snaps = getsnaps('source',$sourcehost,$sourcefs,$sourceisroot); + if ($targetexists) { + # check remote dataset for receive resume token (interrupted receive) + $receivetoken = getreceivetoken($targethost,$targetfs,$targetisroot); - if ($targetexists) { - my %targetsnaps = getsnaps('target',$targethost,$targetfs,$targetisroot); - my %sourcesnaps = %snaps; - %snaps = (%sourcesnaps, %targetsnaps); - } - - if (defined $args{'dumpsnaps'}) { - print "merged snapshot list of $targetfs: \n"; - dumphash(\%snaps); - print "\n\n\n"; - } - - # create a new syncoid snapshot on the source filesystem. - my $newsyncsnap; - if (!defined $args{'no-sync-snap'}) { - $newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot); - } else { - # we don't want sync snapshots created, so use the newest snapshot we can find. - $newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot); - if ($newsyncsnap eq 0) { - warn "CRITICAL: no snapshots exist on source $sourcefs, and you asked for --no-sync-snap.\n"; - return 0; + if ($debug && defined($receivetoken)) { + print "DEBUG: got receive resume token: $receivetoken: \n"; + } } } + my $newsyncsnap; + + # skip snapshot checking/creation in case of resumed receive + if (!defined($receivetoken)) { + # build hashes of the snaps on the source and target filesystems. + + %snaps = getsnaps('source',$sourcehost,$sourcefs,$sourceisroot); + + if ($targetexists) { + my %targetsnaps = getsnaps('target',$targethost,$targetfs,$targetisroot); + my %sourcesnaps = %snaps; + %snaps = (%sourcesnaps, %targetsnaps); + } + + if (defined $args{'dumpsnaps'}) { + print "merged snapshot list of $targetfs: \n"; + dumphash(\%snaps); + print "\n\n\n"; + } + + if (!defined $args{'no-sync-snap'}) { + # create a new syncoid snapshot on the source filesystem. + $newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot); + } else { + # we don't want sync snapshots created, so use the newest snapshot we can find. + $newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot); + if ($newsyncsnap eq 0) { + warn "CRITICAL: no snapshots exist on source $sourcefs, and you asked for --no-sync-snap.\n"; + return 0; + } + } + } + # there is currently (2014-09-01) a bug in ZFS on Linux # that causes readonly to always show on if it's EVER # been turned on... even when it's off... unless and @@ -222,7 +242,7 @@ sub syncdataset { my $oldestsnapescaped = escapeshellparam($oldestsnap); my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); my $disp_pvsize = readablebytes($pvsize); @@ -286,6 +306,27 @@ sub syncdataset { # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly); } } else { + # resume interrupted receive if there is a valid resume $token + # and because this will ony resume the receive to the next + # snapshot, do a normal sync after that + if (defined($receivetoken)) { + my $sendcmd = "$sourcesudocmd $zfscmd send -t $receivetoken"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $pvsize = getsendsize($sourcehost,"","",$sourceisroot,$receivetoken); + my $disp_pvsize = readablebytes($pvsize); + if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } + my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + + if (!$quiet) { print "Resuming interrupted zfs send/receive from $sourcefs to $targetfs (~ $disp_pvsize remaining):\n"; } + if ($debug) { print "DEBUG: $synccmd\n"; } + system("$synccmd") == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; + + # a resumed transfer will only be done to the next snapshot, + # so do an normal sync cycle + return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs); + } + # find most recent matching snapshot and do an -I # to the new snapshot @@ -326,7 +367,7 @@ sub syncdataset { } my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnap"; - my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -931,7 +972,7 @@ sub getsnaps() { sub getsendsize { - my ($sourcehost,$snap1,$snap2,$isroot) = @_; + my ($sourcehost,$snap1,$snap2,$isroot,$receivetoken) = @_; my $snap1escaped = escapeshellparam($snap1); my $snap2escaped = escapeshellparam($snap2); @@ -957,6 +998,12 @@ sub getsendsize { $snaps = "$snap1escaped"; } + # in case of a resumed receive, get the remaining + # size based on the resume token + if (defined($receivetoken)) { + $snaps = "-t $receivetoken"; + } + my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send -nP $snaps"; if ($debug) { print "DEBUG: getting estimated transfer size from source $sourcehost using \"$getsendsizecmd 2>&1 |\"...\n"; } @@ -969,7 +1016,13 @@ sub getsendsize { # size of proposed xfer in bytes, but we need to remove # human-readable crap from it my $sendsize = pop(@rawsize); - $sendsize =~ s/^size\s*//; + # the output format is different in case of + # a resumed receive + if (defined($receivetoken)) { + $sendsize =~ s/.*\s([0-9]+)$/$1/; + } else { + $sendsize =~ s/^size\s*//; + } chomp $sendsize; # to avoid confusion with a zero size pv, give sendsize @@ -1006,6 +1059,21 @@ sub escapeshellparam { return "'$par'"; } +sub getreceivetoken() { + my ($rhost,$fs,$isroot) = @_; + my $token = getzfsvalue($rhost,$fs,$isroot,"receive_resume_token"); + + if ($token ne '-' && $token ne '') { + return $token; + } + + if ($debug) { + print "DEBUG: no receive token found \n"; + } + + return +} + __END__ =head1 NAME @@ -1043,3 +1111,4 @@ Options: --quiet Suppresses non-error output --dumpsnaps Dumps a list of snapshots during the run --no-command-checks Do not check command existence before attempting transfer. Not recommended + --resume Save the state of unfinished receive streams and resume interrupted ones if available From 1f64c9c35aac5d45c433833af56bba4126a0bcbd Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 20 Feb 2018 18:16:35 +0100 Subject: [PATCH 028/148] let monitor-health check the capacity too --- sanoid | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sanoid b/sanoid index d6e58ce..6b1257e 100755 --- a/sanoid +++ b/sanoid @@ -798,6 +798,17 @@ sub check_zpool() { ## determine health of zpool and subsequent error status if ($health eq "ONLINE" ) { $state = "OK"; + + # check capacity + my $capn = $cap; + $capn =~ s/\D//g; + + if ($capn >= 80) { + $state = "WARNING"; + } + if ($capn >= 95) { + $state = "CRITICAL"; + } } else { if ($health eq "DEGRADED") { $state = "WARNING"; From 06d029db684f49a577a4d912f82c4ea0462e137e Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 22 Feb 2018 16:53:30 +0100 Subject: [PATCH 029/148] remove destroyed snapshots from cache file instead of regenerating the whole thing (which can take very long on systems with many snapshots and/or datasets) --- sanoid | 47 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/sanoid b/sanoid index d6e58ce..238f955 100755 --- a/sanoid +++ b/sanoid @@ -39,6 +39,7 @@ my %config = init($conf_file,$default_conf_file); # if we call getsnaps(%config,1) it will forcibly update the cache, TTL or no TTL my $forcecacheupdate = 0; +my $cache = '/var/cache/sanoidsnapshots.txt'; my $cacheTTL = 900; # 15 minutes my %snaps = getsnaps( \%config, $cacheTTL, $forcecacheupdate ); @@ -232,17 +233,23 @@ sub prune_snapshots { # print "found some snaps to prune!\n" if (checklock('sanoid_pruning')) { writelock('sanoid_pruning'); + my @pruned; foreach my $snap( @prunesnaps ){ if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; } if (iszfsbusy($path)) { print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; } else { - if (! $args{'readonly'}) { system($zfs, "destroy",$snap) == 0 or warn "could not remove $snap : $?"; } + if (! $args{'readonly'}) { + if (system($zfs, "destroy", $snap) == 0) { + push(@pruned, $snap); + } else { + warn "could not remove $snap : $?"; + } + } } } removelock('sanoid_pruning'); - $forcecacheupdate = 1; - %snaps = getsnaps(%config,$cacheTTL,$forcecacheupdate); + removecachedsnapshots(@pruned); } else { print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n"; } @@ -484,7 +491,6 @@ sub getsnaps { my ($config, $cacheTTL, $forcecacheupdate) = @_; - my $cache = '/var/cache/sanoidsnapshots.txt'; my @rawsnaps; my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, $mtime, $ctime, $blksize, $blocks) = stat($cache); @@ -1056,6 +1062,39 @@ sub getchilddatasets { return @children; } +#######################################################################################################################3 +#######################################################################################################################3 +#######################################################################################################################3 + +sub removecachedsnapshots { + my @prunedlist = shift; + my %pruned = map { $_ => 1 } @prunedlist; + + if (checklock('sanoid_cacheupdate')) { + writelock('sanoid_cacheupdate'); + + if ($args{'verbose'}) { + print "INFO: removing destroyed snapshots from cache.\n"; + } + open FH, "< $cache"; + my @rawsnaps = ; + close FH; + + open FH, "> $cache" or die 'Could not write to $cache!\n'; + foreach my $snapline ( @rawsnaps ) { + my @columns = split("\t", $snapline); + my $snap = $columns[0]; + print FH $snapline unless ( exists($pruned{$snap}) ); + } + close FH; + + removelock('sanoid_cacheupdate'); + %snaps = getsnaps(\%config,$cacheTTL,$forcecacheupdate); + } else { + if ($args{'verbose'}) { print "WARN: skipping cache update (snapshot removal) - valid cache update lock held by another sanoid process.\n"; } + } +} + __END__ =head1 NAME From d0f1445784c4e54470b72588d9d8954d0e2bf258 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 22 Feb 2018 17:29:58 +0100 Subject: [PATCH 030/148] defer cache updates after snapshot pruning and do them after all pruning is done (waiting for the cache update lock if necessary) --- sanoid | 74 ++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/sanoid b/sanoid index 238f955..e73b4a4 100755 --- a/sanoid +++ b/sanoid @@ -42,6 +42,7 @@ my $forcecacheupdate = 0; my $cache = '/var/cache/sanoidsnapshots.txt'; my $cacheTTL = 900; # 15 minutes my %snaps = getsnaps( \%config, $cacheTTL, $forcecacheupdate ); +my %pruned; my %snapsbytype = getsnapsbytype( \%config, \%snaps ); @@ -233,7 +234,6 @@ sub prune_snapshots { # print "found some snaps to prune!\n" if (checklock('sanoid_pruning')) { writelock('sanoid_pruning'); - my @pruned; foreach my $snap( @prunesnaps ){ if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; } if (iszfsbusy($path)) { @@ -241,7 +241,7 @@ sub prune_snapshots { } else { if (! $args{'readonly'}) { if (system($zfs, "destroy", $snap) == 0) { - push(@pruned, $snap); + $pruned{$snap} = 1; } else { warn "could not remove $snap : $?"; } @@ -249,7 +249,7 @@ sub prune_snapshots { } } removelock('sanoid_pruning'); - removecachedsnapshots(@pruned); + removecachedsnapshots(0); } else { print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n"; } @@ -258,7 +258,9 @@ sub prune_snapshots { } } - + # if there were any deferred cache updates, + # do them now and wait if necessary + removecachedsnapshots(1); } # end prune_snapshots @@ -1067,32 +1069,48 @@ sub getchilddatasets { #######################################################################################################################3 sub removecachedsnapshots { - my @prunedlist = shift; - my %pruned = map { $_ => 1 } @prunedlist; + my $wait = shift; - if (checklock('sanoid_cacheupdate')) { - writelock('sanoid_cacheupdate'); - - if ($args{'verbose'}) { - print "INFO: removing destroyed snapshots from cache.\n"; - } - open FH, "< $cache"; - my @rawsnaps = ; - close FH; - - open FH, "> $cache" or die 'Could not write to $cache!\n'; - foreach my $snapline ( @rawsnaps ) { - my @columns = split("\t", $snapline); - my $snap = $columns[0]; - print FH $snapline unless ( exists($pruned{$snap}) ); - } - close FH; - - removelock('sanoid_cacheupdate'); - %snaps = getsnaps(\%config,$cacheTTL,$forcecacheupdate); - } else { - if ($args{'verbose'}) { print "WARN: skipping cache update (snapshot removal) - valid cache update lock held by another sanoid process.\n"; } + if (not %pruned) { + return; } + + my $unlocked = checklock('sanoid_cacheupdate'); + + if ($wait != 1 && not $unlocked) { + if ($args{'verbose'}) { print "INFO: deferring cache update (snapshot removal) - valid cache update lock held by another sanoid process.\n"; } + return; + } + + # wait until we can get a lock to do our cache changes + while (not $unlocked) { + if ($args{'verbose'}) { print "INFO: waiting for cache update lock held by another sanoid process.\n"; } + sleep(10); + $unlocked = checklock('sanoid_cacheupdate'); + } + + writelock('sanoid_cacheupdate'); + + if ($args{'verbose'}) { + print "INFO: removing destroyed snapshots from cache.\n"; + } + open FH, "< $cache"; + my @rawsnaps = ; + close FH; + + open FH, "> $cache" or die 'Could not write to $cache!\n'; + foreach my $snapline ( @rawsnaps ) { + my @columns = split("\t", $snapline); + my $snap = $columns[0]; + print FH $snapline unless ( exists($pruned{$snap}) ); + } + close FH; + + removelock('sanoid_cacheupdate'); + %snaps = getsnaps(\%config,$cacheTTL,$forcecacheupdate); + + # clear hash + undef %pruned; } __END__ From 6c695f1a86274d8c7763833076b3b17e3a415b1b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 26 Feb 2018 18:16:06 +0100 Subject: [PATCH 031/148] allow monitor-health to optionally check zpool capacity too by providing limits along the flag --- README.md | 2 +- sanoid | 89 ++++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 66 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 9c4e6ba..6239b79 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da + --monitor-health - This option is designed to be run by a Nagios monitoring system. It reports on the health of the zpool your filesystems are on. It only monitors filesystems that are configured in the sanoid.conf file. + This option is designed to be run by a Nagios monitoring system. It reports on the health of the zpool your filesystems are on. It only monitors filesystems that are configured in the sanoid.conf file. Optionally it can check capacity limits too by appending them like '=80,95" (>= 80% warning, >=95% critical). + --force-update diff --git a/sanoid b/sanoid index 6b1257e..9dd6198 100755 --- a/sanoid +++ b/sanoid @@ -17,7 +17,7 @@ use Time::Local; # to parse dates in reverse my %args = ("configdir" => "/etc/sanoid"); GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet", - "monitor-health", "force-update", "configdir=s", + "monitor-health:s", "force-update", "configdir=s", "monitor-snapshots", "take-snapshots", "prune-snapshots" ) or pod2usage(2); @@ -51,7 +51,7 @@ my @params = ( \%config, \%snaps, \%snapsbytype, \%snapsbypath ); if ($args{'debug'}) { $args{'verbose'}=1; blabber (@params); } if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); } -if ($args{'monitor-health'}) { monitor_health(@params); } +if (defined($args{'monitor-health'})) { monitor_health(@params); } if ($args{'force-update'}) { my $snaps = getsnaps( \%config, $cacheTTL, 1 ); } if ($args{'cron'}) { @@ -76,13 +76,32 @@ sub monitor_health { my @messages; my $errlevel=0; + my %capacitylimits; + + # if provided, parse capacity limits + if ($args{'monitor-health'} ne "") { + my @values = split(',', $args{'monitor-health'}); + + if (!check_capacity_limit($values[0])) { + die "ERROR: invalid zpool capacity warning limit!\n"; + } + $capacitylimits{"warn"} = $values[0]; + + if (scalar @values > 1) { + if (!check_capacity_limit($values[1])) { + die "ERROR: invalid zpool capacity critical limit!\n"; + } + $capacitylimits{"crit"} = $values[1]; + } + } + foreach my $path (keys %{ $snapsbypath}) { my @pool = split ('/',$path); $pools{$pool[0]}=1; } foreach my $pool (keys %pools) { - my ($exitcode, $msg) = check_zpool($pool,2); + my ($exitcode, $msg) = check_zpool($pool,2,\%capacitylimits); if ($exitcode > $errlevel) { $errlevel = $exitcode; } chomp $msg; push (@messages, $msg); @@ -748,6 +767,8 @@ sub check_zpool() { my $pool=shift; my $verbose=shift; + my $capacitylimitsref=shift; + my %capacitylimits=%$capacitylimitsref; my $size=""; my $used=""; @@ -799,15 +820,21 @@ sub check_zpool() { if ($health eq "ONLINE" ) { $state = "OK"; - # check capacity - my $capn = $cap; - $capn =~ s/\D//g; + if (%capacitylimits) { + # check capacity + my $capn = $cap; + $capn =~ s/\D//g; - if ($capn >= 80) { - $state = "WARNING"; - } - if ($capn >= 95) { - $state = "CRITICAL"; + if ($capacitylimits{"warn"}) { + if ($capn >= $capacitylimits{"warn"}) { + $state = "WARNING"; + } + } + if ($capacitylimits{"crit"}) { + if ($capn >= $capacitylimits{"crit"}) { + $state = "CRITICAL"; + } + } } } else { if ($health eq "DEGRADED") { @@ -911,6 +938,20 @@ sub check_zpool() { return ($ERRORS{$state},$msg); } # end check_zpool() +sub check_capacity_limit() { + my $value = shift; + + if ($value !~ /^\d+\z/) { + return undef; + } + + if ($value < 1 || $value > 100) { + return undef; + } + + return 1 +} + ###################################################################################################### ###################################################################################################### ###################################################################################################### @@ -1081,19 +1122,19 @@ Assumes --cron --verbose if no other arguments (other than configdir) are specif Options: - --configdir=DIR Specify a directory to find config file sanoid.conf + --configdir=DIR Specify a directory to find config file sanoid.conf - --cron Creates snapshots and purges expired snapshots - --verbose Prints out additional information during a sanoid run - --readonly Simulates creation/deletion of snapshots - --quiet Suppresses non-error output - --force-update Clears out sanoid's zfs snapshot cache + --cron Creates snapshots and purges expired snapshots + --verbose Prints out additional information during a sanoid run + --readonly Simulates creation/deletion of snapshots + --quiet Suppresses non-error output + --force-update Clears out sanoid's zfs snapshot cache - --monitor-health Reports on zpool "health", in a Nagios compatible format - --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format - --take-snapshots Creates snapshots as specified in sanoid.conf - --prune-snapshots Purges expired snapshots as specified in sanoid.conf + --monitor-health[=wlimit[,climit]] Reports on zpool "health", in a Nagios compatible format + --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format + --take-snapshots Creates snapshots as specified in sanoid.conf + --prune-snapshots Purges expired snapshots as specified in sanoid.conf - --help Prints this helptext - --version Prints the version number - --debug Prints out a lot of additional information during a sanoid run + --help Prints this helptext + --version Prints the version number + --debug Prints out a lot of additional information during a sanoid run From 6f29bed441aa0c4db015271ceb3b24931286cc3b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 27 Feb 2018 17:53:04 +0100 Subject: [PATCH 032/148] Revert "allow monitor-health to optionally check zpool capacity too by providing limits along the flag" This reverts commit 6c695f1a86274d8c7763833076b3b17e3a415b1b. --- README.md | 2 +- sanoid | 89 +++++++++++++++---------------------------------------- 2 files changed, 25 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 6239b79..9c4e6ba 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da + --monitor-health - This option is designed to be run by a Nagios monitoring system. It reports on the health of the zpool your filesystems are on. It only monitors filesystems that are configured in the sanoid.conf file. Optionally it can check capacity limits too by appending them like '=80,95" (>= 80% warning, >=95% critical). + This option is designed to be run by a Nagios monitoring system. It reports on the health of the zpool your filesystems are on. It only monitors filesystems that are configured in the sanoid.conf file. + --force-update diff --git a/sanoid b/sanoid index 9dd6198..6b1257e 100755 --- a/sanoid +++ b/sanoid @@ -17,7 +17,7 @@ use Time::Local; # to parse dates in reverse my %args = ("configdir" => "/etc/sanoid"); GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet", - "monitor-health:s", "force-update", "configdir=s", + "monitor-health", "force-update", "configdir=s", "monitor-snapshots", "take-snapshots", "prune-snapshots" ) or pod2usage(2); @@ -51,7 +51,7 @@ my @params = ( \%config, \%snaps, \%snapsbytype, \%snapsbypath ); if ($args{'debug'}) { $args{'verbose'}=1; blabber (@params); } if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); } -if (defined($args{'monitor-health'})) { monitor_health(@params); } +if ($args{'monitor-health'}) { monitor_health(@params); } if ($args{'force-update'}) { my $snaps = getsnaps( \%config, $cacheTTL, 1 ); } if ($args{'cron'}) { @@ -76,32 +76,13 @@ sub monitor_health { my @messages; my $errlevel=0; - my %capacitylimits; - - # if provided, parse capacity limits - if ($args{'monitor-health'} ne "") { - my @values = split(',', $args{'monitor-health'}); - - if (!check_capacity_limit($values[0])) { - die "ERROR: invalid zpool capacity warning limit!\n"; - } - $capacitylimits{"warn"} = $values[0]; - - if (scalar @values > 1) { - if (!check_capacity_limit($values[1])) { - die "ERROR: invalid zpool capacity critical limit!\n"; - } - $capacitylimits{"crit"} = $values[1]; - } - } - foreach my $path (keys %{ $snapsbypath}) { my @pool = split ('/',$path); $pools{$pool[0]}=1; } foreach my $pool (keys %pools) { - my ($exitcode, $msg) = check_zpool($pool,2,\%capacitylimits); + my ($exitcode, $msg) = check_zpool($pool,2); if ($exitcode > $errlevel) { $errlevel = $exitcode; } chomp $msg; push (@messages, $msg); @@ -767,8 +748,6 @@ sub check_zpool() { my $pool=shift; my $verbose=shift; - my $capacitylimitsref=shift; - my %capacitylimits=%$capacitylimitsref; my $size=""; my $used=""; @@ -820,21 +799,15 @@ sub check_zpool() { if ($health eq "ONLINE" ) { $state = "OK"; - if (%capacitylimits) { - # check capacity - my $capn = $cap; - $capn =~ s/\D//g; + # check capacity + my $capn = $cap; + $capn =~ s/\D//g; - if ($capacitylimits{"warn"}) { - if ($capn >= $capacitylimits{"warn"}) { - $state = "WARNING"; - } - } - if ($capacitylimits{"crit"}) { - if ($capn >= $capacitylimits{"crit"}) { - $state = "CRITICAL"; - } - } + if ($capn >= 80) { + $state = "WARNING"; + } + if ($capn >= 95) { + $state = "CRITICAL"; } } else { if ($health eq "DEGRADED") { @@ -938,20 +911,6 @@ sub check_zpool() { return ($ERRORS{$state},$msg); } # end check_zpool() -sub check_capacity_limit() { - my $value = shift; - - if ($value !~ /^\d+\z/) { - return undef; - } - - if ($value < 1 || $value > 100) { - return undef; - } - - return 1 -} - ###################################################################################################### ###################################################################################################### ###################################################################################################### @@ -1122,19 +1081,19 @@ Assumes --cron --verbose if no other arguments (other than configdir) are specif Options: - --configdir=DIR Specify a directory to find config file sanoid.conf + --configdir=DIR Specify a directory to find config file sanoid.conf - --cron Creates snapshots and purges expired snapshots - --verbose Prints out additional information during a sanoid run - --readonly Simulates creation/deletion of snapshots - --quiet Suppresses non-error output - --force-update Clears out sanoid's zfs snapshot cache + --cron Creates snapshots and purges expired snapshots + --verbose Prints out additional information during a sanoid run + --readonly Simulates creation/deletion of snapshots + --quiet Suppresses non-error output + --force-update Clears out sanoid's zfs snapshot cache - --monitor-health[=wlimit[,climit]] Reports on zpool "health", in a Nagios compatible format - --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format - --take-snapshots Creates snapshots as specified in sanoid.conf - --prune-snapshots Purges expired snapshots as specified in sanoid.conf + --monitor-health Reports on zpool "health", in a Nagios compatible format + --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format + --take-snapshots Creates snapshots as specified in sanoid.conf + --prune-snapshots Purges expired snapshots as specified in sanoid.conf - --help Prints this helptext - --version Prints the version number - --debug Prints out a lot of additional information during a sanoid run + --help Prints this helptext + --version Prints the version number + --debug Prints out a lot of additional information during a sanoid run From 01398789f60f31f67002666daacce715c6e626a5 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 27 Feb 2018 17:53:32 +0100 Subject: [PATCH 033/148] Revert "let monitor-health check the capacity too" This reverts commit 1f64c9c35aac5d45c433833af56bba4126a0bcbd. --- sanoid | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/sanoid b/sanoid index 6b1257e..d6e58ce 100755 --- a/sanoid +++ b/sanoid @@ -798,17 +798,6 @@ sub check_zpool() { ## determine health of zpool and subsequent error status if ($health eq "ONLINE" ) { $state = "OK"; - - # check capacity - my $capn = $cap; - $capn =~ s/\D//g; - - if ($capn >= 80) { - $state = "WARNING"; - } - if ($capn >= 95) { - $state = "CRITICAL"; - } } else { if ($health eq "DEGRADED") { $state = "WARNING"; From f961a9f447344b0b5558b3c8382aec3c2b74632e Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 27 Feb 2018 17:58:51 +0100 Subject: [PATCH 034/148] implemented monitor-capacity flag for checking zpool capacity limits in the nagios monitoring format --- README.md | 5 ++ sanoid | 146 ++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 137 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 9c4e6ba..1b3b37f 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,11 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da This option is designed to be run by a Nagios monitoring system. It reports on the health of the zpool your filesystems are on. It only monitors filesystems that are configured in the sanoid.conf file. ++ --monitor-capacity + + This option is designed to be run by a Nagios monitoring system. It reports on the capacity of the zpool your filesystems are on. It only monitors pools that are configured in the sanoid.conf file. The default limits are 80% for the warning and 95% for the critical state. Those can be overridden by providing them + along like '=80,95". + + --force-update This clears out sanoid's zfs snapshot listing cache. This is normally not needed. diff --git a/sanoid b/sanoid index d6e58ce..660dfb2 100755 --- a/sanoid +++ b/sanoid @@ -18,7 +18,8 @@ use Time::Local; # to parse dates in reverse my %args = ("configdir" => "/etc/sanoid"); GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet", "monitor-health", "force-update", "configdir=s", - "monitor-snapshots", "take-snapshots", "prune-snapshots" + "monitor-snapshots", "take-snapshots", "prune-snapshots", + "monitor-capacity:s" ) or pod2usage(2); # If only config directory (or nothing) has been specified, default to --cron --verbose @@ -52,6 +53,7 @@ my @params = ( \%config, \%snaps, \%snapsbytype, \%snapsbypath ); if ($args{'debug'}) { $args{'verbose'}=1; blabber (@params); } if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); } if ($args{'monitor-health'}) { monitor_health(@params); } +if (defined($args{'monitor-capacity'})) { monitor_capacity(@params); } if ($args{'force-update'}) { my $snaps = getsnaps( \%config, $cacheTTL, 1 ); } if ($args{'cron'}) { @@ -174,6 +176,57 @@ sub monitor_snapshots { exit $errorlevel; } + +#################################################################################### +#################################################################################### +#################################################################################### + +sub monitor_capacity { + my ($config, $snaps, $snapsbytype, $snapsbypath) = @_; + my %pools; + my @messages; + my $errlevel=0; + + my %capacitylimits = ( + "warn" => 80, + "crit" => 95 + ); + + # if provided, parse capacity limits + if ($args{'monitor-capacity'} ne "") { + my @values = split(',', $args{'monitor-capacity'}); + + if (!check_capacity_limit($values[0])) { + die "ERROR: invalid zpool capacity warning limit!\n"; + } + $capacitylimits{"warn"} = $values[0]; + + if (scalar @values > 1) { + if (!check_capacity_limit($values[1])) { + die "ERROR: invalid zpool capacity critical limit!\n"; + } + $capacitylimits{"crit"} = $values[1]; + } + } + + foreach my $path (keys %{ $snapsbypath}) { + my @pool = split ('/',$path); + $pools{$pool[0]}=1; + } + + foreach my $pool (keys %pools) { + my ($exitcode, $msg) = check_zpool_capacity($pool,\%capacitylimits); + if ($exitcode > $errlevel) { $errlevel = $exitcode; } + chomp $msg; + push (@messages, $msg); + } + + my @warninglevels = ('','*** WARNING *** ','*** CRITICAL *** '); + my $message = $warninglevels[$errlevel] . join (', ',@messages); + print "$message\n"; + exit $errlevel; +} + #################################################################################### #################################################################################### #################################################################################### @@ -900,6 +953,70 @@ sub check_zpool() { return ($ERRORS{$state},$msg); } # end check_zpool() +sub check_capacity_limit() { + my $value = shift; + + if ($value !~ /^\d+\z/) { + return undef; + } + + if ($value < 1 || $value > 100) { + return undef; + } + + return 1 +} + +sub check_zpool_capacity() { + my %ERRORS=('DEPENDENT'=>4,'UNKNOWN'=>3,'OK'=>0,'WARNING'=>1,'CRITICAL'=>2); + my $state="UNKNOWN"; + my $msg="FAILURE"; + + my $pool=shift; + my $capacitylimitsref=shift; + my %capacitylimits=%$capacitylimitsref; + + my $statcommand="/sbin/zpool list -H -o cap $pool"; + + if (! open STAT, "$statcommand|") { + print ("$state '$statcommand' command returns no result!\n"); + exit $ERRORS{$state}; + } + + my $line = ; + close(STAT); + + chomp $line; + my @row = split(/ +/, $line); + my $cap=$row[0]; + + ## check for valid capacity value + if ($cap !~ m/^[0-9]{1,3}%$/ ) { + $state = "CRITICAL"; + $msg = sprintf "ZPOOL {%s} does not exist and/or is not responding!\n", $pool; + print $state, " ", $msg; + exit ($ERRORS{$state}); + } + + $state="OK"; + + # check capacity + my $capn = $cap; + $capn =~ s/\D//g; + + if ($capn >= $capacitylimits{"warn"}) { + $state = "WARNING"; + } + + if ($capn >= $capacitylimits{"crit"}) { + $state = "CRITICAL"; + } + + $msg = sprintf "ZPOOL %s : %s\n", $pool, $cap; + $msg = "$state $msg"; + return ($ERRORS{$state},$msg); +} # end check_zpool_capacity() + ###################################################################################################### ###################################################################################################### ###################################################################################################### @@ -1070,19 +1187,20 @@ Assumes --cron --verbose if no other arguments (other than configdir) are specif Options: - --configdir=DIR Specify a directory to find config file sanoid.conf + --configdir=DIR Specify a directory to find config file sanoid.conf - --cron Creates snapshots and purges expired snapshots - --verbose Prints out additional information during a sanoid run - --readonly Simulates creation/deletion of snapshots - --quiet Suppresses non-error output - --force-update Clears out sanoid's zfs snapshot cache + --cron Creates snapshots and purges expired snapshots + --verbose Prints out additional information during a sanoid run + --readonly Simulates creation/deletion of snapshots + --quiet Suppresses non-error output + --force-update Clears out sanoid's zfs snapshot cache - --monitor-health Reports on zpool "health", in a Nagios compatible format - --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format - --take-snapshots Creates snapshots as specified in sanoid.conf - --prune-snapshots Purges expired snapshots as specified in sanoid.conf + --monitor-health Reports on zpool "health", in a Nagios compatible format + --monitor-capacity[=wlimit[,climit]] Reports on zpool capacity, in a Nagios compatible format + --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format + --take-snapshots Creates snapshots as specified in sanoid.conf + --prune-snapshots Purges expired snapshots as specified in sanoid.conf - --help Prints this helptext - --version Prints the version number - --debug Prints out a lot of additional information during a sanoid run + --help Prints this helptext + --version Prints the version number + --debug Prints out a lot of additional information during a sanoid run From b405b589801dd05ea12be500766ef333638b3e80 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 1 Mar 2018 09:22:22 +0100 Subject: [PATCH 035/148] let monitor-capacity parse the limits from the configuration file --- README.md | 3 +- sanoid | 92 ++++++++++++++++++++++++-------------------- sanoid.defaults.conf | 4 ++ 3 files changed, 55 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 1b3b37f..324fd30 100644 --- a/README.md +++ b/README.md @@ -64,8 +64,7 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da + --monitor-capacity - This option is designed to be run by a Nagios monitoring system. It reports on the capacity of the zpool your filesystems are on. It only monitors pools that are configured in the sanoid.conf file. The default limits are 80% for the warning and 95% for the critical state. Those can be overridden by providing them - along like '=80,95". + This option is designed to be run by a Nagios monitoring system. It reports on the capacity of the zpool your filesystems are on. It only monitors pools that are configured in the sanoid.conf file. + --force-update diff --git a/sanoid b/sanoid index 660dfb2..a4a256a 100755 --- a/sanoid +++ b/sanoid @@ -19,7 +19,7 @@ my %args = ("configdir" => "/etc/sanoid"); GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet", "monitor-health", "force-update", "configdir=s", "monitor-snapshots", "take-snapshots", "prune-snapshots", - "monitor-capacity:s" + "monitor-capacity" ) or pod2usage(2); # If only config directory (or nothing) has been specified, default to --cron --verbose @@ -53,7 +53,7 @@ my @params = ( \%config, \%snaps, \%snapsbytype, \%snapsbypath ); if ($args{'debug'}) { $args{'verbose'}=1; blabber (@params); } if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); } if ($args{'monitor-health'}) { monitor_health(@params); } -if (defined($args{'monitor-capacity'})) { monitor_capacity(@params); } +if ($args{'monitor-capacity'}) { monitor_capacity(@params); } if ($args{'force-update'}) { my $snaps = getsnaps( \%config, $cacheTTL, 1 ); } if ($args{'cron'}) { @@ -187,35 +187,39 @@ sub monitor_capacity { my @messages; my $errlevel=0; - my %capacitylimits = ( - "warn" => 80, - "crit" => 95 - ); + # build pool list with corresponding capacity limits + foreach my $section (keys %config) { + my @pool = split ('/',$section); - # if provided, parse capacity limits - if ($args{'monitor-capacity'} ne "") { - my @values = split(',', $args{'monitor-capacity'}); + if (scalar @pool == 1 || !defined($pools{$pool[0]}) ) { + my %capacitylimits; - if (!check_capacity_limit($values[0])) { - die "ERROR: invalid zpool capacity warning limit!\n"; - } - $capacitylimits{"warn"} = $values[0]; + if (!check_capacity_limit($config{$section}{'capacity_warn'})) { + die "ERROR: invalid zpool capacity warning limit!\n"; + } - if (scalar @values > 1) { - if (!check_capacity_limit($values[1])) { + if ($config{$section}{'capacity_warn'} != 0) { + $capacitylimits{'warn'} = $config{$section}{'capacity_warn'}; + } + + if (!check_capacity_limit($config{$section}{'capacity_crit'})) { die "ERROR: invalid zpool capacity critical limit!\n"; } - $capacitylimits{"crit"} = $values[1]; - } - } - foreach my $path (keys %{ $snapsbypath}) { - my @pool = split ('/',$path); - $pools{$pool[0]}=1; + if ($config{$section}{'capacity_crit'} != 0) { + $capacitylimits{'crit'} = $config{$section}{'capacity_crit'}; + } + + if (%capacitylimits) { + $pools{$pool[0]} = \%capacitylimits; + } + } } foreach my $pool (keys %pools) { - my ($exitcode, $msg) = check_zpool_capacity($pool,\%capacitylimits); + my $capacitylimitsref = $pools{$pool}; + + my ($exitcode, $msg) = check_zpool_capacity($pool,\%$capacitylimitsref); if ($exitcode > $errlevel) { $errlevel = $exitcode; } chomp $msg; push (@messages, $msg); @@ -956,11 +960,11 @@ sub check_zpool() { sub check_capacity_limit() { my $value = shift; - if ($value !~ /^\d+\z/) { + if (!defined($value) || $value !~ /^\d+\z/) { return undef; } - if ($value < 1 || $value > 100) { + if ($value < 0 || $value > 100) { return undef; } @@ -1004,12 +1008,16 @@ sub check_zpool_capacity() { my $capn = $cap; $capn =~ s/\D//g; - if ($capn >= $capacitylimits{"warn"}) { - $state = "WARNING"; + if (defined($capacitylimits{"warn"})) { + if ($capn >= $capacitylimits{"warn"}) { + $state = "WARNING"; + } } - if ($capn >= $capacitylimits{"crit"}) { - $state = "CRITICAL"; + if (defined($capacitylimits{"crit"})) { + if ($capn >= $capacitylimits{"crit"}) { + $state = "CRITICAL"; + } } $msg = sprintf "ZPOOL %s : %s\n", $pool, $cap; @@ -1187,20 +1195,20 @@ Assumes --cron --verbose if no other arguments (other than configdir) are specif Options: - --configdir=DIR Specify a directory to find config file sanoid.conf + --configdir=DIR Specify a directory to find config file sanoid.conf - --cron Creates snapshots and purges expired snapshots - --verbose Prints out additional information during a sanoid run - --readonly Simulates creation/deletion of snapshots - --quiet Suppresses non-error output - --force-update Clears out sanoid's zfs snapshot cache + --cron Creates snapshots and purges expired snapshots + --verbose Prints out additional information during a sanoid run + --readonly Simulates creation/deletion of snapshots + --quiet Suppresses non-error output + --force-update Clears out sanoid's zfs snapshot cache - --monitor-health Reports on zpool "health", in a Nagios compatible format - --monitor-capacity[=wlimit[,climit]] Reports on zpool capacity, in a Nagios compatible format - --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format - --take-snapshots Creates snapshots as specified in sanoid.conf - --prune-snapshots Purges expired snapshots as specified in sanoid.conf + --monitor-health Reports on zpool "health", in a Nagios compatible format + --monitor-capacity Reports on zpool capacity, in a Nagios compatible format + --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format + --take-snapshots Creates snapshots as specified in sanoid.conf + --prune-snapshots Purges expired snapshots as specified in sanoid.conf - --help Prints this helptext - --version Prints the version number - --debug Prints out a lot of additional information during a sanoid run + --help Prints this helptext + --version Prints the version number + --debug Prints out a lot of additional information during a sanoid run diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 35c804d..d86cc47 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -70,3 +70,7 @@ monthly_warn = 32 monthly_crit = 35 yearly_warn = 0 yearly_crit = 0 + +# default limits for capacity checks (if set to 0, limit will not be checked) +capacity_warn = 80 +capacity_crit = 95 From 8dfdd1a7169b97a8b6d2a6a6c84b6ac3826caa49 Mon Sep 17 00:00:00 2001 From: Janne Savikko Date: Thu, 15 Mar 2018 13:50:12 +0200 Subject: [PATCH 036/148] syncoid: fix --version in options list of helptext syncoid does not have verbose option. Replace verbose with version in options list of helptext. --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 7337f5b..29f0176 100755 --- a/syncoid +++ b/syncoid @@ -956,7 +956,7 @@ Options: --sshoption|o=OPTION Passes OPTION to ssh for remote usage. Can be specified multiple times --help Prints this helptext - --verbose Prints the version number + --version Prints the version number --debug Prints out a lot of additional information during a syncoid run --monitor-version Currently does nothing --quiet Suppresses non-error output From ecf2a852b5e0fa54131c878597c9b24cffb8a663 Mon Sep 17 00:00:00 2001 From: danielewood Date: Thu, 12 Apr 2018 17:43:08 -0700 Subject: [PATCH 037/148] Added support for ZStandard compression. Available in all major distros with a simple yum/apt-get/pkg. References: ZSTD Compression by Allan Jude - https://www.youtube.com/watch?v=hWnWEitDPlM Zstandard - https://facebook.github.io/zstd/ --- syncoid | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 7337f5b..baf5053 100755 --- a/syncoid +++ b/syncoid @@ -368,6 +368,18 @@ sub compressargset { decomrawcmd => '/usr/bin/pigz', decomargs => '-dc', }, + 'zstd-fast' => { + rawcmd => '/usr/bin/zstd', + args => '-3', + decomrawcmd => '/usr/bin/zstd', + decomargs => '-dc', + }, + 'zstd-slow' => { + rawcmd => '/usr/bin/zstd', + args => '-19', + decomrawcmd => '/usr/bin/zstd', + decomargs => '-dc', + }, 'lzo' => { rawcmd => '/usr/bin/lzop', args => '', @@ -378,7 +390,7 @@ sub compressargset { if ($value eq 'default') { $value = $DEFAULT_COMPRESSION; - } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'lzo', 'default', 'none'))) { + } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lzo', 'default', 'none'))) { warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION"; $value = $DEFAULT_COMPRESSION; } From 1b5ab20b0d9fd83be65ab7266e294dc914a78cf0 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 25 Apr 2018 14:58:01 +0200 Subject: [PATCH 038/148] use resumeable zfs send/receive as default if supported by source and target --- README.md | 5 +++++ syncoid | 40 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9c4e6ba..b66deac 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,7 @@ syncoid root@remotehost:data/images/vm backup/images/vm Which would pull-replicate the filesystem from the remote host to the local system over an SSH tunnel. Syncoid supports recursive replication (replication of a dataset and all its child datasets) and uses mbuffer buffering, lzop compression, and pv progress bars if the utilities are available on the systems used. +If ZFS supports resumeable send/receive streams on both the source and target those will be enabled as default. ##### Syncoid Command Line Options @@ -147,6 +148,10 @@ Syncoid supports recursive replication (replication of a dataset and all its chi This argument tells syncoid to restrict itself to existing snapshots, instead of creating a semi-ephemeral syncoid snapshot at execution time. Especially useful in multi-target (A->B, A->C) replication schemes, where you might otherwise accumulate a large number of foreign syncoid snapshots. ++ --no-resume + + This argument tells syncoid to not use resumeable zfs send/receive streams. + + --dumpsnaps This prints a list of snapshots during the run. diff --git a/syncoid b/syncoid index 8a31f37..170799b 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "resume") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -46,6 +46,7 @@ my $rawsourcefs = $args{'source'}; my $rawtargetfs = $args{'target'}; my $debug = $args{'debug'}; my $quiet = $args{'quiet'}; +my $resume = !$args{'no-resume'}; my $zfscmd = '/sbin/zfs'; my $sshcmd = '/usr/bin/ssh'; @@ -163,7 +164,7 @@ sub syncdataset { my $receiveextraargs = ""; my $receivetoken; - if (defined $args{'resume'}) { + if ($resume) { # save state of interrupted receive stream $receiveextraargs = "-s"; @@ -457,6 +458,8 @@ sub checkcommands { $avail{'localmbuffer'} = 1; $avail{'sourcembuffer'} = 1; $avail{'targetmbuffer'} = 1; + $avail{'sourceresume'} = 1; + $avail{'targetresume'} = 1; return %avail; } @@ -564,6 +567,37 @@ sub checkcommands { $avail{'localpv'} = 1; } + # check for ZFS resume feature support + if ($resume) { + my $resumechkcmd = "$zfscmd get receive_resume_token -d 0"; + + if ($debug) { print "DEBUG: checking availability of zfs resume feature on source...\n"; } + $avail{'sourceresume'} = system("$sourcessh $resumechkcmd >/dev/null 2>&1"); + $avail{'sourceresume'} = $avail{'sourceresume'} == 0 ? 1 : 0; + + if ($debug) { print "DEBUG: checking availability of zfs resume feature on target...\n"; } + $avail{'targetresume'} = system("$targetssh $resumechkcmd >/dev/null 2>&1"); + $avail{'targetresume'} = $avail{'targetresume'} == 0 ? 1 : 0; + + if ($avail{'sourceresume'} == 0 || $avail{'targetresume'} == 0) { + # disable resume + $resume = ''; + + my @hosts = (); + if ($avail{'sourceresume'} == 0) { + push @hosts, 'source'; + } + if ($avail{'targetresume'} == 0) { + push @hosts, 'target'; + } + my $affected = join(" and ", @hosts); + print "WARN: ZFS resume feature not available on $affected machine - sync will continue without resume support.\n"; + } + } else { + $avail{'sourceresume'} = 0; + $avail{'targetresume'} = 0; + } + return %avail; } @@ -1111,4 +1145,4 @@ Options: --quiet Suppresses non-error output --dumpsnaps Dumps a list of snapshots during the run --no-command-checks Do not check command existence before attempting transfer. Not recommended - --resume Save the state of unfinished receive streams and resume interrupted ones if available + --no-resume Don't use the ZFS resume feature if available From e6eec0aca041dc5464f30cdededaa597dc3e062f Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 25 Apr 2018 17:15:55 +0200 Subject: [PATCH 039/148] fix snapshot list generation and incremental sync for snapshots containing whitespaces --- syncoid | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/syncoid b/syncoid index 8825dcb..0654ac3 100755 --- a/syncoid +++ b/syncoid @@ -189,6 +189,7 @@ sub syncdataset { return 0; } } + my $newsyncsnapescaped = escapeshellparam($newsyncsnap); # there is currently (2014-09-01) a bug in ZFS on Linux # that causes readonly to always show on if it's EVER @@ -257,7 +258,7 @@ sub syncdataset { # $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); - $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnapescaped $sourcefsescaped\@$newsyncsnap"; + $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap","$sourcefs\@$newsyncsnap",$sourceisroot); $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -325,7 +326,7 @@ sub syncdataset { system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped"); } - my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnap"; + my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); @@ -909,7 +910,7 @@ sub getsnaps() { my $guid = $line; $guid =~ s/^.*\sguid\s*(\d*).*/$1/; my $snap = $line; - $snap =~ s/^.*\@(\S*)\s*guid.*$/$1/; + $snap =~ s/^.*\@(.*)\tguid.*$/$1/; $snaps{$type}{$snap}{'guid'}=$guid; } } @@ -921,7 +922,7 @@ sub getsnaps() { my $creation = $line; $creation =~ s/^.*\screation\s*(\d*).*/$1/; my $snap = $line; - $snap =~ s/^.*\@(\S*)\s*creation.*$/$1/; + $snap =~ s/^.*\@(.*)\tcreation.*$/$1/; $snaps{$type}{$snap}{'creation'}=$creation; } } From 4ebc0abef5a4999c344e9e7d37d2686e101609f3 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 25 Apr 2018 14:58:35 -0400 Subject: [PATCH 040/148] fix use of uninitialized string in escapeshellparam() --- syncoid | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index 784bd90..10c78c1 100755 --- a/syncoid +++ b/syncoid @@ -1087,9 +1087,12 @@ sub getdate { } sub escapeshellparam { - my ($par) = @_; - # "escape" all single quotes - $par =~ s/'/'"'"'/g; + my $par; + if (scalar @_) { + ($par) = @_; + # "escape" all single quotes + $par =~ s/'/'"'"'/g; + } # single-quote entire string return "'$par'"; } From 2732392088cf713898882e54561a20a84b1b6538 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 25 Apr 2018 15:18:00 -0400 Subject: [PATCH 041/148] fix use of uninitialized value in escapeshellparam() --- syncoid | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index 10c78c1..df08122 100755 --- a/syncoid +++ b/syncoid @@ -1087,11 +1087,14 @@ sub getdate { } sub escapeshellparam { - my $par; - if (scalar @_) { - ($par) = @_; + my ($par) = @_; + # avoid use of uninitialized string in regex + if (length($par)) { # "escape" all single quotes $par =~ s/'/'"'"'/g; + } else { + # avoid use of uninitialized string in concatenation below + $par = ''; } # single-quote entire string return "'$par'"; From 72245338a805b565183f7a0579e98757c3e6cb14 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 25 Apr 2018 15:21:56 -0400 Subject: [PATCH 042/148] special character handling and resumable zfs receive support in syncoid --- CHANGELIST | 3 +++ VERSION | 2 +- sanoid | 2 +- syncoid | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 9af90f6..515d05d 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,6 @@ +1.4.18 implemented special character handling and support of ZFS resume/receive tokens by default in syncoid, + thank you @phreaker0! + 1.4.17 changed die to warn when unexpectedly unable to remove a snapshot - this allows sanoid to continue taking/removing other snapshots not affected by whatever lock prevented the first from being taken or removed diff --git a/VERSION b/VERSION index 04e0d3f..f689e8c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.17 +1.4.18 diff --git a/sanoid b/sanoid index d6e58ce..b6dc9fe 100755 --- a/sanoid +++ b/sanoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '1.4.17'; +$::VERSION = '1.4.18'; use strict; use warnings; diff --git a/syncoid b/syncoid index df08122..e63b68c 100755 --- a/syncoid +++ b/syncoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '1.4.16'; +$::VERSION = '1.4.18'; use strict; use warnings; From 979cb8bc3d154e30e7f0fbc016a1fdccc509941d Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Wed, 25 Apr 2018 15:24:24 -0400 Subject: [PATCH 043/148] resume/receive support in syncoid 1.4.18 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b66deac..cc75bdf 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,8 @@ Which would pull-replicate the filesystem from the remote host to the local syst Syncoid supports recursive replication (replication of a dataset and all its child datasets) and uses mbuffer buffering, lzop compression, and pv progress bars if the utilities are available on the systems used. If ZFS supports resumeable send/receive streams on both the source and target those will be enabled as default. +As of 1.4.18, syncoid also automatically supports and enables resume of interrupted replication when both source and target support this feature. + ##### Syncoid Command Line Options + [source] From b3b69c598c6350cedd7db938692f591935417997 Mon Sep 17 00:00:00 2001 From: dcrdev Date: Sat, 28 Apr 2018 17:38:19 +0100 Subject: [PATCH 044/148] Restructure dist packages & bump rpm spec to 1.4.18 --- {debian => packages/debian}/changelog | 0 {debian => packages/debian}/compat | 0 {debian => packages/debian}/control | 0 {debian => packages/debian}/copyright | 0 {debian => packages/debian}/rules | 0 .../debian}/sanoid.README.Debian | 0 {debian => packages/debian}/sanoid.docs | 0 {debian => packages/debian}/sanoid.service | 0 {debian => packages/debian}/sanoid.timer | 0 {debian => packages/debian}/source/format | 0 packages/rhel/sanoid-1.4.18.tar.gz | Bin 0 -> 45685 bytes sanoid.spec => packages/rhel/sanoid.spec | 9 +++++---- packages/rhel/sources | 1 + 13 files changed, 6 insertions(+), 4 deletions(-) rename {debian => packages/debian}/changelog (100%) rename {debian => packages/debian}/compat (100%) rename {debian => packages/debian}/control (100%) rename {debian => packages/debian}/copyright (100%) rename {debian => packages/debian}/rules (100%) rename {debian => packages/debian}/sanoid.README.Debian (100%) rename {debian => packages/debian}/sanoid.docs (100%) rename {debian => packages/debian}/sanoid.service (100%) rename {debian => packages/debian}/sanoid.timer (100%) rename {debian => packages/debian}/source/format (100%) create mode 100644 packages/rhel/sanoid-1.4.18.tar.gz rename sanoid.spec => packages/rhel/sanoid.spec (92%) create mode 100644 packages/rhel/sources diff --git a/debian/changelog b/packages/debian/changelog similarity index 100% rename from debian/changelog rename to packages/debian/changelog diff --git a/debian/compat b/packages/debian/compat similarity index 100% rename from debian/compat rename to packages/debian/compat diff --git a/debian/control b/packages/debian/control similarity index 100% rename from debian/control rename to packages/debian/control diff --git a/debian/copyright b/packages/debian/copyright similarity index 100% rename from debian/copyright rename to packages/debian/copyright diff --git a/debian/rules b/packages/debian/rules similarity index 100% rename from debian/rules rename to packages/debian/rules diff --git a/debian/sanoid.README.Debian b/packages/debian/sanoid.README.Debian similarity index 100% rename from debian/sanoid.README.Debian rename to packages/debian/sanoid.README.Debian diff --git a/debian/sanoid.docs b/packages/debian/sanoid.docs similarity index 100% rename from debian/sanoid.docs rename to packages/debian/sanoid.docs diff --git a/debian/sanoid.service b/packages/debian/sanoid.service similarity index 100% rename from debian/sanoid.service rename to packages/debian/sanoid.service diff --git a/debian/sanoid.timer b/packages/debian/sanoid.timer similarity index 100% rename from debian/sanoid.timer rename to packages/debian/sanoid.timer diff --git a/debian/source/format b/packages/debian/source/format similarity index 100% rename from debian/source/format rename to packages/debian/source/format diff --git a/packages/rhel/sanoid-1.4.18.tar.gz b/packages/rhel/sanoid-1.4.18.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..df137cd5d635538c965b675d9e48d0f1abab742a GIT binary patch literal 45685 zcmV(wKq2Q&StY$ zR}O~+XOQ8LT!NH5uC4p+_dE@Nt>;5B9uh#K(f#z(-B205f49hrd6d0d z>L}6G7oYw!#pmI}hw^`XZvP(-55Jfl>>p2ONBhS|{C#$KG&}ra@f*l;=hN6as<32L z6!pj6`?cq{H}`+!bF{DGV!hUR{rJ)Gqj>(!JU&QfM~C`gs*lrYI*+HbhtVue;y6B< zK6-RGO9ubjGJpPjx{rg}go!{PtoFP#7W{=vcQ4*nnFx8FsH|K#WI4qlmcnQ2A=ohV!C z*hHC%mr)hPwXW1M%9G6G3x)qH+mvNd)v8F!j;+-je1fpD;=$LHGMD`h%CokH(x-G#Be%w6IHpoS?XLh zxqc7XAY-<{KcaaiFI0M6VC6{JJSy$7sMQE77#miLvaGnV%2_%G#6@14yusjTg&6@VhxC1 zR9?))Z}XvA>x#R4vaBLoPiH$;>G!qH6a1E%_mneedQ_;QtWA;69$!b<(0$+McU4B# z*2}7B7E6w}x_UWoC;nr!D&}u?T0InLI+|}rMXuWg>Y{`qbDW*EQMI82Rn${Je+vyn zt5mG3I>M1chl!yawp);%lr)KcG@W*y9bclAs)R#z6P zIkB5HENq3@&U>c#=?7hMH3TGfEPy zr_3TRDm>Sig)iPLO}tb|p`EQ!h9L|JhCZaptc%>#MK$&|b99f*w6?)jXbsBRrFwdC z?tiw*o5xr^ZK?`_!*_}t|MAvk={VRhd}&lX_H#wmqTwdhC~Oi(PV>%nRH?M7n+g)< zS|?$1wpeS-L+wC}%8K0-FsM@09A=&c>s5uDkdoF#Sf#V{?(4patnAD!Mick?IHVmD zu%K3x;)oz{r58XX%T08^13<`&8&x)0W>pNpkxianK2wVZPw`zxI0vf#TkM2N>Abyt zpBq_6b-dJqZyY-AuLnE9P{@OYznabV`O=dl8m*zK7V!ei9jGfW+l|RG5wC-#YU9-x zOiz1gikBu!f=T1#Qlc{*I#_ucnauVrw!`Y;Xn!ut~aWR#Q12#>Zjliz324v+yWAM z;93MzX`?V~!JUo*sc2w{8|O`9cRB`1XkxG%{O(PGN0z5JUVKX$fz{fq^#?I{gSd=C zhCy^|^%fcS*NrkuaJ<%5Fy|)P1Uz$4CLyV(2x!y0I+fmSic)!}`-zIe`q5hdba%6u z&WXXz1+L>yfgw*lS-TU>U@-RJNCcofCecxVh$(?pGe#jpFNrdbOSAYeN`d`8cx^wh z;5lRU3L>h%XvlPSFq-Wj%&re$1?@nnTK}yvAP;o5FhLI~u>an~iw~w0dlmN$KMUEB zuu?X{1dor#`|3~Ty9o^t-W?_Ub@t78=6)TkRzgY;VJtRq^mSNtu4`(wCyq0>Z8<~+ zSr$fe&n~va8mr5Ku^42nw|y9G(!gk1^MFqkWyy2ipZ@FD_X*-76&$Y?oK?G#!r3+N z)f_p2N-7}fo)b6w07AuZKBTK66gsqVG>YqYBqdNb8^Hnv;VX%*jLhMu2o@G)qnn}v zOf1YIngb~=PA|FqN?;9+P~5=!U^geSON0o0(*}Jtn@+!04TR`OAo;M0u3>*dpzQ|7 zwdMdE8srAdk&4WVx`u9byb|i`gMSTCR6al#44{3vxQ;g(`k5+32#-~;MUNGJtKqwv5JEvHB5 zb$J)99ypi?R{1tsBci@WWbp!h(?S91YK@~>cN~pIjQ#VQBkci z!{~flHb;f=Q-%y;&}MG<1^(zmLkGpJ;q~x0jIL&h;|7HZX6K`*p*b5-K$bHWnEw4Ql{_$KJyWVI6t<0!OQZqKVgX%{MtFe~1F@LyYh6L^rJf`AvOMdr9!wfrP39(_lzcWKaYhOmGP|_M zR)7;f#Buj_gwC&(jGBX}iTt)_#)CG9`kZ|?Yz^rIP7xs34xwCo0Kt3*${;u~?w!AV zdG>~91WkB=Lj+RiJ8LojSO4L#hoNC8r2@vJMd3N*yfwdEy}EKRVLmvu8*3XMQ;=>b zZG;)kHBZV%AekB(7&)9UF_=fZiWZn%k~&fWjs?ds*Cuh8;)g2}P76?>nP5{SSZS+y z$gttA1ohp{)T=jAZz){D3%neWCaEk}BkOc7d+5%pe-!*7I1CxNUtcN&BqXw7M^6m% zunPFxWXJ*?pOq|%e(DkwXD0FloDIg4fg?(W1sosIF2ZrN4oq%FCR#4MV9{6$%eRu*LfXGI9*CKdkE~rv`6%qk51)JR zPFe(Waf(z_4%)UvxJ`kV2$IqMbW|4EW_B<=dV*8UF+XyQ^)Ra9lqNY+;hV8^$1d)5 zSxJF*{k*Sne#YuQsB;=` zzWVn)uYxF~6Z3{gC3Q@g=!K9v=L)H4Q>rDjDV&(k_RieWD?%u}-fKYKjEKW%|bp>|mT>!32GfJ^dL zfjU4M@<2Tfa7Rk-O3my&8-XkuQe|3}_~jX^B0)k6PaJSNlvRm3O7<&O?nA^YZD;?HaSr^cR?ojej+LvG1ngi*Jd5An@3nfTlO0D z8XOlo|Db-*b6thBzJ*_8YVqpbhq5TL-MyZ9N+GvmErySj;IgZ=WIlhufsMq9SxFnI zzZMtjdOChMK6Fcj!U4+|Zo-26OsX0jB{R}PTEt{lhy;`6jg&Y7VYahGVoHyEhscSf zGc3{?ASa+42iPjWqbM!b4#hp}gC)FoqCi;TI~Uh!_6v^#QpS}2Nk%0KQc@rvt*o1S z+QO?ZlHZz=O1y8au|MdQ)@9ohlI7OSLJ3)}HbDa5_X%XywfI0$=exi+)9L#FgX7 z3Ycs1?ILFn2a|=27*S)TLFQzqlxGHcZVG%bnzVYmBXX8KAt5U-}uHMEEqCc2I%ZlM3)IJ_v@b`s1U-J=rXq08QUeZX|Y&(|Qc zTjhI1K~PKh=7e<@|79|BUx$j{CsFrcUyaA(J*TkIs9PDLkE$LSe&D72rUjZ_XrHs& z>Vz#^<&HHP*3@biK`5wDSH9up+}#^HULj{D1qL77w|CE=4p#8K4#k0!Juk`eeUeB8 zzDWv~!gei47=fuh=?ejwz4x9WN(D8SKy&%(_372s56{F8CYHgNm=8(ZSz-ZT+;%T= zOAHpuZh`?8d24`yG+V!ihuz>7rMZPapI*Ja_>oPGN}>=g6%$f@tL*K!rOe9VJ9u ziOzV>^S8{bv=AzY-|Jgison8HLdo`e9+TDz(P4zW4rv8yl$e96YaPRVy6{bnp%_VsG7vcMzs$L9TBSEOYD?d0C|#|?#TD0 z@v;;}Kr~nnGIkw~+|A?O?i{nUUPoJ8Q1Ze}TUNp>9+M7$fmJH~+_n{l_x~l)TxFN( zWB*#uM%R0{*WTZH!&qSFys;Z^G6Ij1ROY$hy^j*NCV@e_%-kaCFBzEEMdAWrH=_k$ zxF&9gKB2mYucTw{dAnNY5ABVNHaWCDv`pLVvP&dfo9*)@)qm`>egE(Io3k_6*qC8= ztv}ToAHV-MogM5S-M;^KbTE7P`TpPU@Y%kI<*sD-Yd!v+6(L}|`tOr(ULNz=x_%ET zSQ7z#4dZrqV9<38Zq=dzCek*N1O+tW%JT#19=7cB4A zLMn?|m<)NNZByuh2#B!1DcNaMaV1>p+Tj1f+Ik%X42XO=mbIjzFRCF-&`VrLfztCn zq9h*jeBf)`LsE}(scd+f&@1IjG+%FAt1wR_`<93=TZeq!7nSqH{w0h7r*WZ$R~cNw zI%augaGuH>z8?~~?x3q@Z5?Ph%c1Wk$Yj1;WF(9vP#3R;xqEqS-@$F|WH6vVJek|% zByAq^k~zT(Hs&MyEv5L!1>ry z{*3+q_-7mc{T0nmyvDEK|Ni08!JYfxv;EKazkiR<)sL4?U!6Z2jNE4sZb7+Z$=$88 z%b_Z-hl(lXP_AAe z-ZD-h=%!r*Ve(bzo}H+utV4E(V{+GR{L%2Wr-*5+XH_!fXP1E046%PH3e%3W{o4OhU6bnNsdwR zrpi#_t?QME2vU;rbKRb%E?i;uTD$z7_qCAKuuB|nO1gG&Khi-HgR9fa&UXj?-e}Kr zTk3{{tbtWMl`84U$$4&`v-B*-<*(t^p6%#CxNKo+xYeB@Jnp&fl@o+cJ$-$8N!I0V zwJoLh{g*G*@zMA{n;srMy!-cG)8pgM z@&9-Dytw>cy*Rr(dvkiBUVr}`3_y7!xQ$Zb^Jib7I~c0{N9vCa!r1$*IdOj4VUi)u0Xw}CnX3U1`ir(9QC zo4WStC}tPHYB58zWI5-B8a_0<)GLt^BQp~tvg@`F=Tb>YfTXA>WjhJE>}T{YvflrB z&0CRD-*cC4MQb{j%Y3h!IaWhn;#TDCX5Jn_2%yB7PU}`0kcEK_%HUR9U|bo&>(nKx z{vcw7*O?{?-cqoHRW{_K%P>U&Zlgb9dQ7>=;XbmbR^{Omp`Yhl%7QqbNpzO0YsElVtV_q4kybj> zk-Lk|f(7H{MkI_Hc8DqCkNyrIXW}!?BHHr%w?mi6!V-R9CF4f3s~D{)4mHrqGvp)g zPinxGzrpBrKBR>YKuBUuf{f|!-|e>5Y!|=fehfq#giA4f6Y6=QbHF?Dsa=2E)xw!S z=`PtXs}wiJF87A*676+6e}R>qT2d@3iZv8d`+)-~Y?!(?l<)M*8)13h9fxeOlg_o0 zkSPpA&Q9NQ89PkJ?p!->Qux34k!q6nPq|*ezw>ccF0g0iG(!0-TzL@7@MWQTA?pI0SMk+gl@F$s-Msko_cS%SHlrIL|cS?ON z$+c9I(gI!So+f#6cRpztvb$Yx83J8I_9&Wp&KpqgBa-srMxi_ouBZEzVcH*7s6~o6u7r?RQ_SF5g`-&l`5JL zoHoTdb|mu@qQr!D?`0KwiP!Qu2JR-qd?l(cH8AK5_Mh$=zI%o5cbkitU0-fs{BOeU z$3&Rq@JA=*m&a2pu>59fa-{h&l?Q*j>j_yLMIc?DfDTZ(f%}^p}e19CuJ+B`7%L=pL?K_fe84twl|6Fq{!+uEq? z1q4z}KOUj;ahyluUDV`VO~tm6^f z4$y@afR=VDs4!9sQOw5_`G^lyKtWx|WDu2?Q8?V#Anme0V-p=2os5PNEypNgj^Nk{ zrOPu7mqcLWoyh?jcuft|3$d4|13fH0MG6q<{)&4B0N=YODk!r=FR?6(F+UOX4CiLx zNW&3Dv2@VK#Hx;f4`X0r6xWzpBj-qu0E(cMdh;O*$v(hnKxMol&X2D4$$*_IP~j4i z*7Jm~JGU7K5H(E;fDxtG%j`78xGa`>i5pXg`3#PiP8GFevNDTA@hBxPm?MnvKr<#m zS+!6ZaXd5|iU2X}3`0@X$Zm;8K6)bPVWlCpCAl_ne4`>qkfRGF+pG>FJqN-Gct%=Ek9Z!7fE6MZ0%Cw(V&JRgzx4&VVc;}% z#!5=aB!jExm?p>jWk{NdooNjxqaiLF@e_{WNG(SL0SG(!W^_Fx(?Lh`jNN||oA06< zQyek}49bL<)ngAF<8urgO&rlJ0-K*umIw-|Hmn3nK&-rI3#Y)G0PdF~M?knIN{u2N z4^E*Fm5Ic=dt87HjiD4c7b__UlFPkA9G6YcK&XgBoE(vSeGAb_rXLzGm*H+qmxT_u z_Fo-P#K*?&&gS;v_U_IBzP#AR06=vKr1rH#?~bW4drJGOr|6zNM^m|Q?#N1&HF32} z;z922yB2x<^x|`&>&#NaK%s0Ekz~^p(t-|!{S<)&M-wkIV7~0U6AFujCrWwBpT~wjT4T9l3x)0bULdRbWR6XSYPP{OYq&x1y zF$Yagtt;0FF|ef8kAMa2Fjpd!9(~MvV>QPjFeGd(mWg|pSWV^>A%BpnTt(a<{Z%_6 z`3heo>$CS30b||NI_P_j(OB@TCZG_FI;mI(c1+o9uY2k}QvE6aY=MlYJ3fKqT849% z&_ry`ct6su5KE|QHi2XAL63v~iGVPV%wA%oR89~zIUK5&61Ji2ln#z@H^^N_5RT|m zhu&~|1cw30^apYnWXrthJn+m%N+gYC;y7*$Xx3TG96IjVo~)BS!YC>;w{#>5=m>@9 zVRx!I_K6Azc!C(ty1~qQG^Eqs6t7ds$AB?Yr2b{>!u#}0l;Oa2CB2^A34qaAQbcz= zr+DUtOR!5I+K>#`MS;4GD3*=I8rlW2F0lm4QVK@%63>8vSuv+{;4BvG`zS%yQEV?B z4UulzbTeeLLB5;R8_Z0L>WSwMkYSu1xN*)B9)KPkqiF@;4LtG%5RG+bWH2v@U*E}) z<;ZK1$V9{)Fy^tiE1MVdh=WvbROqOm4%pU$JOnDZ)FG2vqkLgUW_5)f+o9Wl#w>U?8~!4sK`Jb=`d+H)KnX0{CK#&J2leX+vtBDaQxe(WI!cps z*%+%lbOe0+n3P{`AJD&+83T5U6&A@k8)oW8;oUWH1SfXVWMP*f4jY4d5D7%bp882b zLM#$SbSE64SrZXssSQ#BiJluMdg`5D^W@Q*h{`@WVzvO4LhLxv59mDUxA<84hm|D^ zwENU{kI;T2-Mh1eyocnv&`#rk@E&Ne^0oraHTM9?=aC6=O22`4nj#+-*>GnHGvKiK znCwR*s9-3!!SG!>NSMTED#9VqwQ@fk>72&*tueFQ@fxW7q!g36EqPcI^O zqi#aZJ$O;|GkDKQxYi#%74o88xIH>xr<9lJ5eB#71y8r5=PxC&>}7u<8-5I}U&Vh1AZ!4+$_7>~ZiOYJN{%If0qFI} zd!5esQ3GzM(ODw)c}o96Ymu}AcEUEwIx3BuQ6!7?K82~35r;O6CkGsbeODu~OBaga z3x)3zr$R(HpvESu6=qQrw4vNUFz5%Em!K*f4ty1PwSoGVa-*pJ9;DO+W2Q0k>D;=8 zk$ZB6LA_Q4*8uRMA6ZoAWI_$Rk$Hik7o8}QCrCvIX>s_pCs`&?e33jR7S{Y8oFaE@ z`LqEM7`t}lq9HmP!q5%$PDec-g)lRr3CKwr_DM)3KTmK?EN2#@m*P(SP&@-pq z&gW6_xDrr{LMBGj<=ziyowtZc<_T3QEAEa&n)8@W^mYRz2Vk-zTsUPsm62;49!s8W zyP#>U+Q-OV{T?G>|nTv zhnqc@@TeB)9_wd-0HbEyt!%k%p2ay(radc=uo$_H(a5Z9J|Qb!w}OEpq|~34i9_83 z-Zmvg^qwmCNgs?{AgluOnBaMFaWsfms2%D?p6&u)?_(A%t?J-P^GLfT-m zf5aCg@iYlAA_=fi8%)cypm5@#h?#3E&-!YnoD=!s3}`RZ=1p@vN9~lDa8btqNJBPB zosBTG!{#5Vs3Vbda6Dx0QAr5`{--N-O+QSZ1Z=Ucq7Mr@;rM2Cn|A;(Bw8>>b=g?C>< zr$xYL9U0sA*q&{dk#ya{0dv$Ofi|(E$NP1v8Q!2J;p&Ekn z?H;e$?!iL&J|EcU2%dGgty&v2g3?#lZ~-yl&TC`^Yh=iWxSe zb2`X+*$G}IAX~;{#&j_9ltOUqF~b+!=9E7&g9V?+>5%tP7y>42X-Ti3AGh~*J;H$r z!N!=N2*g^>hJHUSL%BWS+#-FFZ1Ohw`c$%60cg0E5_ibPc)lc;Ev@ zR3W65Yzsb*5fHdou~}y492*bFf0V)OPdx|Lz8yze%m{@Kd67!1}oJ38%TDy;KHQ~ zllyr-4_F$$Tq0G|p$iCVtO0nKA#T#ICewp_2m(JFWywD5)L3Y)K`M}++AP50XEsyW zwHi^tBJ3Dyxnnj3lMNS1jAaBvrirp|TtuirQTF7_+HATazsH?UHd0nxNYEo^)L>NV z&X|DpqYs?yRp++qVvSII^V}z@bztyTrrPYHHr#F`d-vcl;f#Oq4ebT2^ELe?Mpd|tqGHU--04_D zRlP6%JCXx9Gq z(X{DF&_|J^(WY$KNj&QI(C-9!ZFXYfPy|*fl0#J>xFBIcK8o6hGMEobl~z5+$@oHS z_D(~?$T^tE)73VVY)W{RekVEuboxN1eWlXn$t`~M-Oba^Q)PlhxNUxZC zX5$C-GS%?1im(y=QgC?{N0J3u&9t~B5Z@^yO&^8tZ0dqmVRaBB_z*A7?mMi9XN95V z6+#}0@kgp%2;{^Hr%5u#YA_GgP@D-Exb^{?iHLCu{Eh-fhv<9Q15E6QwEAixRkaSY z8yK0}!~4W)=bI=nqOgmm>9edi>GQ3I)hkAnY9b6Hv8;e+Vje`b_~Zl;8tt~J&eVa3 z7C+8CE491wW#*BpP?EL^|Yi56<@Kcm?f72*qkYMX{acv zP#D`Akcm;`ZonKUGXnp?)QiwF=t-{T7CJ%^&8yUi6$UG^G>lI+_%5e>@ZOcWVaF>d z(}^(vu?KD&;?>BYcY2W%4=iq8)QOi9)O(`>M;lGz_xi(lkg8sZsYJD^kFtu;Ip|JC ztUuG6KLK+8-@s(dW*}|Eu4lA zDY0adLoRQEWu|B8ZTW`xH^*Xt%t-jCsmjM@H9qh{2;kSpho)LciF21*Tr%6mqBVk} zmuCZUIAH_0j)}6AZ_^Nd#%aGSTTEDmF3HZVRm6i?*w1MjGzmRfAPGALnTQ~cgA;uA zsB0))`dcR+9o~kO@liY^&qQlgIYFO?qkN%oMUs%3t5S&-#aX8l=bB#<@fZX`3XL3f zQueUa+{CI^_FdSkr_Xn&Jv>&9jm+2u7P8dMF98X599b|0ai^}pzA{YcA z1J}X3f&}b=iCuch@Juri2}8RKNoiif-R4wO;)BPYe|)^9+R0eazAnMKL=TAGA=sTZ z=YmtM67jHwo`Q-Qp$nJ*LloBcQ+>lTik8&8^p*5O{cSDo!AeJp0=`~wZLC1PJMPrN= zrB%bWkRWXf9(fKk=xO8MtNJqBrEwo_+$M$*oo4Vm$lYBvUghovRLik7dDG?G>q=+? zP;tzR&${RVxuJK_3liN1QAnU_Q+VUbxuZqKmU~FW?JrntGwNoOqw(>iCuw1=)d999 z>zy&2ZB|s`JmM3Lw(&ivG`hiEPvooRxmcmt(=@dJP=Q}{qHZ%f!&b z$g+`@U@y#EH_PjTE0(x$kjU``c&T?M$W1e-P>XB_;OR~V)HU$}l(Bx4h`S;ujH04{ zUa7o3#aABtUR$~r@`T@Jza18kw~12G-|8OZA2H8Eun)`1x_L9A{L5W53^XMUOsiK~f(ar8isDjxjtR}Db96~CJ=6+C`2<}R zlj1ca#zv~)*R{1OgzWjx|Uv?8UF7?7H=>!K?rnyF5T*Tt?mWZIOxr=dMaOYp(P zEl#463QUFcL5WoQs#qoZBJtsHY}WI=ax~X8i-A6Loo|rG?Q`T7$Y$1v$=6}5LwP4r zJVuUv?Wha-As!&u%`~^3)!rRvSv&Es8YbfjCLU70G-3&wTvh6o>dkP5xvPPTj;G{s zoJ5sZ7_U3jZ!1tqm3Wl+a&IIhwWggh48fm>0m;{BTTKt?bRsO)B^Mj+^&n#du*U~t zu?h+XW1|pd*!9m=0VgHsnmO+P;^eqz-OYSuYD9SF62buN_jg}4tlN@5e@~8@w)C=x zUK|y=aFsG$cPi2&??~?GYCR%^<`k>!8zHtIIylM(6MJ4c9Y)-H0 z6fX)Z0L78Y1j6oj6H<#%D(0dQN=ss>BZZnXyGY9tc7~$cipae)9ZTIU83^WbmS#N) z#G}PcBoh$DaAqC!ZakLD*m>2(ol%zOen;Y>G2<*SRhYWZ$_R~lUj3ATx;xr5k9pa$$S!a;5>+!)wq4yN&))qO=) z;EQ2n!P(%%y1r7(6MD7LO&3?rJz~wbEePTH&|15#gnN$0dIGuNhqWg~O3+cf@%mfraHv1vZ~582A2^_-T)3oiL^rc;9|-!YR`_x z5~74;jubwF3k6$5W{LU~ED_+T*39|sioGa1mubY5^3|3R*Rm7?XP0)K-tC4nHjs)X znlgQ)6G9ZVIF9H=NMZ_YQlZ}+nGZG91L=D!{XX1i;JQ|N(v^#K8 z8Fhj~KNc+0Ri2>TMT!y-#@Ss)Zce9E`-+_HIHiH~-Iz%zLN_J)$r(R3#UY8LNp-(K z`xe@~C`=|*Cc^5Z)SX!sHAN?f+$=pHV!HIOIEPi_G1UmMZ^KPVY#E)|>&u1kc^luS{?_LWFvZGV z!5s`cd6EvSaO-eZrtczdr{t*6O2UYQ`lbaX%DR$h9?r+^h<6j-3Dw< z-&G!*NH!$CDskgghk=v9nn?5qoA)W#g})P4M%M|ufszKg5%w6EXlxl#{r14CZUn*% zE|2{fMf@N$f|O#SH+U9J_c{xg!U+<5ayRa=k7;P|?hgXDI=2!pfr)OJXl51o7om>_ zQs+EauBmI3ZDAY5^QNIya52{wX&)#rKwqwHnvTb{7nspwh4tXVe@|{sb{VSd*9IF? zG4@1j>hUF42>=@%z*Mf_#8&h+p*tJZ6TsggU8{*|KC-T$#NOcAPgz0K4S7Btnng?kT(4v(ys{fv9pz@-ApI&F3v@xPIz!&1 zl#R~LFR~pPuS4Q2*bxAwKj)*&p)G+=w9oo^_kY#J;*c_2xo4F3<7vIzd7)(wR&nIy3&ctD7+WdVT1x=UP+|s0H$Og3Dh|@xG>Tp3O>%is*Gd0O&1VF$^-dU+o zLfkNi1Rn@#PNJir6IvEKgi%q+*by+wmd9T&Ib?a~Co7()rZQCGvMz>U7jo-4JSorB zvliizHj8sl2w06+2Y@K*+cjjlqDSCDqsIKXO#(_fV10z|_d)S!He$r(f-3Bs1a#9J zs4fvO(-xD;Zh(|j3t3kz20QlRMp($VqeV9J6rRX4D)MFX*oIM7V_85FWYN;l0Pn>6 z-U~!Z$Q|4!U@V40C9GjV#a0Zz4Nn8IwZy8A7oJlXG4JUA$SDs={H|mX$HEDZBAYXk z?XI4xy2t<;(h-q^eXwtAgXH6jgC3tj{{;D*yCb>5o=>k zBhYldRnmDRonzK(I^KAIj>Trl?>cZ$&LFIH;CL9)D`O5r4j6v8Xu8DJP$2;t3F9#= zioOEbHY2D9K}gpEV&nnQTT^LC+JaV+Cb_qk=<2n&g40Atsh$;;Yqt6P?Byo32D+Y# zF&a(C{Sh?ru1O8e%W1%9)s`cEd zM2Oi#hRR>b$bsAx$z>K4>b_wIpoQXU%BQ)l_lkX)`XY^-9lI;87h`{`Jq&1iVQ=Oi zz(0{(n%N8?SpGIh(5hYuk-cRD?`7m>5N6-CN&7dX&?S^Kgc%j{;R}vvD}+m^@1$=W zk%Dl=STndN*g)q!SBC%O= z8xkV^rkO>qbyzCtc62ZPeN=X68j?u(rAy54;9 ziZIig**Rb|rY{CWQ^yx-M9HZv<(U#|6e{y%6(vwXvQ2r-xtxhX~#8^ z9K`6ytX^9KQ(m38xwDaI%t{dVw(I(n}025f6#WBbKC zWn}D96eX)+$jFKH8&2ac3kaB3k2G`YYrot*l|g~Y@X@@KF5}eD@3zK1~#bJ=V z|KyQM-^u8MA*B;km7w|TM1}asW7OqkN3grj3ex0T3CIe2%`}~7Bc)@Hdy@aSPu@+U z_41R#;W*CmNuo-YmfB`d(^_iExdCe(vr$bQZwf3Uo++!*rI>&UF}`p~dx`0SY%zN- z11E*C@x>F#d7I=eH=zrJP!|Q6CY(lp);9f~jUfr%bjfghu#1(7YG(M4Be-~yu9uC& zv!{9;H=Wax6#I=k!sNj2;6=);zCLa0O@Ybt$SmmXI5FJ7%ngv;wIShA*=72;h#aIY z^z2uK^j3_`Cpz*Kue2bS%%v$Cne}Qe z;hS@;Q1z1ORQVKM17Z2AGrdN}HJ zF{_rPIxWtb6Nua%=?{@`3JxN35phZj^2@@#l!MOR?sUreh`Y2eoRwk*OTwee!lxky zE3!2D6Ft3|h)}Xvy{<_6R0|i$L}GBE>X8UuO+-7z=J#~W`ZtL$L+5}@#Fr+EQkn2Q z2}!qPIIY{Xa|7+U#0yK7Vr|3Yoh5^q139T!!$kD{{Dz-YqZZbo5(%fL2_l$`7At3GA5rFAK4_`&SdZM;*KC{2~4)TTP8`V2#pRLcaExP3_M}MBf(suETfP^ih$@S82~}Dy43A={S zJJGu!C(G#2OOvz2Ibq1DYoZx2pGbvKW}9J|gCt0kikE9IbgF}UOg5Tno0kp17m!JI zLl4r4%3AP-NW#sM=uF}3WIwE;L&O&N-oZ&O!fRCm9m}BE)I#d*Yv-mHMq~{dxK_>f z=SxZ@X?jqBB~*BWr=R8p6*i-?sC+Ovmy2ErhhWB!6yap4wy6z-VO=i*28OC({*h5i zS4uXzpvc&&6nSi?<79uZI?sEEJ;R$e<4ax?UzoSfT~aTXZ-F@jC+HO1lPve*T#!O{ z)ag=4I~2N#mji=JG&#h)Ix!e8CWoHZ2LiiQ@k=IQ9WJh zbg1IBNnuGo`V_3-MYTvF^FLA1WeB9v61C0Fg$y8HUXn9{=bj+c4tAhK>*2v}4`!b%-jH+lA)013=<+i{o0|(Db%# z$9{6z1|>A%6`yC4-atdk^0Ytcjbp77!=8Gj%v^y=KvSe>q!?-{Xz8w*TrVZPys!R| z31SU1N`6vQ6)80TQ>x*eB^bt%?S zg3M->r}!dAK+8}TQ=|;u3q`^d872FK-bO*?3Gq*@EUPhd`bNUZm`96TUJjS*)la@q zDBvKmG1V@}WQzBl^n(l(~w;dXoZMXkqjn z#&p@!Di+c?Lvs#J9R8&QhXCWCA6y@LXluGY%#bupN2!hG6|Y6B^&;U0XUR^4*r1CM zB|WO4!{tn92wT*Md13^&XJ=CM1lVw9RAL?wntD>_ctY>?1VC!#a)y(P1e^UjTYzkknvXpjXhJVv3wE@9NNI6!9= z9xlWgFcr7r1T9M(a2c_IdkK(|iQR*?bq!}!`ca(2nZ&)=+TSwU2WDs2{J6fqzrJ($ zlXJo_nH-`jh+y|Mo6<(7H5 z{v%dS`tOacy+iZki>)2Ai{1aYeXwN?4%e~4_Kx{+fBSHI=X>gKV|VYT{q65x97Zp8 zUv6&gQ-!Jp7@ZoLz4iUW?X3fx#}C_^TmGC(ZT$eIT{Ay!AHLXqeP|~i?LLQ2elmaE z-q~!Lt!?UP>%aH*w+;?q0?_yNE12;X{J6cd@$&U1%%y3bK@U5-hvwxrtPEZ|+-*iU zn!KyM<7A-!S6llVFW~F?v+bAL052@7^?dtq2ZkWnt@F$`UcX%5kM>^g@9iFJwGD$1 zIst(1Zy)^CzzPJg|Mhy^b_CFcK3=WwY;56J-kKxWCS0!hY4-pBk;r0((O?(rEIe7hQOW=KQ2%SdjFJGFStqqv(`uG1H`~+bNz7L(2U{VXIuF8 z{?-lviq6>j#>VS?IB)m{wt(3jyoM9Fy~DeK>!l;Uy}udhAtStfzP|nP^?niYaO7P8 zA$Ca!=_1M0W1?HpvHj+PaE<%=q*&Om@J*E1X$a z{F{Xek11S=U4$cK36pbHfKD)l3(Nn^ph{2ixybC8j;YhjI1&^CTRu>Yggio>)={4I zFn*pACo}tjJbHST_Pn`Q=uA8w%sG+Q8y3Y%)>$FZFVt1}XbuIr1ue=>#R%HUYXF-y>J|qjAyQ;+M&LAbH#X=(MIH5Sl z9B%GR>@SnN5L?<5%Sb-v97gE4cS;&{>jo-zv2+}TrL&nGqH>$48s&NtVIf|vtZMB` zDjRE&H1ZK^qEmG&%0S1yYUq^Z{M9FLoAl6me2h!O8Qa!$T)T zMa+o!FVSXtSq;Oa#MBdYA>~1d_txxhB>vPaPg6r-S=1tCn2}J&`UTCNb3C!RU8$5U z@}`1Z9}%3?UUR&8>z~gloJhJp8l}lGT2Nw}oJbVp?EphSyuVAyN~By?eWPJMVs^Sq z7=gOTG<@%p$32vKt?ITBZ1;+-Hy&(qI#zj5Ob#43RBch5$tA~mAYVwp&vYKXsbWk? zwcz|HIQ*y{hMP9Zl1;lB0B)VOcGW4G7c@wRq;cl#VLLf1Ey|!x=sdF8xZ zWDb@r4g#{H=R;y1pN_}Fm4$`#^Yiw}VA9S;CkyKCvG4>Yw2rQO7=Cp+~M_G02130eIJ9U+;9ye^e2#56wACT3`oneR{fPkB zQKxizF+@cp848W9KdAGgIoQ^Md0MC5k8uuD!A{Y18nb&$EQmE}J8G#zM|}c5=CDwS zc2(~6?_Y1byvI_&fhIH|ErY26s{|NvlzpgKk4c#_d6}YyQ?_^zD?+kxm9enkLT8!9v&I4>FgDhWZn!P zh&*kNHP+mQ;~8CfJXvQrgT`285Bs?JX~KX!`1`{-!-lRRwf>NMQ4$_Sj?p!2l%WMP zk%C?q5@yE;P&s1*3@H7P`Ivbu|ES#TSv;fSl z-pR1nJ{|XauseVH*ZSn~AWOTgrS|>y(w`Uhx7IgbZMFN|uR6ved>%f$PyfQF@bA*Y zhf9lpTDrIVXmRPm@}md%eChtZ`;Y##_*G{y-S$WdUaRp_gHLN_oQ``*`0UB!h2fJZDh=-s4d?LeUTbOj(T4|1-1T|d9iOhj z69aNH(DbzjOS&Ky6rLRL7)0EL8IX&MR(C|Acs_`SaLu57V-TzPJIlB-*Z+Hdz=6M% zmJ-P&rC|xv*E2Ufn0zpQ{oxhf$e3~<&qhtl%u~L!u@&-&-b$C|acoXU$#LaM$z=+* zqzkY@H$nj5yQ5w_cwc*>ndQg{ChY?u9>=}M3-J?}Bfzh`q5d#Aj02MxPufSx!v8zT z&R_04%O{JA-SOIc@OMMzqIq=k56zFkB>&2aCWT* zD}-nHWcZ}MpJ3oLnz!)H6>5kV4K{8YVX7$LkkKtzv5KuiL&T?3m^_x5<0w>d zic4(m?b8V$pi63hxcllQUbV;R3CqK9CoUx8jsRT#Y2zE1kVj@H;W%U}MO3eKlj9g; zfNIGqn4%2fKjYNIK@nLOgRINd>5h};x0poO)_5?u>M&-Zy9TL3Xc7n`ETlm&(iLk&o;dTi1>N+_Zm`&f*H>b8IYUGgeXB<> zGPg*$B^8V;U2ZS$3?gM|3T5=ShCGpm6Lf!1J`6dUXFvhXGyu`IZaRvhTL;!$O$5p} ztNJ7mCHSrhDBq@#EW{R%M(_}bJ}o;T80S|kM#>UVW3;4HtjpAQG7VyN^dBcNUMaQ%a1=!p8ysrI~8dhKmgiH{P_qAz~Aa;K{y-Huk6 z1;6%NPXWLELaYW&#^YNF{m@}#TWE#!oCh!29FKhFCL^biE$)%5kAibroK{s=M_gNu z`aNj{wCk7S-0|5xcKHF~2&`h;YR_zPa3sK-iC;z$^|1~x1b~Q(dFb52G(Y)}TKEGW zl#G$SOgI9HoXtlgbd4bOqZ{A|iwTDVc_)5!ZpRNnvCB z{S+<&47!Gn*!_;2@mx=^pt6+Ym_r#t6@)&+R%FxkRsXg;>l1T78M+5$7DA_!IFCWf zv=+|G)Z_QD65EDz?WOARbZo?|#RKvqIb>{Q<$lQGF>Au8BzTHhz{Uv{uUNTy0mDgt z+PdZ{?7VqSI0CDcbv%<%Nt}T>ICx>kSgp8Mo^o-HpAGXN%39%+t0u`q*jW}~Sut0@ zYe8gJv&8Y+KIYX6XlOS#sAb4plX86xO(+i3^=LxQjQkG;0QY{$h7<}gN~Bl^-Kb}* z97PIpgtSd^0ZqnQFPv_7>l`mQqhu95(%R8B9}{m$n6+POHV(QA8fw8oX++({)!Ek5 zAa7if&&2^+En^9#8=xQn5#i9hL;L}&^z2Dznh*-Fc1lD&sVkixh4>)TFq%-^XCL5X zprI+2r>AMSNmMybs7jz?J!LA>c+a1?zqd+9#XVR?k&UFQOkDoNLdkfA$$<#xh$R0E zly5AKyTuN@pFXqK)(B|5#mRuPA^2*7kOJ;m#+UKSJoYe+3_VYvs||{k7)wAWSKDUT zteBHu(qYpK)01CX$1!QK@y{HD2SQy_m)&oe|G=vbE?2S3-N-x6S*P!eR#H6y{)-r>*$;-BtDixiJwkB^*_CTQrLqd7=V;_QcqF+7$?JE9*#@E ztjbzxMUJ81oHd7#>Ol{;O}Nf{V_`fwy2yr&>;kwySU~Hmtbn_lxdKJ)ueE|f){^VC z1uRzQJu<(WoU?$wixWngTL)NcIXsd&vz&j=`Zo(g)A=AzpCspG4`N3ODWEJ?jCTW) zKSLye88V?$XabNpDJE(Fa!al_=+d-OpZ&-=?7g)u<49V69~rD(7DbfYiDT|!(i(Iw z54VsuSu&A5teqD~AIJ|MCFAoX0cN_see6p9kVuzOQUc|gYz`kfCEZLklmQi``4lIq zY%J<^c?PJE9K~as?uxA9X;6`@hv!r)aA2Y}4n%Q%AQa^a<^few!_9yxldN?0erTo| ztwTVO*22z_UG%Ue2l4Eknl3tr&SbqJXYr-L|RSt!wNFYS~!-}v$o*0l~z>F z$W7RJ zk6x*FDIYCx|4&axaXwyL!i0R9pXfOKKu)%z|7&HUyjog2i+fG}dl~?^aI2mLk&l`bxau;I`6TtK}6G>w%ViLM8w_4^u)4n-~dl>%l_ym?RIePl{QQpeY zY3zwHKcYVCJF`AHG0TffkId5Y%JQFA?tf=mi}x27Z#a?P_ft*(F>Y@rwfn5k)%^c* zk^Vn;uy{lNe~-_1HwW(D`ZMMHQ!3%FHO5uv|Ivez{Quz5^7763{~bP@_;p42fJma@ zSAcoKET@BL4|CL|<__B<*!un+ZJY<|_t} z>on=1qPYHK&3xC04n*OcwzUDlb805+w+0K?=oA}|%| z^Y=q1Z+)MRuls%*ex^3hM%j6e>$v7P(O%qnPZN^|>!Z$TItHLkFoGA$-`eHQee9=s zXGOkWHsJxRz2`mn#$;-rk%K4WFxvzs`SDgRl2g?sj^563Phd1Z{clbOfV~RTf805iNK*4a`tg|pA zoCiUp#ftOX;YW*YF$+uW#f9kgkoE7aow)B&6d^E=uXn_rMA37^`4w|FvJt&2EGf(s zEHwGt3IrAwNcH-!^+ascM*K?AghFEhA1CPPZ55IKz6gz(`PzVbmpX zlo0gbiW6I^qRW-UBT}o4Y~PfF1p9A$gJTUyGfX@0IRZ%PTDbDo&b_(Mc0DP$!=+le zvgU0?W}5($s{Ej7lmhn{-1A-q2ual{>EtN+ix9wnrXbtVin_CBmo7?;F z+FX6(^*;RBP-GQ81Kk2fX*YS-J$;8>uJ6Qq4!V5We-8x6w1&`FI%+U^gJrLpH_@$I zouT@WsN(@v^QG^-BpF`x;bQwx*?da(E+DBeKp1K{rOB*06G=wCth)PRuT^b=5K>A0 zJZ(%6M;2ro{+oO%>A&o#WP{+V?^ld*mHubx-u)u|fAnzq z#{c8@_#9^JQQBlKmfeGiAO!K3z!6qbl0xuhKv-@L`#-^F*7=8R{wiGHRp)a5lg;j{xq;((_)P!r2M=%jzkip{to>)> z?yn!?YWeR$iT}fAH}wCv|J-XYnmUH~pCx}>5ZcY>5Br4YUyL4Kd5qcge`yK0y;A(o z{d@Or&j0W5@$7%(WrkTNDT9N0kWj`+E;UJJKn!Jft}4^KP3CiSkuhgTUTiJ&7-J{K zBdmVXg>fhveO4~?lVJ~?Y$)?K8WNWQFIr1WEiB|?)>&cgr-={h)Jo-CH0*PU0Gv|F zIn;8DR=B{F1J`p(S~%)uM+^NJxZBaf%k7P=or5h*!iBJ4UxHQo8-Rd8_4#TP^)JjE zh8M4yc@|>f%ls-nz5~oJK0m?i=qGvo>H7ZnKQyEXx`e1x1~kQE`3Ra}8vFVjzW#Au zexAQ(Dhx-{>Ti`>lv78%{5C8YU~ z^8c=JVjldv?sX!6xvpStwqc(Bf(}W-Dxi!*P7bH0=9DhX_&T|H{U;lGjZ$EB)xpvXuo109^=*2F_s@Z__E%) z!qogOl)4=f5J^EjhFZ_>=@Md9QL3*`E9KZWS zgu>hEJWV?OGz6L&9!5R zj5&?rX)0AIXnQFkBGEIrI5C%q=ETqfy!AXKD{MFEPKLmnGZf%&NxQd(6zA5hO6prd zLT~Z8oKE&k@()8`F#qu%W)@wg7wK>5_*O8hC7cx?EdM_a`i(bjY2x*-P;?Vn<9V!3 ze^F<$CKFlZ(Y#aQA(tEyi|zJ^IHX6E(Iav-#(yTtTZCtfh8HnN0v&DYw4-oLD-gPd z*Vp+e+TQuKbhK7S48exbsKJn=O~prp1SUF9duPA4=ZQr?jO4MK^doWw2rjA>DBZ0( z(CIWCaZ&EU-T@)Ja5?7r3tR?rJmja@dVyu&CWA56W;35^LVIiq?ltrH`HLs3G7D&~ zKja;MyIS}+d27`@q^=c}Y$t$AYYRfhZc|9Omlun2 zNntaoe6{g%+oc90m&P@2=Osy_$5eC!I*KvnVV(@*5hW&bb-H>NZ6b&|FPyPV_tuhf4Qt!(f_nsZcW{6f=Na3&Vt{=0q{lcUr4*#X!sAwB;7PI zwXOaA-Tf8l3hy?O7b(4%1BQ&=CUND`UQqsqN{4*M5Mx4*+$H09C<3Y6uL5(*0)R== zsD6dpB8CafMVg%_GE2RI$u%7dz*9CoO81JJ+@Iu}pHbSm9%%z(-n=GA&XgfnqaX(8 zice!oJDZ0PqZuwxD5hESwERqe^ozElZWMTg3Aa=xYmYHg z|GTube1G{~N&mZi6aVqMd~Wo=U#|b1TUn8WiK?qz`ZLOh)78TGZG+sZS5~$MDaEI* zn%h*25(@ww#jSjNA->n-c}-0=K@M11*_`xYi&jbLblVWaj8VC!C0bx9%14o-sBoaB zIK;}z9!jX9uewd!IO>ma>ojyHrD|vID5Vdz`#u?G!|}?>OAJ~3$9a84yum57#%4>! zFBm1%;HB+l-IbNsd3=(%!G>9Pd4iAV4fGztPCIdLRo}qsq46lE1n->YbA)k4Se3(5 zm)C2yY*5XtJux-UvkZl0n6PA$ufMs&QbkRwd(@g(4GqABf1nQ;{)-?-jSc=yxoT_8 zs><`Vu#gMB?Uz2G9(7)8`4xTGgihEjEK!6{3Uza8USCwx)`GIWj5?>=7&l5vC)pc~ zvIG$w-X3I>n6`nq07nDU1Snh5)M5b$dv>eyYM~(Wz#1e1{bRFCoR(bi`0~6mTFxYa zE%mV&9fo8vIH78MHX zoC0k;JbYICQ_<`Tq8yTw5P84IV9X^=W?zZVANSw7iIA*yXkFh{voTqrr5MlSK zR$=})$$1TXhm_7G2c{xw*{I9yNPCb^j&N>IQKaq@B`HtD{=rN1zS2?}p#_e06_<@$ zw?3NU30PZNMOpp`^N81FOyaM-_9b$_wmf___N=f$^p zj#+dKM!^nb6nfwGU7tgg=%9(o%=PmVToD!Act>T~>eRHk+_2h2ece29%cA3=Ur>nF zmsma*6*u&by^tccZXqPYu3Dw_Q{g(PGa`HQ^Ue9YckfaT%I%%+4gM{UHukpQ>b z@-wxZOW!z?ptGOtB;T60`FjSmw?5y5zU*xH5VHwgo{_Qq79AYh9Cq>Y<_P&m$dMd0 zw%j%YE(bX5O@P%m$C9jv$xm~sx!k-*-2Coeo3yP>yOWfclQeB@>WGn1yMITweanoo za0pcy%cWL_6yE@$7;udQg#_zJ4`36q8_ss&%3WZdD_1d~$%ta=uGTV;YZx-=Ht-@D z;km2$E&6p7gX9L9Fx_CZxO&6UjLe>>MaAs=L@7*uqQnOKEgPeydbFH+o}p)Z)89!? zs`*k;T}o;>KDeR|<+dNS$)&x>`OXR7uXww%0BzV-Y!_;ntmuOjbe4Ybnwx6E6gr9E zW{SbZet}3xC1CiA$^^r|T#uJ%MF2438SC6fgRq$Alw&oHWCRjynswEB%{o@B&Oy0o z6RsN}?}&xW!!yF5jQl zSO9uuTZI}nq$9N^kZqg!clcm_)qjOfmj4PqaIa_&C48{`Uz)px3E9TlcXea@bw*=3 z;n2RdWK9DgM5S2z)IZ)I2?eE7Kvb?%H1CxBuBYoXvnp%r0_NJ|w_bd~4rEG#R{eJc zUcrm_EPP7<6TF4b8g`*>{Z9mrpFVjoL7?dRs?WR!;wNTN-@THcyz2svG7elSQ{jl< zva0)q6RDbx(g!4uO;f{LWku);suMs=WnA-8RHK!o_YHa86b&%bdwT zuy0wfYwDuy19|YV+k=K3GqV3R&Wh!20qx<{0Oteo`2eSLd5Jz4bsyrSKq+s-lHG;? zICzbJCE(C<1vr-#RxvlOX|I{u?q9!XTxRTU5#M>sIZZr87to~pj=Be?n;k-X9gw>k zrUQ31{(01Zjq>kcX++o36|LmDzJ-@Lyvl4Ss?nOk*vbcJmsRi4Zw>3bh#fkc8Fx?6 zLH&;Gc;bpXnm6nblfP-Of$(^g0AQUSj|sZvhx3yHFPteie8UVgH(V6IKSqJbZgmNs z2lV#N^W7C{7*wNcx2+ZV3pvbdu$a}Fy5EtHL_Ivg><{21^+k`btKjjKrsny+kOV?o%E80DL&|ZGv;GyXcSt*(w zQ)NnPbHb|fSh7KCE0Wq2s5tquqS|_Z<V(iod^XNep?hI!1pT#pi#2=#h=sM=`bj*1U&uY@jtQ1+=dI1e4Td`DW@NLl zMTc<)f3af@CjH`=c_;40sMY4Z(w!pF8Hta&x8Ax{a_6pFjA#f)bz2EHd3!V4@swS^ zk-@~BVRW&Gg5Lm9I_&$~G_rrllJK8x%{Feq>^_btf2cxEK*U(pptoF*Z$i zaw;8^rr&B6xMIuo)@38T88rfK*X|8xgUz$Puz+N$#YW!%s2!w9w}b?xKF^l6cXAKU zHypUmfm2VaUs51cV2zui#hhE8#|kh`0UuzTJDGNp=D!^0N0aP06BL3jq^79kjuDQ>+W}8lytIN#bZKrI+_VA*dSmDz zzPw;l`}pO0+h4&|lk<3TF1O7!1p}D*G`Zk+<ly?%~zP-sq7AKw)|szFgNUGah=-(DMwx`LM#k*cIiab246zMth| zv7#9JhNETlE~+FDcunLunsX?hV8Gg-wbYzLX`msdACKYkhFZ1o?Zx@n{~bdSd>D&x}_xq+~nFX_kp*c`EosY8(kGz zOATDw-?5YhEH6c20pv8Zc)byEH|&TZya99Yb;5{x*hRzROAC#ji1b0yUieeX38m;r zXP-05Pyf?T7C((=%804%k;MY)Trb1jaOk1&anc4@NvjMjd1fphMjcA2JbD(XdtDeP)Bq< zjtQ&P4oou4fT<;TQMFyHTslg$$W9`^gS3tV`5kL_Wqlw6rZttB^4M6?5pg?Dd*Z6= z(tx5CFvBd*Qw&Y8`^pWbi$UC-2r{T>^y&|t##nl&V4p*+1W5xrD^-NYC)sMSd{mpc zW$0^Juj`4(thZ8G3*)<4O{ap)J0C+)zlVVr^Ox^vQniL}cyy*gR?V>r%5~oGG*IPS z5KiDzFkd0`6~sO6Ck!Y2rb^R7GY+*a+6P(;shb*WTo^=F?P5LXB)XBj+{5wBG4L|z ze2Whoo^OADVE8K^QhtM**VSVix9N7+ z_F!jy@8HGmVc7b1p*bJKDXpLVbokTW*46EK!P9%$TR(hponAycZlORgUt3voZ6`SH z{JHgDbUrRCLI>p)l9&BmROQ0FD&FTWeOn8iRlkEeQ`;4zxoL#_R^$w-97S{&W&ti| z`cK2_T>C-}hp2c%;5AF_Q`VHI;>TZeu<%N)id#QD()**-34Xdsp2BUqGSz@JvvmN% znJ}zhM-Sw|vf&R}f@afT;+gBHrSQFZUBso^pf{W>3~xYHF$)E`Q7zjPCVte>6uKJt z{d(`Ol9O`^yeP+2jk3wfDaZfxv+krvEDlkNmP7@~*A&wLa8d(ugvL$%%4CLLyn~6o z)I#d_Pf4u`r?PM!?crT38bu``68Ng(b6*?R?T{%KqKTcx10p=NN)5x~e?t_DM3Kl$ktj~5iR#b)f0ZUWyqd;KRh;Pp zXD4R(jt`k`m?=|Lsnw$io=?gY`fPu)XIXiQ3L;iLpDA=q1}M`p`C%Fzlhs^F z$8-WA>8WOCrx3C7;v3%T=e+S|)1Jz~J+d_y!m)x*$w?~HDLNI=4W$YQRR!NRBi=mB zxkx`yW`MJJv_Rh^a^IQl#~(E3=u>wbHJ@X024i5ZPs7PRjb!^YlWo_LC-Noy9Bs6h3`2-dr}2&tE#rC1y1zs@W)!i>7{b`oqwvr=ri# z=*qGqg=CZHSJ`?obn0A3+dpwzizhDSFI@QK`$XeDg+P zuy|q5U-%dq3~{T{yQ;KjKDe&s)8ZVX7@UHB>J$|`3_PRa!BpQUQvmW~1;}+^$*i7R z2Kn$7lP0)TY;HY!{k`G>`zSL$)N7_)ac0`4<}0dM6H0kAj@@Z$tWb)zK>JG>NeQ#) zEw|7q>#SO)x5{EvIP0jNhN25mq~pA4Z!-Q%sUGVm(5I%qP;cLDym<=$?Oin9Tk|*4htR$@!whaA65Ko zI>1|cYai#8clMfH^XkmkS$tiy^YfM1r7QU+Oh)cpOimLdTi_fu6=`t-34t$W#3}Xf zY9&>vbZoxH=j(^-FY%lX@R0VhxLfsl%M5!-oTGn!j`2|-LUelQ0gq|AnrtCVb2{ge zcQNUvWHF+=JKyk;^rv*qsZ>mXju6g(DIu9ggL1}42_9fGnhZExhzQ{~5~wwXk;pqy z(gii~76@~8%{SrZZ*j(i$o|`w|#nn^3?o_`fbV(jPPv(0)qFkK6N1at* zZKFPP_)B7j8Rq#(Da->)$x*WO;~1|M%14=FGB}QHI6)Z|MHS`=`2okSTX)N*7a!HE zh}djagw5J#@Yg8mOh$QnhOMieGR0-}X^KYeWUao`tl=NEX6;b^*?+wS|6C+F{P!pR z@74kQ7ju!-cIiJmHP;O-cCEhHtkGZa^K<=o{pA66HOSyUJM^Dj?C1Cx`{4gRe_m@` zVJrb05y#Bi93+J%#}=UhU8|=VYzJQ7lfnBzc0Mqa|0aR0!rTuDMP8-639L)V5sTC# zmZFX{J8HWip27sIOb^_8>eK_5o^kgZqc>Syj|(RHkpbSxqj|VTpsP#kx$p2;4YAwh zM6+Z)O~%A<(sVAe3Gi9Es5d|BjNZJ(@9qq=U9799OQ(AUzw;esF<&3eLh#3f6en%# zU3fJFa2~p5M0M#uXN?3}|NWe_l-dpNf|O1VveQH^;P_0wEHNp&Me)Lk}nuVoNp|a}jyT%GIagnB&SjBMdKVN9AoTYlpm5%-cP^ z4i9M?;tbwM&u?tkZ!Q_DHS3*F5t?9`7<-KHZGSI__qLXXQC-cm^+PABFCv+R7?k_X4ZiS#Jq0+bl4IxQ(ygL_wQ3c#TH!nONDU` zPZ_18zw#^w5*shtVjt~-4@SDm6eVm@Lg~4Q1PDk>G!qy|LVg__+{&+mYqA(iJm7#@ z3gFgvjpeodfTS59pR<Xj>c3H39CZP&^#=QVyyWZ$QMtCJS+ln37cwC!b z*Y%Vr{X6XrlVb2Jn?Ucn6IxKSf1$rOin2;@3w)gS`?6XIy58-4T^28Mw`uOqH?QcU zWat5D%}%%OI#xuuO$l6wzt zZNDk+-t5~^-*p)MSt$J*Beh-RTqRmxh1YI%K023!vnsF}c@H9^a$^dX2V~Tw<2>md zU!Map0SofG6AgO{3o9a*fd|p6;2Az%EkMw@o3U{NnQMU?a&^_rHvYm7sHTPGySt8= z2+XiGDI^-PjS3U^BEL=D)V+NdgSr&p(w;WvEmSFyJam?$h!RM zk)dh=RVM5eeRv2FK(OM?KvXbc*)jM8YD3j{sSQt+XE$m@G2L-G$d84fT!3g00s|X5 ze;GzvVJJpL^j<|cRCLBa>F7FXrQZF@@xt%dHPp1l|>duC9qsMECa9DHk&fza&Q(vZmCghKgh&l zM7`9?JK8D-d6j4NuDr;V63A-BZL$Fo#gV;9t(5i*yHUdB9mN})2U9CA+oEwbSZ`Bm z0u$S4Y5{Q52q;rxrzy~gifypwJR9|G7o5advO%g}g7@Pbr4hoC0Anwyp!Fbz|G=SN zwJq!$tu<+i$6Noie1A3Qn>~%Ec0R3NZe}|U`;>wJmAy{uP}`JyQ^PCu=BLdJmVxk9 z@VcM4NF~64_}<_pN<7E_LZk0|cOH3+pdl|36KBrsf?lIOPRo7hy0DjP_9soK$9~Xb zq#3M}rf&pweI4*FZ#;|W!24O$iO1Dk*69ARQ%sMo=^2rqJ(vv0 zeUaKy_Uesncri*(PT@Fo8fJNM@ew|Sx1n|1Ge3^f@tBpiJ29oux+8 zdo_?mWqZIr6vt>CInPG#vGISVeRz%~giIZ47JL-1_^V;l9jzBas zFS+cIynBdRIIQXp9izB+jMaLyDAOuhEJ))#O1daUr$>ORR6K*#%l)jI9$&Egr#BkR zs4rQ-c~~AaeaS( zedq9}Rco_I&ZLMRR$8WNeOPyIFus84V-K&k_BUQYll5oYFSieWLJQaP?Zcg|g9Gz? zci*g=z4iUW?Tyzj*Z0ld>;1jmgRQnP2T8)k`EJhwG2`k8>pPF*w3q9`e}awm!#pK* z6vlc|y{juXk8rhzRhuMI&;fFl6aI~x&X=jQt=Qb{Y}0HHI_;);@SQnKs2ufPk181+ zOt6D{_ZFMxS(cCS%~$KjEG{o$eXe_pkBoVJuucZwA9!b1%wqey_A)-%L}_CMEz ze@;LG)sx7(&W9N_7s&l!me~Q#@!7$qdDLEPZM@#!-`Y7OMb$ypi$~CCX%XB1$ki5) zQi_B<27-*b+a@ANM|0yQABND7Roxhfd@?|jGdo1>9*_bNyE>kXQB77)+9z!U+w<24 zTL9w068wJa z|LyJWzQji;#l!yZ?!4UI*)kswVDT&W+TT6;9tKUiD@%XA_gAxyh1}%nMm$_uyn`1a z38-Y)Jy@&HZ*J{v?Q8-n%&$GU-<*HF^Vgl-A9vu(z2^MxU*X@y=DZZUfG)ueH4)fNa?t36-ee))QTtELXBxRf=jn<;Mz9Q2uV2+LWkX-7biy6zj5 zr*mIB@z8x?=)3RT{)yKQm4T0p4RgfrrLZmc6-%do25Bnze zY|cc?aQ(;-xmEzny3W5PX5N2i-d3w)mWk4kkCJ@S8-HVVb`Q5!0s?|bVskjsCZ7z4=mU== z7x;lDojA#ds_J*F?uaU~GdN)<3Hd@cu7C{kA(PH~zKeiu1Ns5fCb64|57x|MT$wt|^mwMHF zyWkSnTfhAJ9BYl$5gK<+k+3P4N^%f%=0))UkoG52TQjQK3{ z0l=?#pjV_{{Emm3M#V2LXxJ4UUxHW3=dzBACwG?H!ub)5*!RdoLqLRKMmHerdW{l8b{XjwC#mxMT6SXxYJ}m`%p`Rg20z z8elu-8`s)BgjMwdT50W`+gRHXIWLt< zT4~+8f2B}Vr<*S}+}!$pe|>Xnvu4Baw5gEOT1mNI4b)4!By`mF_#M2{T|NxDjp%xt z5C;NBr5B&z4ULfKjIB+>!a=UHba0mS&d41WxJub{CDnM`xKh+~3MTz1nYt#Gva5AB z%IZHE@w$yHgN95|t_oogCgb6R>OvB=VkjNwXa*cu)?)@?vXtpfF0^TZtYUUU*0jci zd$ddhiCm==tvIy8Yh+5T7Ok;}2HaVHwFP7lXN3RmZ>?{_KR?3$;Ma}69=v|FQ0ikT z=)(X#UASr>`y8`?ENgbd+fwu!Jt1R+))3gQI3@o_ge6w}*7};a9EfsER+C$IYO3Td zW;}bL!!Y;63_>gc{TH3#e?9>DW9#7UN=pLbEeLO2`h$FYa7p95x%NQpEIy8r{x3+| zZ|iRkzHKbbn90iAM`&|-o%xKCA;=K`oX?sQrr_>W%n4Ycr{q@fJ55V*)-M^*Wu_OEJ9Rt`7l!vp%!(gF2uuP>lKfn$HAv;gfQJ1TS z>G0m(x_UG2>2leD>^2fh(fso59||B!K8)W#Qkqq1&d!(GgY2u7g33KEPw8=4q2=J$ zSe^B5-GWvxW8~HK83Q|bw7mm@X06e%o1LQ}9P1%pkf=wP8l0wkmy8~E&q54+j4DaQ zE&Q@VUvR8820Mu}s}fx5?$PY->QlRG_}wLytt;DUvw&Sdw;o(ARn*NuF8(Hprm$#t za*!qAKP*SWAE+z9f04sP_U%sY7&D*6)d)c3NG$`5GhMYq=~a$jJ;O|qb!$rPMKv%7 zI?LZija#8Ey&2~J?l4Z3O_6w{#%7b16jC&E_ydY$stKf;@DvI=nhX})&(r~VVwqt| z$}x7}Cs7iawAl0~RV%A z%FkGQRTZ(-rSlZHfB;+rQ~Gji#dEcSf5K-Gr=mj*fk8s};L2F&V=pwmL1i7%a7ikf z6xzzK%N5-=NlknX6F+BFF#ax+0?IqW3l%-;$?FZ>*3_N|5?_ZrQO^@{ki%#l3S+?z zR$&j=k*xT-OvQdke~k`KM_KoR38(tplLAy`WF%~1#E~^rYaD}g2!1T_C|r?1dMYY> ztd-HHY=Wnr!LTcB(U2HUfhQG5zoisVtK;6bzp-$fw;68{iEz$>28d4SxUFcGV!dya z*xVRZwY^%v(IO=&B(_qS^-7=|EvxJQ@}Yp9T@0I#toBJtfH65KliQSRu#L zUjvS7Sf|$;9$0bc@TW-qziivp;1RU2`a9dhnwgtZSQikHq7iTdwBQjRUPQ*@$q0ee zc3ZM6TS80Tdr9dU^G+YI*Nu+KI`j-%)rt$_@}sTY8%4Xt{+it`?$>i=+dOICjS5 zqosv0oG?yDJEDa5>EOLXl+_}_Uos(g;oR4vETxPDCJ}FfIw; z+R0p?Ay1NTm6q8=h?r4!0(4m^1Y2iE8uyN&jY%&d_Yufjfno*uYB3h+uO2e#Bj1}`;azuK z=(T#S%o_>-?MFLw%Q1xyYe407bWBI!+^r)rd=ePpzh{m1U4SJi5sF>7JyI@DVeJrW zp@>n)^CQDJh9xCqA7fj=9X@@q>j0z(2QG~W`y#`_lD?S&82vbQq4WS)u&I8$97QK9bIGmW9s-rSHi7((UMyFaau)+Z0UpN9smb>3{YRLC9Wc;%+y^ zCB!|rF1d^YLbW>?G3O_61P+pV(~SFmZwG0A!jZk2{%C>tb}{;f)A`3F1%eoU4&eNZ zm~alVL5mIJY|NxKex@@E3Dnu}5_pPm7X@kiDJ2v`Qv6C=VkM`~eu77q(4Xz3HU72M zBnC>oQ>iA!I8~4HcpP--wjt=y&f#g_b?4T~r!xWkpJ;CJ9&$BJm=Kw9~7V*N`8lRjq7#uh}fs=FuD1aW~V)@R`8pZ=Xb zyzbUg`+j@r&kNq}eZ?^r;q&m}efk$Z#eeTFJ$mq`rF+Yd7MC6@KYD=A;g6+9e_H&C zbC~*>VCE7S5}hb!y?52`#l`)Te6GjdMB9#jJ1{3GuvgGXmLUu@?Mf?X>!An6kH9`3!Ss7Ax14jj2Vf$P0P&_ zX`}=iBn*=Wdy3?gK-n~2c8`7rw-8mmlbisW^}rRWJ11ttadn)i%fz zP$~DNWy6&ti8Wa=QnJlmitHtcaRCyX9Nm%zYtR_jNofGz5Q}t$pR2LHSNlPy9&Zu7mBt2uO5dxz7e$u2I5S97H z+Nk`gc%_}wb|%2KMq{I6+qRu_Y}@MCwr$(CojbDsOi<0xwy=5 z3y>VbbH9_AQg|>33;7vFDhYh02oavcSvCL39Won)#e|`x$eDAqR}nvK3vrYN?tsJd zjfDdbVjspksPSR@u8##o0{eKRI#+4_=t*4Ya6AuP5)^y4Y|>=Bau%O>_ss+(YSqPs z-2_(UrpQ@%gBe(f&(8wy2gn*51%mH=65ve$o)9*A!mhu(qP`2dFt!gi+(KbYEy(%# zNZIl7h^uiTDn2*wd3NffY}alA4eyS;4hu;lI=|yL@Q}tGYUu=&KLxKbEQ}S=7cnFA z$9`ugW%lcEM~iIj1q(aeqK!+S(0ay?>ja;gAEZ{wlbK-(7r|>|+qbGHMGKeCp_%<= z(m8B4w5pyMjz{JN4AwA{~9;f?xn~i$LiV5NA zDi>T?z5dZ87~=^i+8HUK0BvW1#esxn=-W>X^zZ$epPHR6%lug-wbPgU)gJic*WTW( zarCo)^TF}?ud%!XpXOUL@645I84ugaZGL*&+Qyb3a}Z+_^5pG#lCG(v2hig1dSBL> z@z&2z!@oITRq|#Lf5a@awpQKEQ1%q@jq%+g!Pi3<&fJ@SCGR7Z2e@DH4OjA{ zC|;0!fuV%(`G)wwPuUNRhCh94(V@_Hs0Jxh&{(kI8j-1Z z)s&-Z-a+u!;de+#u2Rr7k43&j3^b!a_?C?;7#;T~g`(F&@u2lVy$_g2EyY=FmyMwA z!EVu5guy#TcC6DT9S;jkPPiep9daNT3oA3eXTH3bwg4awm_}dy3BFTBar~QfroS`< zVXBWeNJC>S9?sfT?j71h<5#L4;%2Ms3v?ks>-P#dPKqBf%F+NfD1 z(}fAJCK@CSG#DN$X(hE{&XLH-w?*XkvR&984EMKu>|JvP(fwE4os;+O;Mn2&`}rOo z_Z}$bjv~|nHU#8gwqNqG(M$Q4@kn_+8VIG4h6J{i5uWjgNTrShmYCywr~ICfvv&Fy z_jTbWN6Qmq=c1(OdTa)pGhAepvWF zAzaaB$eMn3f^LVcb<}&y0FRU@*jdZ)z0MiNn$nqOEV}J`hH}P`YSgmYRXes|E|QSO zXvtY?C74!yY=InB9(16Aq5XjWW}pY%R_C!74Pq~g3$+igOd}E3GaOSXZR%>dC2NP~ zo9(miR`vAi$0pA%3n4HEPPDIzI4;+z*L@D8@2#%$rX{r&KQ$!`9vKDI2gAfw2N-9m zysnOprq0kPW3;B_Z{Gw@tzP}R+&i7!`l?hB`<{XdjW&+^zE6s5AdYz^|_=`V#9`8K*aEH=~~59=_=$BvGrx zS)7dO7LbU7Xn2C9GHkSFWR>=hRaXbvfP>T6km-0weKrl}O=PBSTa6^C$2c((y2Ipt ziGM=RrYKkcoD9^k#rh@8riPb|1Qgr5oytHus2)`V|4~ugSSex;RH&m#N30Eu)FR^R zc8d2IP`90_RNCtQCUPk!hW*ta+y0+!Feq4_KJoPNhk=XRvMeZd12;@@nYI z60!MD~bm8cv8~-=BlaQ|1EIDCtO&oXGqj&u9t+_Mf`OD!vIG+~B zV0D@+z1B8T!?0?spmC(_>71uSLBZSc{7aEF^U-<@VvQM_K4<lpOogEEZ;9WNfi!$FkvzDjHRO`@d{-997T z=W|AUJyZIt;#hD6BH1?Y6B+pIn+dprtjq)CX2~k^d;Y)5%Xv?L$zUl~uBe-TmIhcm z1dj9h#yyzJVQ$}YQ!_5p=Qhv1lQRXDvruL1wSPBQ8cYYfXd-D$^{Be76swE!Mlv2O z8rK+&eHl{YP4S8aIvLO2;W}~QYU2YjaU2B$B+B+%>xoTZ1;-p6S7Ej{T~^PH{mZg6 z&3gkVv}Uhi2jM1M5l^3!0bY(@Ul2*Ke6q6~i!vyZoJh&J1H7+1q@cTJ&)LkV+s)C9 z^x_6TuG&*awa}%9{Z}idS5tb9p$b-8Ea3r$2aOS0wB9yn^r;yl=4~ zA}-F?{C}X;GkxqzT)@mLVqqR)`)S)eQGK@+p|>H!{7P7{lYfivN;jRpLY%ZCXS6-} zYBv8i(jo;wpaBcLO@W=GPJ%dS9nWa6ZT{MTUb;DJzs?HefD!&RQNDLMKMuc zoq*WCU{|ucF(`zUf|v~wTP4joaJ|iJZ>@tV2`+HtI{*G{kDvPs-u{R3jv7#>scYM` zd*$BP+3@kKS(6A|54t{)W*qzvY^o`TtGPu&Y1+&9)q2Y(eq^9@9`3qNJX-Srj4NsM zMkOFmG+~NLbG51B$?D@5B|8!9hP_?>88Z_$7&Sw65`Ae(F|X_7&dqe0gyLj3PRl^g z$DD<=m0~e^We(Jhdp$Th)yMnSwSI>Ao=Kk3g4-db=D6H+rAv2G`c^2|)=#E$>+eCD za~Ie)%nj6wl^-SE_i1rh?(e1jeJXp14v%GahQs|iUi=rY+ndEC{_r^=yO)Wc7Di|@ zy9HU8*RTnywWLpVxEI~!KR7C{NUz~d?#aZv{TuVXaAoYHyE!)uOnc&ey6LCE@{b4n zGAE*Nq@VP2Gfbb3CaKr^-S~Q5dT@(2u1icGUUgIduvA32jTqlO{VV18AH3s_NaK9o z&$Ia;i~FC0FY$js>Ce|NB**=KZw+75AZCL+kq^qP%Z+I_yAPttP<`Ql5Rh?cMmRAWqYISMnEzP=gDVPWFtlRk&jcBUF2ExSw#sp5-k~Q=j_S2= z2D$}IPLjBQxn`!klCsORZcpc+wZa6KiiXo~ZXXx~;bBknxQgp|N2VSC&E7ZtgCD~W zem~zuSr0#6-g}?&_ek?VONg4-`4uPd9`(pse;s$bqVE@#24+vo(OJx2HljAVBTikD zfzSeif=4rw>C0cVG&MK{1nlzQ$p2b9Hxk~Q!cq+C!E*n}1IY?x$CxVF9r!$m+bG{D z@;*bh7R=oik0>zFE3QIXb#lxb`_6S{Jf3=?diyl{7ijOSA@ia&(HT;t=h_UEjl!$@ z%Mg^Ze`_Lgg3`(?H}2Pn32`{1u8(=QlwVaqG{6RAUR!e=OpgKUF6k!8fgQK5y8HLR zbKJvw?{YoulT0!ZGIAhe3IgA_(!3*03bx1Hn%Ny#>+zBNjvGF`4-n7 zL?KyX;vFYbW^oBZQ+lDc^|iFJUnHg*W$b|D@q6T~pUtxX)=Q*Q`11g!HpX(gvi?Sk zljxc_n^(ZNeGPy_ZdKuJ=7rjLc4*VyuTo@_{U55HS#KZae0K1Uz%@eW?jHGDVj zCd08QZ>e(YlkSp65zgl_6)cBSE*Kxv{V?wrAI&DAkvF4nbTHsiBfix^Mwfi5JO&A0CeIRvwWqJoqc!)iOLV+e1RjGgd-f7bV5%jFktPQNhzi_ zGwA5Ujr;c>+)(u_e!cVIa84Jt7t|e!D%FUmpU#g*^C+ZfYzfY+6Q54+qV23MX?N?OSYQ@cX zd&pti`w6$3ukNp${Ks38#wU8&kJ5@i+wx7+PQ#s40KtZ))1v`iK!BzdD_0VQX$#da zZIruw5@Am)#N3b|()L#ElAtsNwD;$Y&yphzzn6JZ9=&sXeZXMhQmATE?;a!e=H{3l z1&bMsUQ;Qs)ln!pBrm1OoQX`^WUx0tC;vbeb2Ywj%_g^&`7;`RgPnO9V|tkYa% zuIaFdruCKQml`469|uFCE|_`Ys%AAxEPFo?X zLj<*3q+10Ki@_|SlO@t@9!IFk7FqjPFe4;+55rf$$oWGeY+TgiJUVUbXmQLXrbnGq z1NH*bP8Y7xEv9pH%y=@@Iev2%EM?+sxGU)pP<|<^DKUgI(5t{@xDp>;u5Dt80rSDW z8}KS^c_q3t-s0aIS4UD7n&UH@lZ^#LJ?6I7p1)%1LtV!jOd4};{c zk`t=!<f=&I3QF!0rl&l@k6t4+NBe4EAP03DeKv!seUVI*Rp;F|mI6sY zd8gPcaW0)ITz~XiyEwFAlf`;;-qTsc2ZBFow!XJ(eOCdKe|Pu0S`Wq^ZJRPPAU37i#38b^vN9DoAdpa#f3o7id^R#EJ?K&s8ge-+Hz1CW zPk6lIbL!mF($iFx$OEg@Ri6bU<*-RPPjsr%!&C3gk`qB3Z7QQoJ~0R$6A4_VU-j!+ zD7RfRwC)XgBSl$Eq87&O4zKtQauh%lwnoOEa`k>>CEU&}tLKG= zq}K3&n0kt>%=auFTaCIl4$x11x^x-Et2>u5ZX#YsT(Rj4%_Wf~Ihrl6wtX9JGyGVi zt*8%Iy7PYpOxNqysQQ1Y1QM>gDx~tw(*1qddSD{t9T;&Nz*UKk8_tP5xKoEA?+Iqz*4k%W`NwsRu?BX= zbyGkMe8bb=3jZm~sQU`^NUQO0O0n94RZi93V`Ajd(N~p;P+=vJx)`zM0Jj4c-l;|l7}*fiI^LL>oh<}cAZ_6Sz{y;HW$OBY z68zi1Bh4vVRfuAw2YpsAXk{25)jMcz&!#~0$p>5MvNcO9$Aa>Qg%dd)SQ?L%-qQ5a z5HN~rh*{Zqyc=3bY_vDEJTv3eoN10e9&a@qRT#2hu}j|(=$HdT4svXabIS^&Pdd1J zGUWTEcuVUqOF-1~*tiMw`RkhzBWUp?cdXK!^G+HNJ(Vj>JbPN{PN_f#fYPOcr4 zRJ%=dK3RKwEqS!{I(@s2*AsD6g%o%<)XR;yI` zNkZeUrJC@KSpwEJcNIcHy zG*VW9a9D*_$#~tga_I|$KVKtpulu4D0h*WF(SL00Q(jnfTr7l^c_||eq^FU9yfd8~ zBqNvHVxEymnE1_pi1N2jgb-F0Kgo-8zQ!ayfwuOxhC7!G^lwc?Hc7VU@ndyA znvsV#L%xPps%i>-VE@zP{C#foO0GJa(~?dagzpeO)>9#8)RyTMySYaFNf-L%%Cre1 z%)Gz)4gwGY?q92wU`tlJnX4V_R*p7WCs#E3ncb`UQeA*-_aE}5F3N0eguZ()@^&+Q z<86=*MH**-v)Q7M`ETGQ!W7vKV?`ANR^(x6w_&E$Jh!U*%ce6zP^SwvQ4G~`)C?cx zZR();QzbJYr+Dzb13t0=0QT+G_edvfF>SgBaRh3$n9BvGQ~0K5dFsKs3hRYcP(m9o zV+ic0q4>Zyd=G>1UEgNriBnbNC7C=*Gc zi+E0)v&CB;V)dx~Ct{<7zjRUW!7n>9X4(RGjS7AYmwT7GoE1)15DmT?JCgE_-n!l> z%U3_}13cQbo9j+>gnzfTwpj2Bb%2*MziDP;9u&?MiAhB~R>c@dQc0H}j0&b|qC+GS zC_Z27Z9Bf?352kX)w}WZg0_qMyY;z$cQGXn?`9NG48$8#eor=0_0rlW(#ZhX{_luY-jt%b$P>C*;wA$QZ|Q(5>yv->y7F7c6J;R{6qhH`=ZQ&$7^s*u{q zl&Vy#hZOS}BOb*%>F7}GR0eB_KNz-%9FSK-Xomi}h80 zMERB<>7}nw;cs@u@AlL&FytSw$u~cxl|j1-PJ%P=*>e0m6D(kf=Ki;}I;SOKPc8XE zBWsi2M#@{hyvfvr_a8^Z9BzAK3SJm3?vg@f$KO?Hc#i~q`D=jJCdHq5G6@X{9G~-w ze&2pWmAWSuiakiV<1FpjReb=jcU!}UrJdiU;Ng}Rz<`yg*FoIRqwU`V3D~m-cZ01TSTiAC;)Y*cxS*xGVX{A>r1W0MRc(2p<$43 zLC>H22#Zd`eR*z@m_CQkC!p)$E0cI7Ej)~C3Y9!@8_Tn9#&1<+^45b1(T2%eyb#>b zk6G~1Hxjpg#o}~Jl1?@XJ4h~|6%)&<{+loQ&mnE-sYIU{oCvr~pWvU8bcH&uzmxz> z)|Ysqvq2*uH`2y!9NirF2q+Nr4)!h-vnpZQbwew6wa9wXxDMfXvfhb+REZ(BTB~GW zP$XkgI+1Ko%_k9RK}V#hRYJzzssy7&cySLU8h}f5U?@T7AZ`3M2V<&2v`edel9ml73l zREUiH`~zZlzIC4yC{KtWr?B3i{ZBfZX6u?dap*ff&S1X>1>psHnahny#JjfTU4xy0 zAoZy%KdPxsqAdd68^V!Z%lAiN!{>)%`>Lb8a#U3fl&It;`w{8#l}*G>aAf_`?RhH$>ffw~Vg7?Or`|jP z-H-g6Mwtm6D>PCD)!&RRwX8o54VgcoO-NGtHAEB4he;D(jmIZ8s_D< z0WC8>zjc)Jhnq;Pf2gkwSQefY-xtW^f`Td&)*_V-TGYOaTKnb1(&W>zjUbdpUDtW= zA9liHwk$*O0xlojsH**Ac4g8xEqewg$wh`>YV>nYdxqyHQF{P+Nb+jwKu>oqm3>|6 zQ#IxQu5ys&tM&TFK%fv9BWd6nB`zSKiD+$+l=2_&TjaGv{6{36YAZvn^{PyLUV{ zoRh*;R_HyC2zgx*S35{Dx7bE?Wn&JhJZ?>eTLFF|cI?XO} zp}pSE*o$pJ=Js0I`)|8BNl-mA_{W7AjAEU7_K+e1LJh;Si=C=h!!>K!N;HdPe$&Vq zMwqqR)`@Zy^Keb^bLET-yMOd@qR=F_(Ebq@{j$vglv1=cLX1UDDVcfdQC0PIUuXn` z$5R|p#71QNa@{IB6@mbXx#1*78cS4(d}wW5VOcspwi?0XlRRvGX*^d_K->K$jd041 zo_wR5l2{8>CLk}m6J=>`^s}vP6`-Bpn5xz} zm@~wXbd`jFJ{5RitWB@FwkpY|;p `eY~bgi^vmTxo4Des{JG?Scyn;d2qyT3%f z9mvgRglC6jMh{7@PD%_oXDcodpN*dqodHnIa-qyBt9tj28p0}nAdyg^6)!_LMwuNw zsdxazI9~9Y6ah+FLY>8D>fHtRMD_(wAslak$c)V^+K8aOH>byKGmo|Oz=0tFNd2^3_g`Jl0+})BJH-`5ym6N;Jbz0LwURbz_q?tJ;1}5+ zPCm>ZdRfFfUS==SKfI6MWlEc`A#6I<^*b_x&>1%I))`qML1dhwsQSF=19Y*4Ua?|i z*N?9bb-dpk^cH)k_4)O@lzfjV-N6~XGC{LCR3Ltz$eY@WBWv5$$xX~2DV1_&y=9yH z=^(|S*YZgyB7?uC0%y=s=^Sd4u>+xnH}e_Ux*za*Jfz=Y(aMHcL2emPnb zB;r)x)a!O@^kN?Z3H4y=%N1z4{*g@+m_zY^Ny=SnGM#;c;QE#|D^xpigd{}nFsY~y z2@7qZ;ls7}J7Fs!E+;2Fz6pk^E-od6Rx~dnh4jj(Z#&Gl(ddhg@!N(OKk)JGFzQKU zL$fE9&HVHk_|jaK`Ak8q73qV4WHVUUnqxV?c=Li>!Wl>nF>OmR8exo8jki7)9krMl zi=vi74og6WxtheuH#5U_LHeJUo5C&;QRQrrU@%lXfhTNo;@Uj@l+b7f=G47s7i#7z z3=O2thFxRA?R}p5VvpxY+x0f;^Ila3VDlT@j*<%rgBdeEKn~s! z)z*S8HQ76kIC95I#+BWOkJQWz3DL=Mfkz4w?5SHa;@8JrNXvJ}cRi2VSr>3eG*ICV zBOdt)?J#CltH-xz8Qxn0Y$`C7ry45SN3QJ)dzSPADirsn{U$Q1nRg!AW|D26uY z6oY?pHG91*PCrwI*aT=-J^E~hr_4kN=gC)-({Vxu1Ay()T}9z5vStMfmBag)UPb;T z^Gx^p@S6j+F*j+`YGd)*;x5H@5*Svqz`@X`;>+^1Q1^+@r+}AWueF-G z>{%wO88#iXwoK2p-Szvu?u~P`T04qnJ8FQoDFJk$o#-#$i42|I$P$xQ$*hl&Ix1Ih z>Xz)wR`$sjS~RQGCgpNN?0jL^qhG9%Jf~+C5XdE#!&@q~l!Oz>l+#6IY&-;O9;qqO zE+IKjA&1r6KUV19;+~!_CjgV--rr#i#|S%%`ZTnDRi?m>IJt=jn|L^YBx>bf_1M@oKiVW-sPHJL6Qj5bb_ z4p>z^+luU$`&C&+3y9R);TWeVL>fggfD{tIfzYgvgIN}t1^F6ov2} z#ls}XE}MFlB*8LVEaV{)(g z8BK#2s?mV$Mm*Bx zr(%WOTDC9k0+Axw>|cBaL#NrC^rJnTFe-dvJOD7 zp#I8bNhDAQ+uQ@sI7^9+jM|qPc+lr!5M}}s*fR%QN1^B5odseBZ61M>kMLgmrknPF z3%L#*tE$gL8i)d-+!(+0wqe2}>fMCi`$tQ^8SQnTw)*hw|H}kXv;VVW_>09&4}R3; zDw+u|GH)kK3qAoW-F<*@vF3vr`D8T$+kKzZ@EQC)W!U|cg-7~4VhDA5RS3vi-+mWk zJeFiUf?a7gZvFQ9*t54fowKebs3ow*+n{7_mo;;f=0eD?iutN}8my6hxT46+UBSH< zck%ct&FcAVmRI-6)l16z5y8vzg%v2NXoM)KIU_z$2xR`0CO|8pY}#H~X&xgL$x4~IOV@*FIqwU?g3x}OTjv-egX?zTuo2|DEctFMq4 zTXYq4b;?xtG`$!CQaH#^72W51Ch%!4I`DHW`e`FdNOteP*f@|At`wzvS91%AOp@Dr z^X?e@!jtJ^zfv7#D51!i9QN{jPVDrZEwSok4g5^Te-SVQ`mgtdlIsf9JiS!vMgwT4 zKVnI}8PZ)0edKFDFn&6fqYj;+EM!Af1>BxhxJbFXw1+QuctdIT z!s|g9?pg2)Pb4Ki2EGCx*-K^Yat(=iR>ogjaAF!-7t<+@&U;%~7Tbfaud?qviG&Dv zNP01}pqIP}4WqfLa&=6c`XQ2KETFP*?^$QPumYY>`*qqWD{B+1*4oq*Wk*_!h6WNy z8CiofIQFz(V5P!>gtIaEtj@45@ot#W;@b>-2LNL6Q`FN{()ZDV3DK(>s@oLYf=v72 zKcKyc&il>QvJI+UU0>ZLW3MIY_Lk9heNM@*Zax+1c-^LHU;K{N*Ov`9nZb6zy>9{vePEi&b>|U{M>aQ1|5vF<70v=0>o$E?O z9HE4@{Fi&lQm*o#InE?BrG~R)YZ<~tGbiwBO@_H%x|1w@ie!A`q!Rc#+w6-dPKKF- zDD)TpheGb0>O$`a`DmTpeb*Dmc2~^vc1TAF_@-K7hnI~i(zH$9%BCBR-N+i~4&H`} z3YfEf)O!FM_73%IFPuqd006{#W0-bMlNVZWFIVf=MUg(F*xzGI1#a27%$!(AC3qTC zQ1H?Y#RY23Bv?H&EX>)}$Bm8~~ ztb_E2c4~1QQXZ$Ldp;$f%S^`Gimtp+Y8kOp34nEz@*bU6Lfk;-NF(sdahkg-HH4gM zWDm1^IwFR9OLN?#>_x+#l!T2;PO}SL@?9l!T{~iyfvPA2Oa{p)>@TjTvWsb@3FFWv zP^8pRYMF5YK&xd-tkQBoCH@~%^!^+2m}#__;%Ydux}iZw)Kf|I6alS|@JUh+%aw_H zQCm6Hl!(tsZMjkxV*7??)jPYhFM+kB9>6tPjs3&K{M`GWB%1>aH}Yz4jdXoYemJn9 zi8i)k3=19#QGgQrQS=h-bR}MGI{p}JHu{>`5tzX5nrV*D*s$HQ~! z0aa778*_BWX@+r1<93AEz50AR0Q>Rsg!B*ih*aN96?|}&EahTyRK!oC~Dup;;Av#0s;Uza>_cq(GNz@vsiL z4!n+kBb`_NHMPR#gq$OsP}Rs@pqAoQcEX$3j$?wNjl6F}0JZx^HQ{y8QgqLyI5f-X z`A=OW@$>k+PjUBX8=iDrzWa-0X+#=_P}#32yet3&6sqO!IhMJZ1#+D<5;!2 z??|SqB-Y1j*6l^V#_0#=p4Kp`bfr&O3|_4Mcz_cVnoH@9Zb&9BT@PSk?dbD{qI}6A zdy^LZ3NsbH%lKo+sLvuh6ngH1y}n#yT%3=Pg%O#jST#rG`j-jvsrzT?$NApkb12>1 za7~%WTe_>XR=924%B{qu;j`Ad;|>PLr%;QsuAS>&jUzs;0U1|;m0B8l3zNKH)+Uaf zQjtGPbAVg90qH{(F{`@VzFwJx?}q#K{+|WB<#dw)&k{d;1!P%c>=KbeoYBzvT2L-p z_eq7WHYXq3#Kz8T_uv@fCzpmonyJ0qBu@OWndKb#Z;7+XLT?fRu(*KB3VZ8|p7mdx zCJdB~arI`Lei@V3d<@%&{)`O_V&9qsG<#Oa~qZOhx1 apCd18cKY=HGgSBf765;sEdZJW0s0Tr??sRR literal 0 HcmV?d00001 diff --git a/sanoid.spec b/packages/rhel/sanoid.spec similarity index 92% rename from sanoid.spec rename to packages/rhel/sanoid.spec index c0f33ed..ab299a5 100644 --- a/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -1,4 +1,4 @@ -%global version 1.4.14 +%global version 1.4.18 %global git_tag v%{version} # Enable with systemctl "enable sanoid.timer" @@ -6,13 +6,13 @@ Name: sanoid Version: %{version} -Release: 2%{?dist} +Release: 1%{?dist} BuildArch: noarch Summary: A policy-driven snapshot management tool for ZFS file systems Group: Applications/System License: GPLv3 URL: https://github.com/jimsalterjrs/sanoid -Source0: https://github.com/jimsalterjrs/%{name}/archive/%{git_tag}/%{name}-%{version}.tar.gz +Source0: https://github.com/jimsalterjrs/%{name}/archive/%{git_tag}/%{name}-%{version}.tar.gz Requires: perl, mbuffer, lzop, pv %if 0%{?_with_systemd} @@ -110,6 +110,8 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name} %endif %changelog +* Sat Apr 28 2018 Dominic Robinson - 1.4.18-1 +- Bump to 1.4.18 * Thu Aug 31 2017 Dominic Robinson - 1.4.14-2 - Add systemd timers * Wed Aug 30 2017 Dominic Robinson - 1.4.14-1 @@ -121,6 +123,5 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name} - Version bump - Clean up variables and macros - Compatible with both Fedora and Red Hat - * Sat Feb 13 2016 Thomas M. Lapp - 1.4.4-1 - Initial RPM Package diff --git a/packages/rhel/sources b/packages/rhel/sources new file mode 100644 index 0000000..d6068d4 --- /dev/null +++ b/packages/rhel/sources @@ -0,0 +1 @@ +cf0ec23c310d2f9416ebabe48f5edb73 sanoid-1.4.18.tar.gz From faf825003fbb7448be5a566fe1c238711a031298 Mon Sep 17 00:00:00 2001 From: Eric Coutu Date: Sun, 29 Apr 2018 15:06:31 -0400 Subject: [PATCH 045/148] Document compatibility with (t)csh shells Also, recommend installing mbuffer on FreeBSD systems when remote user is using bourne compatible shell. --- FREEBSD.readme | 11 +++++++++++ INSTALL | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/FREEBSD.readme b/FREEBSD.readme index 9960201..732940d 100644 --- a/FREEBSD.readme +++ b/FREEBSD.readme @@ -11,3 +11,14 @@ If you don't want to have to change the shebangs, your other option is to drop a root@bsd:~# ln -s /usr/local/bin/perl /usr/bin/perl After putting this symlink in place, ANY perl script shebanged for Linux will work on your system too. + +Syncoid assumes a bourne style shell on remote hosts. Using (t)csh (the default for root under FreeBSD) +has some known issues: + +* If mbuffer is present, syncoid will fail with an "Ambiguous output redirect." error. So if you: + root@bsd:~# ln -s /usr/local/bin/mbuffer /usr/bin/mbuffer + make sure the remote user is using an sh compatible shell. + +To change to a compatible shell, use the chsh command: + +root@bsd:~# chsh -s /bin/sh diff --git a/INSTALL b/INSTALL index 1492546..33b510d 100644 --- a/INSTALL +++ b/INSTALL @@ -9,7 +9,7 @@ is not available on either end of the transport. On Ubuntu: apt install pv lzop mbuffer On CentOS: yum install lzo pv mbuffer lzop -On FreeBSD: pkg install pv lzop +On FreeBSD: pkg install pv mbuffer lzop FreeBSD notes: FreeBSD may place pv and lzop in somewhere other than /usr/bin ; syncoid currently does not check path. @@ -19,6 +19,8 @@ FreeBSD notes: FreeBSD may place pv and lzop in somewhere other than or similar, as appropriate, to create links in /usr/bin to wherever the utilities actually are on your system. + See note about mbuffer in FREEBSD.readme + SANOID ------ From f670631ca281961c318b0b9fb56ca28017e5de38 Mon Sep 17 00:00:00 2001 From: Gionatan Danti Date: Mon, 30 Apr 2018 21:50:17 +0200 Subject: [PATCH 046/148] Added "no-clone-rollback" option to prevent clone rollback on target host --- syncoid | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/syncoid b/syncoid index 7337f5b..e91c126 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-clone-rollback") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -306,13 +306,17 @@ sub syncdataset { if (!$quiet) { print "INFO: no snapshots on source newer than $newsyncsnap on target. Nothing to do, not syncing.\n"; } } else { # rollback target to matchingsnap + my $rollbacktype="-R"; + if (defined $args{'no-clone-rollback'}) { + $rollbacktype = "-r"; + } if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; } if ($targethost ne '') { - if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap\n"; } - system ("$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap"); + if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap\n"; } + system ("$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap"); } else { - if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap\n"; } - system ("$targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap"); + if ($debug) { print "$targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap\n"; } + system ("$targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap"); } my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefs\@$matchingsnap $sourcefs\@$newsyncsnap"; @@ -949,6 +953,7 @@ Options: --target-bwlimit= Bandwidth limit on the target transfer --no-stream Replicates using newest snapshot instead of intermediates --no-sync-snap Does not create new snapshot, only transfers existing + --no-clone-rollback Does not rollback clones on target --sshkey=FILE Specifies a ssh public key to use to connect --sshport=PORT Connects to remote on a particular port From 670e76458c0dff00323aa4ceff67b16cb2fc1841 Mon Sep 17 00:00:00 2001 From: Gionatan Danti Date: Mon, 30 Apr 2018 22:45:05 +0200 Subject: [PATCH 047/148] Reintroduce "no-resume", erroneously removed from previous commit --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 512e153..c6c85ed 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-clone-rollback") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-clone-rollback", "no-resume") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set From 793f1c782b703a9050f95a06eb8d6847af58f64e Mon Sep 17 00:00:00 2001 From: Gionatan Danti Date: Tue, 1 May 2018 09:32:18 +0200 Subject: [PATCH 048/148] Reintroduced escapeshellparam for remote command and updated README file --- README.md | 8 ++++++++ syncoid | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cc75bdf..7a9b76c 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,10 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da This will process your sanoid.conf file, it will NOT create snapshots, but it will purge expired ones. ++ --force-prune + + Purges expired snapshots even if a send/recv is in progress + + --monitor-snapshots This option is designed to be run by a Nagios monitoring system. It reports on the health of your snapshots. @@ -150,6 +154,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This argument tells syncoid to restrict itself to existing snapshots, instead of creating a semi-ephemeral syncoid snapshot at execution time. Especially useful in multi-target (A->B, A->C) replication schemes, where you might otherwise accumulate a large number of foreign syncoid snapshots. ++ --no-clone-rollback + + Does not rollback clones on target + + --no-resume This argument tells syncoid to not use resumeable zfs send/receive streams. diff --git a/syncoid b/syncoid index c6c85ed..0a2fe4a 100755 --- a/syncoid +++ b/syncoid @@ -366,7 +366,7 @@ sub syncdataset { if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; } if ($targethost ne '') { if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap\n"; } - system ("$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap"); + system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped")); } else { if ($debug) { print "$targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap\n"; } system ("$targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap"); From 679f1f3bda6feec0392eada8ae65072ac0393627 Mon Sep 17 00:00:00 2001 From: Gionatan Danti Date: Tue, 1 May 2018 09:39:35 +0200 Subject: [PATCH 049/148] Reintroduced targetfsescaped --- syncoid | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/syncoid b/syncoid index 0a2fe4a..49e3aac 100755 --- a/syncoid +++ b/syncoid @@ -359,19 +359,18 @@ sub syncdataset { } else { my $matchingsnapescaped = escapeshellparam($matchingsnap); # rollback target to matchingsnap - my $rollbacktype="-R"; + my $rollbacktype = "-R"; if (defined $args{'no-clone-rollback'}) { $rollbacktype = "-r"; } if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; } if ($targethost ne '') { - if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap\n"; } + if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped\n"; } system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped")); } else { - if ($debug) { print "$targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap\n"; } - system ("$targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap"); + if ($debug) { print "$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped\n"; } + system ("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped"); } - my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); From 63979973b2ed5da4c7c1884c43f428041cdcde84 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 2 May 2018 00:15:14 +0200 Subject: [PATCH 050/148] if syncoid is instructed to skip snapshot creation it shouldn't attempt to prune existing ones --- syncoid | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index e63b68c..999ee79 100755 --- a/syncoid +++ b/syncoid @@ -387,9 +387,11 @@ sub syncdataset { } } - # prune obsolete sync snaps on source and target. - pruneoldsyncsnaps($sourcehost,$sourcefs,$newsyncsnap,$sourceisroot,keys %{ $snaps{'source'}}); - pruneoldsyncsnaps($targethost,$targetfs,$newsyncsnap,$targetisroot,keys %{ $snaps{'target'}}); + if (!defined $args{'no-sync-snap'}) { + # prune obsolete sync snaps on source and target (only if this run created ones). + pruneoldsyncsnaps($sourcehost,$sourcefs,$newsyncsnap,$sourceisroot,keys %{ $snaps{'source'}}); + pruneoldsyncsnaps($targethost,$targetfs,$newsyncsnap,$targetisroot,keys %{ $snaps{'target'}}); + } } # end syncdataset() From bb654c1cf6d59f06727be8b93c66bbdd2a42b9d1 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 5 Jun 2018 11:41:48 +0200 Subject: [PATCH 051/148] use utc for timestamps as default --- README.md | 4 +++- packages/debian/sanoid.service | 1 + packages/rhel/sanoid.spec | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cc75bdf..be6195d 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,11 @@ More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: ``` -* * * * * /usr/local/bin/sanoid --cron +* * * * * TZ=UTC /usr/local/bin/sanoid --cron ``` +`Note`: Using UTC as timezone is recommend to prevent problems with daylight saving times + And its /etc/sanoid/sanoid.conf might look something like this: ``` diff --git a/packages/debian/sanoid.service b/packages/debian/sanoid.service index b54c586..2d01bbf 100644 --- a/packages/debian/sanoid.service +++ b/packages/debian/sanoid.service @@ -5,5 +5,6 @@ After=zfs.target ConditionFileNotEmpty=/etc/sanoid/sanoid.conf [Service] +Environment=TZ=UTC Type=oneshot ExecStart=/usr/sbin/sanoid --cron diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index ab299a5..3a9412f 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -58,6 +58,7 @@ Requires=zfs.target After=zfs.target [Service] +Environment=TZ=UTC Type=oneshot ExecStart=%{_sbindir}/sanoid --cron EOF From 52661afb4125ba350ce0ec54e3928a4ab2140f33 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 5 Jun 2018 12:45:13 +0200 Subject: [PATCH 052/148] updated debian changelog --- packages/debian/changelog | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/debian/changelog b/packages/debian/changelog index ab530b0..2bcf423 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -1,3 +1,18 @@ +sanoid (1.4.18) unstable; urgency=medium + + implemented special character handling and support of ZFS resume/receive tokens by default in syncoid, + thank you @phreaker0! + + -- Jim Salter Wed, 25 Apr 2018 16:24:00 -0400 + +sanoid (1.4.17) unstable; urgency=medium + + changed die to warn when unexpectedly unable to remove a snapshot - this + allows sanoid to continue taking/removing other snapshots not affected by + whatever lock prevented the first from being taken or removed + + -- Jim Salter Wed, 8 Nov 2017 15:25:00 -0400 + sanoid (1.4.16) unstable; urgency=medium * merged @hrast01's extended fix to support -o option1=val,option2=val passthrough to SSH. merged @JakobR's From e65879d1f8c4a1304ee0d16d4393645b5e07de1e Mon Sep 17 00:00:00 2001 From: Lucas Salibian Date: Fri, 8 Jun 2018 11:52:52 -0400 Subject: [PATCH 053/148] Fix --help typo --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 999ee79..a15f872 100755 --- a/syncoid +++ b/syncoid @@ -1148,7 +1148,7 @@ Options: --sshoption|o=OPTION Passes OPTION to ssh for remote usage. Can be specified multiple times --help Prints this helptext - --verbose Prints the version number + --version Prints the version number --debug Prints out a lot of additional information during a syncoid run --monitor-version Currently does nothing --quiet Suppresses non-error output From f9049085c8b9d7ff4800c44d4cd286bd67337107 Mon Sep 17 00:00:00 2001 From: Gionatan Danti Date: Mon, 11 Jun 2018 09:48:45 +0200 Subject: [PATCH 054/148] Added "no-rollback" option to prevent any rollback on target host --- syncoid | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/syncoid b/syncoid index 49e3aac..9dc83fa 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-clone-rollback", "no-resume") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-clone-rollback", "no-rollback", "no-resume") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -151,6 +151,12 @@ sub syncdataset { my $sourcefsescaped = escapeshellparam($sourcefs); my $targetfsescaped = escapeshellparam($targetfs); + # if no rollbacks are allowed, disable forced receive + my $forcedrecv = "-F"; + if (defined $args{'no-rollback'}) { + $forcedrecv = ""; + } + if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; } # make sure target is not currently in receive. @@ -244,7 +250,7 @@ sub syncdataset { my $oldestsnapescaped = escapeshellparam($oldestsnap); my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); my $disp_pvsize = readablebytes($pvsize); @@ -313,7 +319,7 @@ sub syncdataset { # snapshot, do a normal sync after that if (defined($receivetoken)) { my $sendcmd = "$sourcesudocmd $zfscmd send -t $receivetoken"; - my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"","",$sourceisroot,$receivetoken); my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -359,20 +365,22 @@ sub syncdataset { } else { my $matchingsnapescaped = escapeshellparam($matchingsnap); # rollback target to matchingsnap - my $rollbacktype = "-R"; - if (defined $args{'no-clone-rollback'}) { - $rollbacktype = "-r"; - } - if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; } - if ($targethost ne '') { - if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped\n"; } - system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped")); - } else { - if ($debug) { print "$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped\n"; } - system ("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped"); + if (!defined $args{'no-rollback'}) { + my $rollbacktype = "-R"; + if (defined $args{'no-clone-rollback'}) { + $rollbacktype = "-r"; + } + if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; } + if ($targethost ne '') { + if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped\n"; } + system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped")); + } else { + if ($debug) { print "$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped\n"; } + system ("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped"); + } } my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -1143,6 +1151,7 @@ Options: --no-stream Replicates using newest snapshot instead of intermediates --no-sync-snap Does not create new snapshot, only transfers existing --no-clone-rollback Does not rollback clones on target + --no-rollback Does not rollback clones or snapshots on target (it probably requires a readonly target) --sshkey=FILE Specifies a ssh public key to use to connect --sshport=PORT Connects to remote on a particular port From 4ed6ff0e447f436fb3b7a77405050839927e61a6 Mon Sep 17 00:00:00 2001 From: Gionatan Danti Date: Mon, 11 Jun 2018 10:06:36 +0200 Subject: [PATCH 055/148] Updated README file --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7a9b76c..58ec7dd 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --no-clone-rollback - Does not rollback clones on target + Do not rollback clones on target + ++ --no-rollback + Do not rollback anything (clones or snapshots) on target host + --no-resume From 0aaac4205790e1e179d41140ba38142cf442389c Mon Sep 17 00:00:00 2001 From: Gionatan Danti Date: Mon, 11 Jun 2018 10:16:18 +0200 Subject: [PATCH 056/148] skip pruning with --no-sync-snap (as by PR #218) --- syncoid | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index 9dc83fa..ae31131 100755 --- a/syncoid +++ b/syncoid @@ -398,9 +398,11 @@ sub syncdataset { } } - # prune obsolete sync snaps on source and target. - pruneoldsyncsnaps($sourcehost,$sourcefs,$newsyncsnap,$sourceisroot,keys %{ $snaps{'source'}}); - pruneoldsyncsnaps($targethost,$targetfs,$newsyncsnap,$targetisroot,keys %{ $snaps{'target'}}); + if (!defined $args{'no-sync-snap'}) { + # prune obsolete sync snaps on source and target (only if this run created ones). + pruneoldsyncsnaps($sourcehost,$sourcefs,$newsyncsnap,$sourceisroot,keys %{ $snaps{'source'}}); + pruneoldsyncsnaps($targethost,$targetfs,$newsyncsnap,$targetisroot,keys %{ $snaps{'target'}}); + } } # end syncdataset() From c0c30500765395cd22f975abb198ce3d4fb1d1f3 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 19 Jun 2018 18:21:06 +0200 Subject: [PATCH 057/148] added option for skipping the parent dataset in recursive replication --- README.md | 4 ++++ syncoid | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cc75bdf..7392fa7 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This will also transfer child datasets. ++ --skip-parent + + This will skip the syncing of the parent dataset. Does nothing without '--recursive' option. + + --compress Currently accepted options: gzip, pigz-fast, pigz-slow, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. diff --git a/syncoid b/syncoid index 999ee79..5cbc1c4 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-resume") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "skip-parent") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -141,6 +141,11 @@ sub getchilddatasets { my @children = ; close FH; + if (defined $args{'skip-parent'}) { + # parent dataset is the first element + shift @children; + } + return @children; } @@ -1137,6 +1142,7 @@ Options: --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none --recursive|r Also transfers child datasets + --skip-parent Skipp the syncing of the parent dataset. Doesg nothing without '--recursive' option. --source-bwlimit= Bandwidth limit on the source transfer --target-bwlimit= Bandwidth limit on the target transfer --no-stream Replicates using newest snapshot instead of intermediates From 70b259ac3cb9ef9167776eac0424d0a78ea0c0c4 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 19 Jun 2018 18:24:34 +0200 Subject: [PATCH 058/148] typos --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 5cbc1c4..8ab3a41 100755 --- a/syncoid +++ b/syncoid @@ -1142,7 +1142,7 @@ Options: --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none --recursive|r Also transfers child datasets - --skip-parent Skipp the syncing of the parent dataset. Doesg nothing without '--recursive' option. + --skip-parent Skips syncing of the parent dataset. Does nothing without '--recursive' option. --source-bwlimit= Bandwidth limit on the source transfer --target-bwlimit= Bandwidth limit on the target transfer --no-stream Replicates using newest snapshot instead of intermediates From add7bf6de769e32a35f200c87143f4dfcbc126dc Mon Sep 17 00:00:00 2001 From: Gionatan Danti Date: Tue, 19 Jun 2018 19:08:30 +0200 Subject: [PATCH 059/148] Added a missing newline in README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 58ec7dd..4215ee4 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup Do not rollback clones on target + --no-rollback + Do not rollback anything (clones or snapshots) on target host + --no-resume From 0f8fee7637ec1265bd0a72011a08b0666e6123c4 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 19 Jun 2018 19:43:36 +0200 Subject: [PATCH 060/148] added option for using extra identification in the snapshot name for replicating to multiple targets --- README.md | 4 ++++ syncoid | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cc75bdf..632a0af 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This is the destination dataset. It can be either local or remote. ++ --identifier= + + Adds the given identifier to the snapshot name after "syncoid_" prefix and before the hostname. This enables the use case of reliable replication to multiple targets from the same host. The following chars are allowed: a-z, A-Z, 0-9 and -. + + -r --recursive This will also transfer child datasets. diff --git a/syncoid b/syncoid index 999ee79..dcb9e2f 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-resume") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "identifier=s") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -71,6 +71,17 @@ if (length $args{'sshkey'}) { } my $sshoptions = join " ", map { "-o " . $_ } @{$args{'sshoption'}}; # deref required +my $identifier = ""; +if (length $args{'identifier'}) { + if ($args{'identifier'} !~ /^[a-zA-Z0-9-]+$/) { + # invalid extra identifier + print("CRITICAL: extra identifier contains invalid chars!\n"); + pod2usage(2); + exit 127; + } + $identifier = "$args{'identifier'}_"; +} + # figure out if source and/or target are remote. $sshcmd = "$sshcmd $args{'sshcipher'} $sshoptions $args{'sshport'} $args{'sshkey'}"; if ($debug) { print "DEBUG: SSHCMD: $sshcmd\n"; } @@ -812,7 +823,7 @@ sub pruneoldsyncsnaps { # only prune snaps beginning with syncoid and our own hostname foreach my $snap(@snaps) { - if ($snap =~ /^syncoid_\Q$hostid\E/) { + if ($snap =~ /^syncoid_\Q$identifier$hostid\E/) { # no matter what, we categorically refuse to # prune the new sync snap we created for this run if ($snap ne $newsyncsnap) { @@ -898,7 +909,7 @@ sub newsyncsnap { if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } my $hostid = hostname(); my %date = getdate(); - my $snapname = "syncoid\_$hostid\_$date{'stamp'}"; + my $snapname = "syncoid\_$identifier$hostid\_$date{'stamp'}"; my $snapcmd = "$rhost $mysudocmd $zfscmd snapshot $fsescaped\@$snapname\n"; system($snapcmd) == 0 or die "CRITICAL ERROR: $snapcmd failed: $?"; @@ -1136,6 +1147,7 @@ syncoid - ZFS snapshot replication tool Options: --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none + --identifier=EXTRA Extra identifier which is included in the snapshot name. Can be used for replicating to multiple targets. --recursive|r Also transfers child datasets --source-bwlimit= Bandwidth limit on the source transfer --target-bwlimit= Bandwidth limit on the target transfer From 34b942ea45cbc686c1e0fe16c8dd6c4515115826 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 21 Jun 2018 18:18:28 +0200 Subject: [PATCH 061/148] correctly parse zfs column output (space can be included in the values) --- sanoid | 2 +- syncoid | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sanoid b/sanoid index b6dc9fe..c07e090 100755 --- a/sanoid +++ b/sanoid @@ -521,7 +521,7 @@ sub getsnaps { } foreach my $snap (@rawsnaps) { - my ($fs,$snapname,$snapdate) = ($snap =~ m/(.*)\@(.*ly)\s*creation\s*(\d*)/); + my ($fs,$snapname,$snapdate) = ($snap =~ m/(.*)\@(.*ly)\t*creation\t*(\d*)/); # avoid pissing off use warnings if (defined $snapname) { diff --git a/syncoid b/syncoid index 999ee79..ca7328f 100755 --- a/syncoid +++ b/syncoid @@ -664,7 +664,7 @@ sub getzfsvalue { open FH, "$rhost $mysudocmd $zfscmd get -H $property $fsescaped |"; my $value = ; close FH; - my @values = split(/\s/,$value); + my @values = split(/\t/,$value); $value = $values[2]; return $value; } @@ -985,7 +985,7 @@ sub getsnaps() { if ($line =~ /\Q$fs\E\@.*guid/) { chomp $line; my $guid = $line; - $guid =~ s/^.*\sguid\s*(\d*).*/$1/; + $guid =~ s/^.*\tguid\t*(\d*).*/$1/; my $snap = $line; $snap =~ s/^.*\@(.*)\tguid.*$/$1/; $snaps{$type}{$snap}{'guid'}=$guid; @@ -997,7 +997,7 @@ sub getsnaps() { if ($line =~ /\Q$fs\E\@.*creation/) { chomp $line; my $creation = $line; - $creation =~ s/^.*\screation\s*(\d*).*/$1/; + $creation =~ s/^.*\tcreation\t*(\d*).*/$1/; my $snap = $line; $snap =~ s/^.*\@(.*)\tcreation.*$/$1/; $snaps{$type}{$snap}{'creation'}=$creation; @@ -1056,9 +1056,9 @@ sub getsendsize { # the output format is different in case of # a resumed receive if (defined($receivetoken)) { - $sendsize =~ s/.*\s([0-9]+)$/$1/; + $sendsize =~ s/.*\t([0-9]+)$/$1/; } else { - $sendsize =~ s/^size\s*//; + $sendsize =~ s/^size\t*//; } chomp $sendsize; From 1f885801993eec2ce259d3ba4beb7704c6eb496d Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 28 Jun 2018 17:45:18 +0200 Subject: [PATCH 062/148] implemented support for excluding datasets from replication with a regular expression --- README.md | 4 ++++ syncoid | 17 ++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cc75bdf..e98763b 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This argument tells syncoid to restrict itself to existing snapshots, instead of creating a semi-ephemeral syncoid snapshot at execution time. Especially useful in multi-target (A->B, A->C) replication schemes, where you might otherwise accumulate a large number of foreign syncoid snapshots. ++ --exclude=REGEX + + The given regular expression will be matched against all datasets which would be synced by this run and excludes them. This argument can be specified multiple times. + + --no-resume This argument tells syncoid to not use resumeable zfs send/receive streams. diff --git a/syncoid b/syncoid index 999ee79..5cd925c 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-resume") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -141,6 +141,20 @@ sub getchilddatasets { my @children = ; close FH; + if (defined $args{'exclude'}) { + my $excludes = $args{'exclude'}; + foreach (@$excludes) { + for my $i ( 0 .. $#children ) { + if ($children[$i] =~ /$_/) { + if ($debug) { print "DEBUG: excluded $children[$i] because of $_\n"; } + undef $children[$i] + } + } + + @children = grep{ defined }@children; + } + } + return @children; } @@ -1141,6 +1155,7 @@ Options: --target-bwlimit= Bandwidth limit on the target transfer --no-stream Replicates using newest snapshot instead of intermediates --no-sync-snap Does not create new snapshot, only transfers existing + --exclude=REGEX Exclude specific datasets which match the given regular expression. Can be specified multiple times --sshkey=FILE Specifies a ssh public key to use to connect --sshport=PORT Connects to remote on a particular port From ba3836ec520efc30689238e294de1d1c2026fc1b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 6 Jul 2018 15:52:54 +0200 Subject: [PATCH 063/148] fixed monitor-health command for pools containing cache and log devices --- sanoid | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 9cd9d33..485ee08 100755 --- a/sanoid +++ b/sanoid @@ -976,6 +976,11 @@ sub check_zpool() { ## other cases my ($dev, $sta) = /^\s+(\S+)\s+(\S+)/; + if (!defined($sta)) { + # cache and logs are special and don't have a status + next; + } + ## pool online, not degraded thanks to dead/corrupted disk if ($state eq "OK" && $sta eq "UNAVAIL") { $state="WARNING"; @@ -1111,7 +1116,7 @@ sub checklock { # make sure lockfile contains something if ( -z $lockfile) { # zero size lockfile, something is wrong - die "ERROR: something is wrong! $lockfile is empty\n"; + die "ERROR: something is wrong! $lockfile is empty\n"; } # lockfile exists. read pid and mutex from it. see if it's our pid. if not, see if From f9c1cbb74a3c07fc9e0368721cdcedcc0b2b4a0f Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Sat, 7 Jul 2018 12:06:35 -0400 Subject: [PATCH 064/148] Update INSTALL --- INSTALL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL b/INSTALL index 33b510d..f0de17b 100644 --- a/INSTALL +++ b/INSTALL @@ -8,7 +8,7 @@ default for SSH transport since v1.4.6. Syncoid runs will fail if one of them is not available on either end of the transport. On Ubuntu: apt install pv lzop mbuffer -On CentOS: yum install lzo pv mbuffer lzop +On CentOS: yum install lzo pv mbuffer lzop perl-Data-Dumper On FreeBSD: pkg install pv mbuffer lzop FreeBSD notes: FreeBSD may place pv and lzop in somewhere other than From dc0afebb30865faa16f42c496b2c59e5ed72b690 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sat, 7 Jul 2018 20:06:17 +0200 Subject: [PATCH 065/148] allow extra identifier to contain all characters for snapshots names which are allowed by zfs --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index d25d9de..aaf5490 100755 --- a/syncoid +++ b/syncoid @@ -73,7 +73,7 @@ my $sshoptions = join " ", map { "-o " . $_ } @{$args{'sshoption'}}; # deref req my $identifier = ""; if (length $args{'identifier'}) { - if ($args{'identifier'} !~ /^[a-zA-Z0-9-]+$/) { + if ($args{'identifier'} !~ /^[a-zA-Z0-9-_:.]+$/) { # invalid extra identifier print("CRITICAL: extra identifier contains invalid chars!\n"); pod2usage(2); From f409b955694b9dc5f403f16593cc6540cac7fe67 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sat, 7 Jul 2018 20:10:02 +0200 Subject: [PATCH 066/148] updated parameter documention of --identifier regarding allowed characters --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e1f651..e359886 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --identifier= - Adds the given identifier to the snapshot name after "syncoid_" prefix and before the hostname. This enables the use case of reliable replication to multiple targets from the same host. The following chars are allowed: a-z, A-Z, 0-9 and -. + Adds the given identifier to the snapshot name after "syncoid_" prefix and before the hostname. This enables the use case of reliable replication to multiple targets from the same host. The following chars are allowed: a-z, A-Z, 0-9, _, -, : and . . + -r --recursive From 75d8174e6997286aad57d03d1c6ad86d2fd89c79 Mon Sep 17 00:00:00 2001 From: Piotr Paczynski Date: Wed, 18 Jul 2018 01:19:37 +0200 Subject: [PATCH 067/148] Fix 'resume support' detection on FreeBSD --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index aaf5490..31ff757 100755 --- a/syncoid +++ b/syncoid @@ -614,7 +614,7 @@ sub checkcommands { # check for ZFS resume feature support if ($resume) { - my $resumechkcmd = "$zfscmd get receive_resume_token -d 0"; + my $resumechkcmd = "$zfscmd get -d 0 receive_resume_token"; if ($debug) { print "DEBUG: checking availability of zfs resume feature on source...\n"; } $avail{'sourceresume'} = system("$sourcessh $resumechkcmd >/dev/null 2>&1"); From f5508a240387bb80f3db6937c8724fa006e4a5b2 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 26 Jul 2018 21:46:50 +0200 Subject: [PATCH 068/148] fix typo to make local source bwlimit work --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index f53aee4..cf941bf 100755 --- a/syncoid +++ b/syncoid @@ -757,7 +757,7 @@ sub buildsynccmd { $synccmd = "$sendcmd |"; # avoid confusion - accept either source-bwlimit or target-bwlimit as the bandwidth limiting option here my $bwlimit = ''; - if (length $args{'bwlimit'}) { + if (length $args{'source-bwlimit'}) { $bwlimit = $args{'source-bwlimit'}; } elsif (length $args{'target-bwlimit'}) { $bwlimit = $args{'target-bwlimit'}; From e85c375bbea636aa068bb3cd4add7d621568784c Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 26 Jul 2018 21:53:56 +0200 Subject: [PATCH 069/148] fixed typo --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index f53aee4..8be70e5 100755 --- a/syncoid +++ b/syncoid @@ -863,7 +863,7 @@ sub pruneoldsyncsnaps { $prunecmd = escapeshellparam($prunecmd); } system("$rhost $prunecmd") == 0 - or warn "CRITICAL ERROR: $rhost $prunecmd failed: $?"; + or warn "WARNING: $rhost $prunecmd failed: $?"; $prunecmd = ''; $counter = 0; } From 99dcf7f3406b64f7f6ae261b41666fd2bbbaeec9 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 27 Jul 2018 22:52:36 +0200 Subject: [PATCH 070/148] implemented support for zfs bookmarks, if no matching snapshots are found bookmarks are tried as fallback, both stream and no-stream cases are supported --- syncoid | 180 ++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 148 insertions(+), 32 deletions(-) diff --git a/syncoid b/syncoid index 6c569b2..3cbff6b 100755 --- a/syncoid +++ b/syncoid @@ -370,11 +370,48 @@ sub syncdataset { my $targetsize = getzfsvalue($targethost,$targetfs,$targetisroot,'-p used'); - my $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, $targetsize, \%snaps); + my $bookmark = 0; + my $bookmarkcreation = 0; + + my $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, \%snaps); if (! $matchingsnap) { - # no matching snapshot; we whined piteously already, but let's go ahead and return false - # now in case more child datasets need replication. - return 0; + # no matching snapshots, check for bookmarks as fallback + my %bookmarks = getbookmarks($sourcehost,$sourcefs,$sourceisroot); + + # check for matching guid of source bookmark and target snapshot (oldest first) + foreach my $snap ( sort { $snaps{'target'}{$b}{'creation'}<=>$snaps{'target'}{$a}{'creation'} } keys %{ $snaps{'target'} }) { + my $guid = $snaps{'target'}{$snap}{'guid'}; + + if (defined $bookmarks{$guid}) { + # found a match + $bookmark = $bookmarks{$guid}{'name'}; + $bookmarkcreation = $bookmarks{$guid}{'creation'}; + $matchingsnap = $snap; + last; + } + } + + if (! $bookmark) { + # if we got this far, we failed to find a matching snapshot/bookmark. + + print "\n"; + print "CRITICAL ERROR: Target $targetfs exists but has no snapshots matching with $sourcefs!\n"; + print " Replication to target would require destroying existing\n"; + print " target. Cowardly refusing to destroy your existing target.\n\n"; + + # experience tells me we need a mollyguard for people who try to + # zfs create targetpool/targetsnap ; syncoid sourcepool/sourcesnap targetpool/targetsnap ... + + if ( $targetsize < (64*1024*1024) ) { + print " NOTE: Target $targetfs dataset is < 64MB used - did you mistakenly run\n"; + print " \`zfs create $args{'target'}\` on the target? ZFS initial\n"; + print " replication must be to a NON EXISTENT DATASET, which will\n"; + print " then be CREATED BY the initial replication process.\n\n"; + } + + # return false now in case more child datasets need replication. + return 0; + } } # make sure target is (still) not currently in receive. @@ -398,17 +435,68 @@ sub syncdataset { system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped"); } - my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; - my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); - my $disp_pvsize = readablebytes($pvsize); - if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } - my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + my $nextsnapshot = 0; - if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; } - if ($debug) { print "DEBUG: $synccmd\n"; } - system("$synccmd") == 0 - or die "CRITICAL ERROR: $synccmd failed: $?"; + if ($bookmark) { + my $bookmarkescaped = escapeshellparam($bookmark); + + if (!defined $args{'no-stream'}) { + # if intermediate snapshots are needed we need to find the next oldest snapshot, + # do an replication to it and replicate as always from oldest to newest + # because bookmark sends doesn't support intermediates directly + foreach my $snap ( sort { $snaps{'source'}{$a}{'creation'}<=>$snaps{'source'}{$b}{'creation'} } keys %{ $snaps{'source'} }) { + if ($snaps{'source'}{$snap}{'creation'} >= $bookmarkcreation) { + $nextsnapshot = $snap; + last; + } + } + } + + # bookmark stream size can't be determined + my $pvsize = 0; + my $disp_pvsize = "UNKNOWN"; + + if ($nextsnapshot) { + my $nextsnapshotescaped = escapeshellparam($nextsnapshot); + + my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$nextsnapshotescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + + if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $nextsnapshot (~ $disp_pvsize):\n"; } + if ($debug) { print "DEBUG: $synccmd\n"; } + system("$synccmd") == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; + + $matchingsnap = $nextsnapshot; + $matchingsnapescaped = escapeshellparam($matchingsnap); + } else { + my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$newsyncsnapescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + + if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $newsyncsnap (~ $disp_pvsize):\n"; } + if ($debug) { print "DEBUG: $synccmd\n"; } + system("$synccmd") == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; + } + } + + # do a normal replication if bookmarks aren't used or if previous + # bookmark replication was only done to the next oldest snapshot + if (!$bookmark || $nextsnapshot) { + my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); + my $disp_pvsize = readablebytes($pvsize); + if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } + my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + + if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; } + if ($debug) { print "DEBUG: $synccmd\n"; } + system("$synccmd") == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; + } # restore original readonly value to target after sync complete # dyking this functionality out for the time being due to buggy mount/unmount behavior @@ -900,31 +988,15 @@ sub pruneoldsyncsnaps { } sub getmatchingsnapshot { - my ($sourcefs, $targetfs, $targetsize, $snaps) = @_; + my ($sourcefs, $targetfs, $snaps) = @_; foreach my $snap ( sort { $snaps{'source'}{$b}{'creation'}<=>$snaps{'source'}{$a}{'creation'} } keys %{ $snaps{'source'} }) { - if (defined $snaps{'target'}{$snap}{'guid'}) { + if (defined $snaps{'target'}{$snap}) { if ($snaps{'source'}{$snap}{'guid'} == $snaps{'target'}{$snap}{'guid'}) { return $snap; } } } - # if we got this far, we failed to find a matching snapshot. - - print "\n"; - print "CRITICAL ERROR: Target $targetfs exists but has no snapshots matching with $sourcefs!\n"; - print " Replication to target would require destroying existing\n"; - print " target. Cowardly refusing to destroy your existing target.\n\n"; - - # experience tells me we need a mollyguard for people who try to - # zfs create targetpool/targetsnap ; syncoid sourcepool/sourcesnap targetpool/targetsnap ... - - if ( $targetsize < (64*1024*1024) ) { - print " NOTE: Target $targetfs dataset is < 64MB used - did you mistakenly run\n"; - print " \`zfs create $args{'target'}\` on the target? ZFS initial\n"; - print " replication must be to a NON EXISTENT DATASET, which will\n"; - print " then be CREATED BY the initial replication process.\n\n"; - } return 0; } @@ -1049,6 +1121,50 @@ sub getsnaps() { return %snaps; } +sub getbookmarks() { + my ($rhost,$fs,$isroot,%bookmarks) = @_; + my $mysudocmd; + my $fsescaped = escapeshellparam($fs); + if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } + + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } + + my $getbookmarkcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t bookmark guid,creation $fsescaped |"; + if ($debug) { print "DEBUG: getting list of bookmarks on $fs using $getbookmarkcmd...\n"; } + open FH, $getbookmarkcmd; + my @rawbookmarks = ; + close FH; + + # this is a little obnoxious. get guid,creation returns guid,creation on two separate lines + # as though each were an entirely separate get command. + + my $lastguid; + + foreach my $line (@rawbookmarks) { + # only import bookmark guids, creation from the specified filesystem + if ($line =~ /\Q$fs\E\#.*guid/) { + chomp $line; + $lastguid = $line; + $lastguid =~ s/^.*\tguid\t*(\d*).*/$1/; + my $bookmark = $line; + $bookmark =~ s/^.*\#(.*)\tguid.*$/$1/; + $bookmarks{$lastguid}{'name'}=$bookmark; + } elsif ($line =~ /\Q$fs\E\#.*creation/) { + chomp $line; + my $creation = $line; + $creation =~ s/^.*\tcreation\t*(\d*).*/$1/; + my $bookmark = $line; + $bookmark =~ s/^.*\#(.*)\tcreation.*$/$1/; + $bookmarks{$lastguid}{'creation'}=$creation; + } + } + + return %bookmarks; +} sub getsendsize { my ($sourcehost,$snap1,$snap2,$isroot,$receivetoken) = @_; From a1d9e79e70548b88e4f702d3790d9f5e7c028ffc Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 27 Jul 2018 22:58:42 +0200 Subject: [PATCH 071/148] added two tests for zfs bookmark replication --- .../run.sh | 56 +++++++++++++++++++ .../run.sh | 56 +++++++++++++++++++ tests/syncoid/run-tests.sh | 27 +++++++++ 3 files changed, 139 insertions(+) create mode 100755 tests/syncoid/1_bookmark_replication_intermediate/run.sh create mode 100755 tests/syncoid/2_bookmark_replication_no_intermediate/run.sh create mode 100755 tests/syncoid/run-tests.sh diff --git a/tests/syncoid/1_bookmark_replication_intermediate/run.sh b/tests/syncoid/1_bookmark_replication_intermediate/run.sh new file mode 100755 index 0000000..11edb04 --- /dev/null +++ b/tests/syncoid/1_bookmark_replication_intermediate/run.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# test replication with fallback to bookmarks and all intermediate snapshots + +set -x +set -e + +. ../../common/lib.sh + +POOL_IMAGE="/tmp/syncoid-test-1.zpool" +POOL_SIZE="200M" +POOL_NAME="syncoid-test-1" +TARGET_CHECKSUM="a23564d5bb8a2babc3ac8936fd82825ad9fff9c82d4924f5924398106bbda9f0 -" + +truncate -s "${POOL_SIZE}" "${POOL_IMAGE}" + +zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}" + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +zfs create "${POOL_NAME}"/src +zfs snapshot "${POOL_NAME}"/src@snap1 +zfs bookmark "${POOL_NAME}"/src@snap1 "${POOL_NAME}"/src#snap1 +# initial replication +../../../syncoid --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst +# destroy last common snapshot on source +zfs destroy "${POOL_NAME}"/src@snap1 + +# create intermediate snapshots +# sleep is needed so creation time can be used for proper sorting +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap2 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap3 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap4 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap5 + +# replicate which should fallback to bookmarks +../../../syncoid --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1 + +# verify +output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name) +checksum=$(echo "${output}" | grep -v syncoid_ | sha256sum) + +if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then + exit 1 +fi + +exit 0 diff --git a/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh b/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh new file mode 100755 index 0000000..94ac690 --- /dev/null +++ b/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# test replication with fallback to bookmarks and all intermediate snapshots + +set -x +set -e + +. ../../common/lib.sh + +POOL_IMAGE="/tmp/syncoid-test-2.zpool" +POOL_SIZE="200M" +POOL_NAME="syncoid-test-2" +TARGET_CHECKSUM="2460d4d4417793d2c7a5c72cbea4a8a584c0064bf48d8b6daa8ba55076cba66d -" + +truncate -s "${POOL_SIZE}" "${POOL_IMAGE}" + +zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}" + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +zfs create "${POOL_NAME}"/src +zfs snapshot "${POOL_NAME}"/src@snap1 +zfs bookmark "${POOL_NAME}"/src@snap1 "${POOL_NAME}"/src#snap1 +# initial replication +../../../syncoid --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst +# destroy last common snapshot on source +zfs destroy "${POOL_NAME}"/src@snap1 + +# create intermediate snapshots +# sleep is needed so creation time can be used for proper sorting +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap2 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap3 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap4 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap5 + +# replicate which should fallback to bookmarks +../../../syncoid --no-stream --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1 + +# verify +output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name) +checksum=$(echo "${output}" | sha256sum) + +if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then + exit 1 +fi + +exit 0 diff --git a/tests/syncoid/run-tests.sh b/tests/syncoid/run-tests.sh new file mode 100755 index 0000000..a9843a5 --- /dev/null +++ b/tests/syncoid/run-tests.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# run's all the available tests + +for test in */; do + if [ ! -x "${test}/run.sh" ]; then + continue + fi + + testName="${test%/}" + + LOGFILE=/tmp/syncoid_test_run_"${testName}".log + + pushd . > /dev/null + + echo -n "Running test ${testName} ... " + cd "${test}" + echo | bash run.sh > "${LOGFILE}" 2>&1 + + if [ $? -eq 0 ]; then + echo "[PASS]" + else + echo "[FAILED] (see ${LOGFILE})" + fi + + popd > /dev/null +done From 089516c58ec43c30342a45abebd60b60b94e41a4 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 29 Jul 2018 12:43:46 +0200 Subject: [PATCH 072/148] implemented force-delete flag to let syncoid destroy target datasets without matching snapshots/bookmarks --- syncoid | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index 3cbff6b..8554d43 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,8 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", + "force-delete") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -176,7 +177,7 @@ sub getchilddatasets { sub syncdataset { - my ($sourcehost, $sourcefs, $targethost, $targetfs) = @_; + my ($sourcehost, $sourcefs, $targethost, $targetfs, $skipsnapshot) = @_; my $sourcefsescaped = escapeshellparam($sourcefs); my $targetfsescaped = escapeshellparam($targetfs); @@ -228,7 +229,7 @@ sub syncdataset { print "\n\n\n"; } - if (!defined $args{'no-sync-snap'}) { + if (!defined $args{'no-sync-snap'} && !defined $skipsnapshot) { # create a new syncoid snapshot on the source filesystem. $newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot); } else { @@ -392,6 +393,30 @@ sub syncdataset { } if (! $bookmark) { + if ($args{'force-delete'}) { + if (!$quiet) { print "Removing $targetfs because no matching snapshots were found\n"; } + + my $rcommand = ''; + my $mysudocmd = ''; + my $targetfsescaped = escapeshellparam($targetfs); + + if ($targethost ne '') { $rcommand = "$sshcmd $targethost"; } + if (!$targetisroot) { $mysudocmd = $sudocmd; } + + my $prunecmd = "$mysudocmd $zfscmd destroy -r $targetfsescaped; "; + if ($targethost ne '') { + $prunecmd = escapeshellparam($prunecmd); + } + + my $ret = system("$rcommand $prunecmd"); + if ($ret != 0) { + warn "WARNING: $rcommand $prunecmd failed: $?"; + } else { + # redo sync and skip snapshot creation (already taken) + return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs, 1); + } + } + # if we got this far, we failed to find a matching snapshot/bookmark. print "\n"; From 8905c003355b27c5bd84ce4da74f834bef85df1c Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 29 Jul 2018 12:45:12 +0200 Subject: [PATCH 073/148] added test for force-delete flag --- tests/syncoid/3_force_delete/run.sh | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 tests/syncoid/3_force_delete/run.sh diff --git a/tests/syncoid/3_force_delete/run.sh b/tests/syncoid/3_force_delete/run.sh new file mode 100755 index 0000000..03ad9fa --- /dev/null +++ b/tests/syncoid/3_force_delete/run.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# test replication with deletion of target if no matches are found + +set -x +set -e + +. ../../common/lib.sh + +POOL_IMAGE="/tmp/syncoid-test-3.zpool" +POOL_SIZE="200M" +POOL_NAME="syncoid-test-3" +TARGET_CHECKSUM="0409a2ac216e69971270817189cef7caa91f6306fad9eab1033955b7e7c6bd4c -" + +truncate -s "${POOL_SIZE}" "${POOL_IMAGE}" + +zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}" + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +zfs create "${POOL_NAME}"/src +zfs create "${POOL_NAME}"/src/1 +zfs create "${POOL_NAME}"/src/2 +zfs create "${POOL_NAME}"/src/3 + +# initial replication +../../../syncoid -r --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst +# destroy last common snapshot on source +zfs destroy "${POOL_NAME}"/src/2@% +zfs snapshot "${POOL_NAME}"/src/2@test +sleep 1 +../../../syncoid -r --force-delete --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1 + +# verify +output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name | sed 's/@syncoid_.*$'/@syncoid_/) +checksum=$(echo "${output}" | sha256sum) + +if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then + exit 1 +fi + +exit 0 From 7c68ef5e8f2f491fe7de3b23a53f8b64a026a883 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 29 Jul 2018 13:16:53 +0200 Subject: [PATCH 074/148] return a non zero exit code if there was a problem replicating datasets --- syncoid | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index 6c569b2..6453d0f 100755 --- a/syncoid +++ b/syncoid @@ -97,6 +97,7 @@ my $targetsudocmd = $targetisroot ? '' : $sudocmd; my %avail = checkcommands(); my %snaps; +my $exitcode = 0; ## break here to call replication individually so that we ## ## can loop across children separately, for recursive ## @@ -127,7 +128,7 @@ if ($targethost ne '') { close FH; } -exit 0; +exit $exitcode; ############################################################################## ############################################################################## @@ -186,6 +187,7 @@ sub syncdataset { # make sure target is not currently in receive. if (iszfsbusy($targethost,$targetfs,$targetisroot)) { warn "Cannot sync now: $targetfs is already target of a zfs receive process.\n"; + if ($exitcode < 1) { $exitcode = 1; } return 0; } @@ -236,6 +238,7 @@ sub syncdataset { $newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot); if ($newsyncsnap eq 0) { warn "CRITICAL: no snapshots exist on source $sourcefs, and you asked for --no-sync-snap.\n"; + if ($exitcode < 1) { $exitcode = 1; } return 0; } } @@ -292,6 +295,7 @@ sub syncdataset { # make sure target is (still) not currently in receive. if (iszfsbusy($targethost,$targetfs,$targetisroot)) { warn "Cannot sync now: $targetfs is already target of a zfs receive process.\n"; + if ($exitcode < 1) { $exitcode = 1; } return 0; } system($synccmd) == 0 @@ -318,6 +322,7 @@ sub syncdataset { # make sure target is (still) not currently in receive. if (iszfsbusy($targethost,$targetfs,$targetisroot)) { warn "Cannot sync now: $targetfs is already target of a zfs receive process.\n"; + if ($exitcode < 1) { $exitcode = 1; } return 0; } @@ -325,9 +330,12 @@ sub syncdataset { if ($debug) { print "DEBUG: $synccmd\n"; } if ($oldestsnap ne $newsyncsnap) { - system($synccmd) == 0 - or warn "CRITICAL ERROR: $synccmd failed: $?"; + my $ret = system($synccmd); + if ($ret != 0) { + warn "CRITICAL ERROR: $synccmd failed: $?"; + if ($exitcode < 1) { $exitcode = 1; } return 0; + } } else { if (!$quiet) { print "INFO: no incremental sync needed; $oldestsnap is already the newest available snapshot.\n"; } } @@ -380,6 +388,7 @@ sub syncdataset { # make sure target is (still) not currently in receive. if (iszfsbusy($targethost,$targetfs,$targetisroot)) { warn "Cannot sync now: $targetfs is already target of a zfs receive process.\n"; + if ($exitcode < 1) { $exitcode = 1; } return 0; } @@ -910,6 +919,7 @@ sub getmatchingsnapshot { } # if we got this far, we failed to find a matching snapshot. + if ($exitcode < 2) { $exitcode = 2; } print "\n"; print "CRITICAL ERROR: Target $targetfs exists but has no snapshots matching with $sourcefs!\n"; From 1cb1209c785a43bd6d162d80ea75dc36f86a99b4 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 30 Jul 2018 09:13:26 +0200 Subject: [PATCH 075/148] documented force-delete flag --- README.md | 4 ++++ syncoid | 1 + 2 files changed, 5 insertions(+) diff --git a/README.md b/README.md index e359886..4755571 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This argument tells syncoid to not use resumeable zfs send/receive streams. ++ --force-delete + + Remove target datasets recursively (WARNING: this will also affect child datasets with matching snapshots/bookmarks), if there are no matching snapshots/bookmarks. + + --dumpsnaps This prints a list of snapshots during the run. diff --git a/syncoid b/syncoid index 8554d43..a6500bb 100755 --- a/syncoid +++ b/syncoid @@ -1341,3 +1341,4 @@ Options: --dumpsnaps Dumps a list of snapshots during the run --no-command-checks Do not check command existence before attempting transfer. Not recommended --no-resume Don't use the ZFS resume feature if available + --force-delete Remove target datasets recursively, if there are no matching snapshots/bookmarks From 020f57cb746ad7b5ed9a0fafa0a80b427cf6f3ae Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 30 Jul 2018 09:43:03 +0200 Subject: [PATCH 076/148] check for error while listing snapshots/bookmarks as this can be dangerous with force-delete --- syncoid | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index a6500bb..26307ba 100755 --- a/syncoid +++ b/syncoid @@ -1114,7 +1114,7 @@ sub getsnaps() { if ($debug) { print "DEBUG: getting list of snapshots on $fs using $getsnapcmd...\n"; } open FH, $getsnapcmd; my @rawsnaps = ; - close FH; + close FH or die "CRITICAL ERROR: snapshots couldn't be listed for $fs (exit code $?)"; # this is a little obnoxious. get guid,creation returns guid,creation on two separate lines # as though each were an entirely separate get command. @@ -1158,11 +1158,21 @@ sub getbookmarks() { $fsescaped = escapeshellparam($fsescaped); } - my $getbookmarkcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t bookmark guid,creation $fsescaped |"; + my $error = 0; + my $getbookmarkcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t bookmark guid,creation $fsescaped 2>&1 |"; if ($debug) { print "DEBUG: getting list of bookmarks on $fs using $getbookmarkcmd...\n"; } open FH, $getbookmarkcmd; my @rawbookmarks = ; - close FH; + close FH or $error = 1; + + if ($error == 1) { + if ($rawbookmarks[0] =~ /invalid type/) { + # no support for zfs bookmarks, return empty hash + return %bookmarks; + } + + die "CRITICAL ERROR: bookmarks couldn't be listed for $fs (exit code $?)"; + } # this is a little obnoxious. get guid,creation returns guid,creation on two separate lines # as though each were an entirely separate get command. From 63eec4994c20d7eb6207f6c6927badf138442f22 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 30 Jul 2018 22:21:14 +0200 Subject: [PATCH 077/148] don't die on some critical sync errors, but continue to replicate all the other datasets. after all is done exit with an error code --- syncoid | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/syncoid b/syncoid index 6453d0f..1ef1506 100755 --- a/syncoid +++ b/syncoid @@ -298,8 +298,11 @@ sub syncdataset { if ($exitcode < 1) { $exitcode = 1; } return 0; } - system($synccmd) == 0 - or die "CRITICAL ERROR: $synccmd failed: $?"; + system($synccmd) == 0 or do { + warn "CRITICAL ERROR: $synccmd failed: $?"; + if ($exitcode < 2) { $exitcode = 2; } + return 0; + }; # now do an -I to the new sync snapshot, assuming there were any snapshots # other than the new sync snapshot to begin with, of course - and that we @@ -359,8 +362,11 @@ sub syncdataset { if (!$quiet) { print "Resuming interrupted zfs send/receive from $sourcefs to $targetfs (~ $disp_pvsize remaining):\n"; } if ($debug) { print "DEBUG: $synccmd\n"; } - system("$synccmd") == 0 - or die "CRITICAL ERROR: $synccmd failed: $?"; + system("$synccmd") == 0 or do { + warn "CRITICAL ERROR: $synccmd failed: $?"; + if ($exitcode < 2) { $exitcode = 2; } + return 0; + }; # a resumed transfer will only be done to the next snapshot, # so do an normal sync cycle @@ -416,8 +422,11 @@ sub syncdataset { if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; } if ($debug) { print "DEBUG: $synccmd\n"; } - system("$synccmd") == 0 - or die "CRITICAL ERROR: $synccmd failed: $?"; + system("$synccmd") == 0 or do { + warn "CRITICAL ERROR: $synccmd failed: $?"; + if ($exitcode < 2) { $exitcode = 2; } + return 0; + }; # restore original readonly value to target after sync complete # dyking this functionality out for the time being due to buggy mount/unmount behavior From 9668567a870def5418032fb922d3a27a643059fa Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 30 Jul 2018 22:53:48 +0200 Subject: [PATCH 078/148] continue replication on more critical errors --- syncoid | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index 1ef1506..73205ce 100755 --- a/syncoid +++ b/syncoid @@ -233,6 +233,10 @@ sub syncdataset { if (!defined $args{'no-sync-snap'}) { # create a new syncoid snapshot on the source filesystem. $newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot); + if (!$newsyncsnap) { + # we already whined about the error + return 0; + } } else { # we don't want sync snapshots created, so use the newest snapshot we can find. $newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot); @@ -267,6 +271,11 @@ sub syncdataset { } my $oldestsnap = getoldestsnapshot(\%snaps); if (! $oldestsnap) { + if (defined ($args{'no-sync-snap'}) ) { + # we already whined about the missing snapshots + return 0; + } + # getoldestsnapshot() returned false, so use new sync snapshot if ($debug) { print "DEBUG: getoldestsnapshot() returned false, so using $newsyncsnap.\n"; } $oldestsnap = $newsyncsnap; @@ -752,7 +761,7 @@ sub getoldestsnapshot { # must not have had any snapshots on source - luckily, we already made one, amirite? if (defined ($args{'no-sync-snap'}) ) { # well, actually we set --no-sync-snap, so no we *didn't* already make one. Whoops. - die "CRIT: --no-sync-snap is set, and getoldestsnapshot() could not find any snapshots on source!\n"; + warn "CRIT: --no-sync-snap is set, and getoldestsnapshot() could not find any snapshots on source!\n"; } return 0; } @@ -774,6 +783,7 @@ sub getnewestsnapshot { # we also probably need an argument to mute this WARN, for people who deliberately exclude # datasets from recursive replication this way. warn "WARN: --no-sync-snap is set, and getnewestsnapshot() could not find any snapshots on source for current dataset. Continuing.\n"; + if ($exitcode < 2) { $exitcode = 2; } } return 0; } @@ -961,8 +971,12 @@ sub newsyncsnap { my %date = getdate(); my $snapname = "syncoid\_$identifier$hostid\_$date{'stamp'}"; my $snapcmd = "$rhost $mysudocmd $zfscmd snapshot $fsescaped\@$snapname\n"; - system($snapcmd) == 0 - or die "CRITICAL ERROR: $snapcmd failed: $?"; + system($snapcmd) == 0 or do { + warn "CRITICAL ERROR: $snapcmd failed: $?"; + if ($exitcode < 2) { $exitcode = 2; } + return 0; + }; + return $snapname; } From fb8edad885522040369c6ab95edfbb922fb6901d Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Fri, 3 Aug 2018 09:13:35 +0100 Subject: [PATCH 079/148] compression warnings are no longer hidden by --quiet --- syncoid | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index aec99cd..85d5e16 100755 --- a/syncoid +++ b/syncoid @@ -452,13 +452,13 @@ sub checkcommands { if ($avail{'sourcecompress'} eq '') { if ($compressargs{'rawcmd'} ne '') { - if (!$quiet) { print "WARN: $compressargs{'rawcmd'} not available on source $s- sync will continue without compression.\n"; } + print "WARN: $compressargs{'rawcmd'} not available on source $s- sync will continue without compression.\n"; } $avail{'compress'} = 0; } if ($avail{'targetcompress'} eq '') { if ($compressargs{'rawcmd'} ne '') { - if (!$quiet) { print "WARN: $compressargs{'rawcmd'} not available on target $t - sync will continue without compression.\n"; } + print "WARN: $compressargs{'rawcmd'} not available on target $t - sync will continue without compression.\n"; } $avail{'compress'} = 0; } @@ -471,7 +471,7 @@ sub checkcommands { # corner case - if source AND target are BOTH remote, we have to check for local compress too if ($sourcehost ne '' && $targethost ne '' && $avail{'localcompress'} eq '') { if ($compressargs{'rawcmd'} ne '') { - if (!$quiet) { print "WARN: $compressargs{'rawcmd'} not available on local machine - sync will continue without compression.\n"; } + print "WARN: $compressargs{'rawcmd'} not available on local machine - sync will continue without compression.\n"; } $avail{'compress'} = 0; } From 39d1fd38c1edd52f45b44707beabe32e3a20900d Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Fri, 5 Jan 2018 21:54:06 +0000 Subject: [PATCH 080/148] added ability to skip datasets... simply set syncoid:no-sync=true --- syncoid | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/syncoid b/syncoid index 6c569b2..8adaa87 100755 --- a/syncoid +++ b/syncoid @@ -183,6 +183,11 @@ sub syncdataset { if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; } + if (getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:no-sync') eq 'true') { + print "Skipping dataset (syncoid:nosync=true): $sourcefs...\n"; + return 0; + } + # make sure target is not currently in receive. if (iszfsbusy($targethost,$targetfs,$targetisroot)) { warn "Cannot sync now: $targetfs is already target of a zfs receive process.\n"; From 22c137627a9a923807565dff01794d9d3775e767 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Sat, 6 Jan 2018 10:15:30 +0000 Subject: [PATCH 081/148] updated doc for syncoid:no-sync --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index e359886..ba325e6 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,14 @@ If ZFS supports resumeable send/receive streams on both the source and target th As of 1.4.18, syncoid also automatically supports and enables resume of interrupted replication when both source and target support this feature. +##### Syncoid Dataset Properties + ++ syncoid:no-sync + + Setting this to `true` will prevent the dataset from being handled by syncoid in _any_ way - it will be skipped. This can be useful for preventing certain datasets from being transferred when recursively handling a tree. + + Note that this will also prevent syncoid from handling the dataset if given explicitly on the command line. + ##### Syncoid Command Line Options + [source] From b02b9a582bd3f92c01f16c954add64a78e647896 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Sat, 6 Jan 2018 10:43:30 +0000 Subject: [PATCH 082/148] now obeying --quiet, and added 'INFO: ' prefix to skip message --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 8adaa87..2201f41 100755 --- a/syncoid +++ b/syncoid @@ -184,7 +184,7 @@ sub syncdataset { if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; } if (getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:no-sync') eq 'true') { - print "Skipping dataset (syncoid:nosync=true): $sourcefs...\n"; + if (!$quiet) { print "INFO: Skipping dataset (syncoid:nosync=true): $sourcefs...\n"; } return 0; } From c188e47a6da2d8ee2857a8eb1c0400bc78fc7a53 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Fri, 9 Mar 2018 15:15:06 +0000 Subject: [PATCH 083/148] inverted sync/no-sync logic --- syncoid | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index 2201f41..20c0853 100755 --- a/syncoid +++ b/syncoid @@ -183,8 +183,8 @@ sub syncdataset { if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; } - if (getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:no-sync') eq 'true') { - if (!$quiet) { print "INFO: Skipping dataset (syncoid:nosync=true): $sourcefs...\n"; } + if (getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:sync') eq 'false') { + if (!$quiet) { print "INFO: Skipping dataset (syncoid:sync=false): $sourcefs...\n"; } return 0; } From f0a37310a54b5e421264d05a02ad226a1409bb98 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Fri, 9 Mar 2018 15:18:03 +0000 Subject: [PATCH 084/148] implemented sync true/false/${HOSTS} filtering --- syncoid | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 20c0853..7877723 100755 --- a/syncoid +++ b/syncoid @@ -183,9 +183,20 @@ sub syncdataset { if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; } - if (getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:sync') eq 'false') { + my $sync = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:sync'); + + if ($sync eq 'true' || $sync eq '-') { + # definitely sync this dataset - if a host is called 'true' or '-', then you're special + } elsif ($sync eq 'false') { if (!$quiet) { print "INFO: Skipping dataset (syncoid:sync=false): $sourcefs...\n"; } return 0; + } else { + my $hostid = hostname(); + my @hosts = split(/,/,$sync); + if (!(grep $hostid eq $_, @hosts)) { + if (!$quiet) { print "INFO: Skipping dataset (syncoid:sync doesn't include $hostid): $sourcefs...\n"; } + return 0; + } } # make sure target is not currently in receive. From a4e490f430b73454422c28254012a3328f9bbe98 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Fri, 9 Mar 2018 15:28:39 +0000 Subject: [PATCH 085/148] updated documentation for the syncoid:sync property's new behaviour --- README.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ba325e6..bb6ec89 100644 --- a/README.md +++ b/README.md @@ -120,11 +120,27 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup ##### Syncoid Dataset Properties -+ syncoid:no-sync ++ syncoid:sync - Setting this to `true` will prevent the dataset from being handled by syncoid in _any_ way - it will be skipped. This can be useful for preventing certain datasets from being transferred when recursively handling a tree. + Available values: - Note that this will also prevent syncoid from handling the dataset if given explicitly on the command line. + + `true` (default if unset) + + This dataset will be synchronised to all hosts. + + + `false` + + This dataset will not be synchronised to any hosts - it will be skipped. This can be useful for preventing certain datasets from being transferred when recursively handling a tree. + + + `host1,host2,...` + + A comma seperated list of hosts. This dataset will only be synchronised by hosts listed in the property. + + _Note_: this check is performed by the host running `syncoid`, thus the local hostname must be present for inclusion during a push operation // the remote hostname must be present for a pull. + + _Note_: this will also prevent syncoid from handling the dataset if given explicitly on the command line. + + _Note_: syncing a child of a no-sync dataset will currently result in a critical error ##### Syncoid Command Line Options From 60b2dedc456a8464f86cf75dacfe62c0b6410dbc Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Mon, 6 Aug 2018 14:07:58 +0100 Subject: [PATCH 086/148] fixed typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bb6ec89..8c05fb3 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + `host1,host2,...` - A comma seperated list of hosts. This dataset will only be synchronised by hosts listed in the property. + A comma separated list of hosts. This dataset will only be synchronised by hosts listed in the property. _Note_: this check is performed by the host running `syncoid`, thus the local hostname must be present for inclusion during a push operation // the remote hostname must be present for a pull. From d0ba0bc284bb92a8f0e63cfbd773e8c61e175973 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Mon, 6 Aug 2018 14:16:25 +0100 Subject: [PATCH 087/148] handled empty syncoid:sync property - behaves like unset --- README.md | 4 +++- syncoid | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8c05fb3..b833dec 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,9 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup _Note_: this will also prevent syncoid from handling the dataset if given explicitly on the command line. - _Note_: syncing a child of a no-sync dataset will currently result in a critical error + _Note_: syncing a child of a no-sync dataset will currently result in a critical error. + + _Note_: empty properties will be handled as if they were unset. ##### Syncoid Command Line Options diff --git a/syncoid b/syncoid index 7877723..fa00751 100755 --- a/syncoid +++ b/syncoid @@ -185,7 +185,8 @@ sub syncdataset { my $sync = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:sync'); - if ($sync eq 'true' || $sync eq '-') { + if ($sync eq 'true' || $sync eq '-' || $sync eq '') { + # empty is handled the same as unset (aka: '-') # definitely sync this dataset - if a host is called 'true' or '-', then you're special } elsif ($sync eq 'false') { if (!$quiet) { print "INFO: Skipping dataset (syncoid:sync=false): $sourcefs...\n"; } From f51bb9db7e44a0d512c29c8ba01b0ad2265d237c Mon Sep 17 00:00:00 2001 From: Martin Schrodt Date: Wed, 15 Aug 2018 16:07:01 +0200 Subject: [PATCH 088/148] add xz compression --- syncoid | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index b62eee8..18708d2 100755 --- a/syncoid +++ b/syncoid @@ -509,6 +509,12 @@ sub compressargset { decomrawcmd => '/usr/bin/zstd', decomargs => '-dc', }, + 'xz' => { + rawcmd => '/usr/bin/xz', + args => '', + decomrawcmd => '/usr/bin/xz', + decomargs => '-d', + }, 'lzo' => { rawcmd => '/usr/bin/lzop', args => '', @@ -519,7 +525,7 @@ sub compressargset { if ($value eq 'default') { $value = $DEFAULT_COMPRESSION; - } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lzo', 'default', 'none'))) { + } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lzo', 'xz', 'default', 'none'))) { warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION"; $value = $DEFAULT_COMPRESSION; } @@ -1227,7 +1233,7 @@ syncoid - ZFS snapshot replication tool Options: - --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none + --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, xz, lzo (default) & none --identifier=EXTRA Extra identifier which is included in the snapshot name. Can be used for replicating to multiple targets. --recursive|r Also transfers child datasets --skip-parent Skips syncing of the parent dataset. Does nothing without '--recursive' option. From 0deaacfc06bdb891525fb1bdc1ed34abcb91b9c1 Mon Sep 17 00:00:00 2001 From: Martin Schrodt Date: Wed, 15 Aug 2018 22:21:56 +0200 Subject: [PATCH 089/148] add xz to README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b833dec..dc1162b 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --compress - Currently accepted options: gzip, pigz-fast, pigz-slow, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. + Currently accepted options: gzip, pigz-fast, pigz-slow, xz, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. + --source-bwlimit From 8de3cdce212ae15a849768e824d929afd6bc1501 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 3 Sep 2018 17:45:03 +0200 Subject: [PATCH 090/148] let monitor-health also check vdev member io/checksum errors --- sanoid | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) mode change 100755 => 100644 sanoid diff --git a/sanoid b/sanoid old mode 100755 new mode 100644 index 485ee08..7ae1b5b --- a/sanoid +++ b/sanoid @@ -974,7 +974,7 @@ sub check_zpool() { } ## other cases - my ($dev, $sta) = /^\s+(\S+)\s+(\S+)/; + my ($dev, $sta, $read, $write, $cksum) = /^\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/; if (!defined($sta)) { # cache and logs are special and don't have a status @@ -994,8 +994,21 @@ sub check_zpool() { ## no display for verbose level 1 next if ($verbose==1); ## don't display working devices for verbose level 2 - next if ($verbose==2 && $state eq "OK"); - next if ($verbose==2 && ($sta eq "ONLINE" || $sta eq "AVAIL" || $sta eq "INUSE")); + if ($verbose==2 && ($state eq "OK" || $sta eq "ONLINE" || $sta eq "AVAIL" || $sta eq "INUSE")) { + # check for io/checksum errors + + my @vdeverr = (); + if ($read != 0) { push @vdeverr, "read" }; + if ($write != 0) { push @vdeverr, "write" }; + if ($cksum != 0) { push @vdeverr, "cksum" }; + + if (scalar @vdeverr) { + $dmge=$dmge . "(" . $dev . ":" . join(", ", @vdeverr) . " errors) "; + if ($state eq "OK") { $state = "WARNING" }; + } + + next; + } ## show everything else if (/^\s{3}(\S+)/) { From 997487d12bf0f7d725199fb9fb9dd5cb4e484504 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 3 Sep 2018 17:46:21 +0200 Subject: [PATCH 091/148] restore filemode --- sanoid | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 sanoid diff --git a/sanoid b/sanoid old mode 100644 new mode 100755 From f39ed1ec49e989aa4fc7ae8c79f6329f5764cf3b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 3 Sep 2018 18:32:17 +0200 Subject: [PATCH 092/148] for remote target/source it's required to specify a user --- syncoid | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index b62eee8..30aef49 100755 --- a/syncoid +++ b/syncoid @@ -1218,9 +1218,9 @@ syncoid - ZFS snapshot replication tool =head1 SYNOPSIS syncoid [options]... SOURCE TARGET - or syncoid [options]... SOURCE [USER@]HOST:TARGET - or syncoid [options]... [USER@]HOST:SOURCE [TARGET] - or syncoid [options]... [USER@]HOST:SOURCE [USER@]HOST:TARGET + or syncoid [options]... SOURCE USER@HOST:TARGET + or syncoid [options]... USER@HOST:SOURCE TARGET + or syncoid [options]... USER@HOST:SOURCE USER@HOST:TARGET SOURCE Source ZFS dataset. Can be either local or remote TARGET Target ZFS dataset. Can be either local or remote From 03a074e5c286d77d049df9555cabd3d2cc71d4a2 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 4 Sep 2018 08:13:47 +0200 Subject: [PATCH 093/148] show warning if a key is ignored from a template --- sanoid | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/sanoid b/sanoid index 485ee08..5764849 100755 --- a/sanoid +++ b/sanoid @@ -691,10 +691,12 @@ sub init { # override with values from user-defined default template, if any foreach my $key (keys %{$ini{'template_default'}}) { - if (! ($key =~ /template|recursive/)) { - if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value from user-defined default template.\n"; } - $config{$section}{$key} = $ini{'template_default'}{$key}; + if ($key =~ /template|recursive/) { + warn "ignored key '$key' from user-defined default template.\n"; + next; } + if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value from user-defined default template.\n"; } + $config{$section}{$key} = $ini{'template_default'}{$key}; } } @@ -708,10 +710,12 @@ sub init { my $template = 'template_'.$rawtemplate; foreach my $key (keys %{$ini{$template}}) { - if (! ($key =~ /template|recursive/)) { - if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value from user-defined template $template.\n"; } - $config{$section}{$key} = $ini{$template}{$key}; + if ($key =~ /template|recursive/) { + warn "ignored key '$key' from '$rawtemplate' template.\n"; + next; } + if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value from user-defined template $template.\n"; } + $config{$section}{$key} = $ini{$template}{$key}; } } } From 65c7be5b1d58287e0fa637c01b6b8265e350fe2a Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 4 Sep 2018 20:17:46 +0200 Subject: [PATCH 094/148] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b833dec..0354c6b 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ And its /etc/sanoid/sanoid.conf might look something like this: autoprune = yes ``` -Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 dailies, 3 monthlies, and no yearlies for all datasets under data/images (but not data/images itself, since process_children_only is set). Except in the case of data/images/win7-spice, which follows the same template (since it's a child of data/images) but only keeps 4 hourlies for whatever reason. +Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 dailies, 3 monthlies, and no yearlies for all datasets under data/images (but not data/images itself, since process_children_only is set). Except in the case of data/images/win7, which follows the same template (since it's a child of data/images) but only keeps 4 hourlies for whatever reason. ##### Sanoid Command Line Options From 2fe97f13ad29ed9f07000475bb0e9aa1430e95f7 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 10:07:05 +0200 Subject: [PATCH 095/148] prepare for v1.4.19 release --- CHANGELIST | 23 +++++++++++++++++++++++ VERSION | 2 +- packages/debian/changelog | 27 +++++++++++++++++++++++++++ packages/rhel/sanoid.spec | 6 ++++-- sanoid | 2 +- syncoid | 2 +- 6 files changed, 57 insertions(+), 5 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 515d05d..3ef74ae 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,26 @@ +1.4.19 [sanoid] monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0) + [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie) + [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0) + [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0) + [syncoid] make local source bwlimit work (@phreaker0) + [syncoid] fix 'resume support' detection on FreeBSD (@pit3k) + [sanoid] updated INSTALL with missing dependency + [sanoid] fixed monitor-health command for pools containing cache and log devices (@phreaker0) + [sanoid] quiet flag suppresses all info output (@martinvw) + [sanoid] check for empty lockfile which lead to sanoid failing on start (@jasonblewis) + [sanoid] added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0) + [sanoid] cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0) + [sanoid] implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0) + [syncoid] Added support for ZStandard compression.(@danielewood) + [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0) + [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0) + [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0) + [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0) + [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0) + [sanoid] use UTC by default in unit template and documentation (@phreaker0) + [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0) + [syncoid] documented compatibility issues with (t)csh shells (@ecoutu) + 1.4.18 implemented special character handling and support of ZFS resume/receive tokens by default in syncoid, thank you @phreaker0! diff --git a/VERSION b/VERSION index f689e8c..fd4ca57 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.18 +1.4.19 diff --git a/packages/debian/changelog b/packages/debian/changelog index 2bcf423..67cca4d 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -1,3 +1,30 @@ +sanoid (1.4.19) unstable; urgency=medium + + [sanoid] monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0) + [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie) + [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0) + [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0) + [syncoid] make local source bwlimit work (@phreaker0) + [syncoid] fix 'resume support' detection on FreeBSD (@pit3k) + [sanoid] updated INSTALL with missing dependency + [sanoid] fixed monitor-health command for pools containing cache and log devices (@phreaker0) + [sanoid] quiet flag suppresses all info output (@martinvw) + [sanoid] check for empty lockfile which lead to sanoid failing on start (@jasonblewis) + [sanoid] added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0) + [sanoid] cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0) + [sanoid] implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0) + [syncoid] Added support for ZStandard compression.(@danielewood) + [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0) + [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0) + [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0) + [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0) + [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0) + [sanoid] use UTC by default in unit template and documentation (@phreaker0) + [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0) + [syncoid] documented compatibility issues with (t)csh shells (@ecoutu) + + -- Jim Salter Wed, 05 Sep 2018 04:00:00 -0400 + sanoid (1.4.18) unstable; urgency=medium implemented special character handling and support of ZFS resume/receive tokens by default in syncoid, diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index 3a9412f..4971404 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -1,4 +1,4 @@ -%global version 1.4.18 +%global version 1.4.19 %global git_tag v%{version} # Enable with systemctl "enable sanoid.timer" @@ -12,7 +12,7 @@ Summary: A policy-driven snapshot management tool for ZFS file systems Group: Applications/System License: GPLv3 URL: https://github.com/jimsalterjrs/sanoid -Source0: https://github.com/jimsalterjrs/%{name}/archive/%{git_tag}/%{name}-%{version}.tar.gz +Source0: https://github.com/jimsalterjrs/%{name}/archive/%{git_tag}/%{name}-%{version}.tar.gz Requires: perl, mbuffer, lzop, pv %if 0%{?_with_systemd} @@ -111,6 +111,8 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name} %endif %changelog +* Wed Sep 05 2018 Christoph Klaffl - 1.4.19 +- Bump to 1.4.19 * Sat Apr 28 2018 Dominic Robinson - 1.4.18-1 - Bump to 1.4.18 * Thu Aug 31 2017 Dominic Robinson - 1.4.14-2 diff --git a/sanoid b/sanoid index 7ae1b5b..fd68145 100755 --- a/sanoid +++ b/sanoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '1.4.18'; +$::VERSION = '1.4.19'; use strict; use warnings; diff --git a/syncoid b/syncoid index 30aef49..8aa5205 100755 --- a/syncoid +++ b/syncoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '1.4.18'; +$::VERSION = '1.4.19'; use strict; use warnings; From 055f26b9709aa412b2597e53c99e00c09cf5f5df Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 18:47:16 +0200 Subject: [PATCH 096/148] ignore unknown interval type to prevent perl warnings --- sanoid | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sanoid b/sanoid index 7ae1b5b..31376ad 100755 --- a/sanoid +++ b/sanoid @@ -438,6 +438,9 @@ sub take_snapshots { push @preferredtime,$datestamp{'year'}; $lastpreferred = timelocal(@preferredtime); if ($lastpreferred > time()) { $lastpreferred -= 60*60*24*31*365.25; } # preferred time is later this year - so look at last year + } else { + # unknown type + next; } # reconstruct our human-formatted most recent preferred snapshot time into an epoch time, to compare with the epoch of our most recent snapshot From d5f4d1c121e40da89fcca720ffd55332c3847b04 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 19:01:15 +0200 Subject: [PATCH 097/148] add lz4 compression --- README.md | 2 +- syncoid | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b833dec..ce09a37 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --compress - Currently accepted options: gzip, pigz-fast, pigz-slow, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. + Currently accepted options: gzip, pigz-fast, pigz-slow, zstd-fast, zstd-slow, lz4, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. + --source-bwlimit diff --git a/syncoid b/syncoid index 30aef49..c42a008 100755 --- a/syncoid +++ b/syncoid @@ -515,11 +515,17 @@ sub compressargset { decomrawcmd => '/usr/bin/lzop', decomargs => '-dfc', }, + 'lz4' => { + rawcmd => '/usr/bin/lz4', + args => '', + decomrawcmd => '/usr/bin/lz4', + decomargs => '-dc', + }, ); if ($value eq 'default') { $value = $DEFAULT_COMPRESSION; - } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lzo', 'default', 'none'))) { + } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lz4', 'lzo', 'default', 'none'))) { warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION"; $value = $DEFAULT_COMPRESSION; } @@ -1227,7 +1233,7 @@ syncoid - ZFS snapshot replication tool Options: - --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none + --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, zstd-fast, zstd-slow, lz4, lzo (default) & none --identifier=EXTRA Extra identifier which is included in the snapshot name. Can be used for replicating to multiple targets. --recursive|r Also transfers child datasets --skip-parent Skips syncing of the parent dataset. Does nothing without '--recursive' option. From 4af294838256fee6cee00d034bd75936992063d3 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 19:18:21 +0200 Subject: [PATCH 098/148] fix uninitialized value warning in debug mode on initial run (no snapshots yet) --- sanoid | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/sanoid b/sanoid index 7ae1b5b..0af79a4 100755 --- a/sanoid +++ b/sanoid @@ -492,16 +492,20 @@ sub blabber { my $path = $config{$section}{'path'}; print "Filesystem $path has:\n"; print " $snapsbypath{$path}{'numsnaps'} total snapshots "; - print "(newest: "; - my $newest = sprintf("%.1f",$snapsbypath{$path}{'newest'} / 60 / 60); - print "$newest hours old)\n"; + if ($snapsbypath{$path}{'numsnaps'} == 0) { + print "(no current snapshots)" + } else { + print "(newest: "; + my $newest = sprintf("%.1f",$snapsbypath{$path}{'newest'} / 60 / 60); + print "$newest hours old)\n"; - foreach my $type (keys %{ $snapsbytype{$path} }){ - print " $snapsbytype{$path}{$type}{'numsnaps'} $type\n"; - print " desired: $config{$section}{$type}\n"; - print " newest: "; - my $newest = sprintf("%.1f",($snapsbytype{$path}{$type}{'newest'} / 60 / 60)); - print "$newest hours old, named $snapsbytype{$path}{$type}{'newestname'}\n"; + foreach my $type (keys %{ $snapsbytype{$path} }){ + print " $snapsbytype{$path}{$type}{'numsnaps'} $type\n"; + print " desired: $config{$section}{$type}\n"; + print " newest: "; + my $newest = sprintf("%.1f",($snapsbytype{$path}{$type}{'newest'} / 60 / 60)); + print "$newest hours old, named $snapsbytype{$path}{$type}{'newestname'}\n"; + } } print "\n\n"; } From 253b8b467d2de2962826842052645d4439c0fd40 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 19:34:44 +0200 Subject: [PATCH 099/148] parse values for recursive key like the others booleans, previously any non empty value was considered as true --- sanoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 7ae1b5b..e5d3e05 100755 --- a/sanoid +++ b/sanoid @@ -743,7 +743,7 @@ sub init { # how 'bout some recursion? =) my @datasets; - if ($ini{$section}{'recursive'}) { + if (grep( /^$ini{$section}{'recursive'}$/, @istrue )) { @datasets = getchilddatasets($config{$section}{'path'}); foreach my $dataset(@datasets) { chomp $dataset; From 807fc53afb4d28b3742af506d8e3f9b0ee2a4337 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 22:46:14 +0200 Subject: [PATCH 100/148] added all available syncoid/sanoid parameters to README --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index b833dec..9fcfcf6 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,13 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da This prints out quite alot of additional information during a sanoid run, and is normally not needed. ++ --readonly + + Skip creation/deletion of snapshots (Simulate). + ++ --help + + Show help message. ---------- @@ -206,6 +213,14 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup Allow sync to/from boxes running SSH on non-standard ports. ++ --sshcipher + + Instruct ssh to use a particular cipher set. + ++ --sshoption + + Passes option to ssh. This argument can be specified multiple times. + + --sshkey Use specified identity file as per ssh -i. @@ -218,6 +233,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This prints out quite alot of additional information during a sanoid run, and is normally not needed. ++ --help + + Show help message. + + --version Print the version and exit. From e7bd567acb1aed210f7d36cd9933460cab080da2 Mon Sep 17 00:00:00 2001 From: Andrew DeMaria Date: Mon, 10 Sep 2018 22:14:05 -0400 Subject: [PATCH 101/148] Do not monitor snapshots types that are set to 0. Signed-off-by: Andrew DeMaria --- sanoid | 1 + 1 file changed, 1 insertion(+) diff --git a/sanoid b/sanoid index 7ae1b5b..aa6369d 100755 --- a/sanoid +++ b/sanoid @@ -127,6 +127,7 @@ sub monitor_snapshots { my @types = ('yearly','monthly','daily','hourly'); foreach my $type (@types) { + if ($config{$section}{$type} == 0) { next; } my $smallerperiod = 0; # we need to set the period length in seconds first From 108fe5e2fc1d31a1f74e3dfa0448814de3af6ee3 Mon Sep 17 00:00:00 2001 From: Gionatan Danti Date: Wed, 26 Sep 2018 14:02:19 +0200 Subject: [PATCH 102/148] Reversed zfsisbusy and force-prune checks --- sanoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanoid b/sanoid index fcbb630..32651ab 100755 --- a/sanoid +++ b/sanoid @@ -293,7 +293,7 @@ sub prune_snapshots { writelock('sanoid_pruning'); foreach my $snap( @prunesnaps ){ if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; } - if (iszfsbusy($path) && !$args{'force-prune'}) { + if (!$args{'force-prune'} && iszfsbusy($path)) { if ($args{'verbose'}) { print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; } } else { if (! $args{'readonly'}) { From f04be06f392911a2044d893065ca52b3e7cf99ea Mon Sep 17 00:00:00 2001 From: Gionatan Danti Date: Wed, 26 Sep 2018 14:24:38 +0200 Subject: [PATCH 103/148] Fixed indentation --- sanoid | 2 +- syncoid | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index 32651ab..7a5fdee 100755 --- a/sanoid +++ b/sanoid @@ -1323,7 +1323,7 @@ Options: --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format --take-snapshots Creates snapshots as specified in sanoid.conf --prune-snapshots Purges expired snapshots as specified in sanoid.conf - --force-prune Purges expired snapshots even if a send/recv is in progress + --force-prune Purges expired snapshots even if a send/recv is in progress --help Prints this helptext --version Prints the version number diff --git a/syncoid b/syncoid index f2335dc..8e67f5e 100755 --- a/syncoid +++ b/syncoid @@ -1246,7 +1246,7 @@ Options: --target-bwlimit= Bandwidth limit on the target transfer --no-stream Replicates using newest snapshot instead of intermediates --no-sync-snap Does not create new snapshot, only transfers existing - --no-clone-rollback Does not rollback clones on target + --no-clone-rollback Does not rollback clones on target --no-rollback Does not rollback clones or snapshots on target (it probably requires a readonly target) --exclude=REGEX Exclude specific datasets which match the given regular expression. Can be specified multiple times --sshkey=FILE Specifies a ssh public key to use to connect From e9a330f89a8ed9894df7d1a7bf6a996bcec3918f Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 5 Oct 2018 16:19:55 +0200 Subject: [PATCH 104/148] adapt test as it works correctly know after the DST patch --- tests/1_one_year/run.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/1_one_year/run.sh b/tests/1_one_year/run.sh index 7cec813..1cae7b4 100755 --- a/tests/1_one_year/run.sh +++ b/tests/1_one_year/run.sh @@ -10,7 +10,7 @@ set -x POOL_NAME="sanoid-test-1" POOL_TARGET="" # root RESULT="/tmp/sanoid_test_result" -RESULT_CHECKSUM="aa15e5595b0ed959313289ecb70323dad9903328ac46e881da5c4b0f871dd7cf" +RESULT_CHECKSUM="68c67161a59d0e248094a66061972f53613067c9db52ad981030f36bc081fed7" # UTC timestamp of start and end START="1483225200" @@ -46,10 +46,4 @@ done saveSnapshotList "${POOL_NAME}" "${RESULT}" # hourly daily monthly -verifySnapshotList "${RESULT}" 8759 366 12 "${RESULT_CHECKSUM}" - -# hourly count should be 8760 but one hour get's lost because of DST - -# daily count should be 365 but one additional daily is taken -# because the DST change leads to a day with 25 hours -# which will trigger an additional daily snapshot +verifySnapshotList "${RESULT}" 8760 365 12 "${RESULT_CHECKSUM}" From e83ec060fb68ad47545e24cde71e60c3d63f2752 Mon Sep 17 00:00:00 2001 From: Michael Bushey Date: Sun, 14 Oct 2018 15:10:25 -0700 Subject: [PATCH 105/148] INSTALL: Fix name p5-Config-Inifiles -> p5-Config-IniFiles --- INSTALL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL b/INSTALL index f0de17b..15c4896 100644 --- a/INSTALL +++ b/INSTALL @@ -30,4 +30,4 @@ strongly recommends using your distribution's repositories instead. On Ubuntu: apt install libconfig-inifiles-perl On CentOS: yum install perl-Config-IniFiles -On FreeBSD: pkg install p5-Config-Inifiles +On FreeBSD: pkg install p5-Config-IniFiles From c6ffbf5c4c05e8805a441656b253836631272221 Mon Sep 17 00:00:00 2001 From: Julien Riou Date: Sat, 13 Jan 2018 14:26:26 +0100 Subject: [PATCH 106/148] Add pre and post snapshot scripts --- sanoid | 25 ++++++++++++++++++++++++- sanoid.conf | 11 +++++++++++ sanoid.defaults.conf | 4 ++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 7ae1b5b..e45133b 100755 --- a/sanoid +++ b/sanoid @@ -455,6 +455,18 @@ sub take_snapshots { if ( (scalar(@newsnaps)) > 0) { foreach my $snap ( @newsnaps ) { + my $dataset = (split '@', $snap)[0]; + my $presnapshotfailure = 0; + if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) { + $ENV{'SANOID_TARGET'} = $dataset; + if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; } + if (system($config{$dataset}{'pre_snapshot_script'}) != 0) { + warn "WARN: pre_snapshot_script failed, $?"; + $config{$dataset}{'no_inconsistent_snapshot'} and next; + $presnapshotfailure = 1; + } + delete $ENV{'SANOID_TARGET'}; + } if ($args{'verbose'}) { print "taking snapshot $snap\n"; } if (!$args{'readonly'}) { system($zfs, "snapshot", "$snap") == 0 @@ -462,6 +474,17 @@ sub take_snapshots { # make sure we don't end up with multiple snapshots with the same ctime sleep 1; } + if ($config{$dataset}{'post_snapshot_script'} and !$args{'readonly'}) { + if (!$presnapshotfailure or $config{$dataset}{'force_post_snapshot_script'}) { + $ENV{'SANOID_TARGET'} = $dataset; + if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; } + if (system($config{$dataset}{'post_snapshot_script'}) != 0) { + warn "WARN: post_snapshot_script failed, $?"; + $config{$dataset}{'no_inconsistent_snapshot'} and next; + } + delete $ENV{'SANOID_TARGET'}; + } + } } $forcecacheupdate = 1; %snaps = getsnaps(%config,$cacheTTL,$forcecacheupdate); @@ -661,7 +684,7 @@ sub init { tie my %ini, 'Config::IniFiles', ( -file => $conf_file ) or die "FATAL: cannot load $conf_file - please create a valid local config file before running sanoid!"; # we'll use these later to normalize potentially true and false values on any toggle keys - my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only'); + my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only','no_inconsistent_snapshot','force_post_snapshot_script'); my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON"); my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF"); diff --git a/sanoid.conf b/sanoid.conf index 9b1f19d..218a492 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -67,6 +67,17 @@ daily_warn = 48 daily_crit = 60 +[template_scripts] + ### run script before snapshot + ### dataset name will be supplied as an environment variable $SANOID_TARGET + pre_snapshot_script = /path/to/script.sh + ### run script after snapshot + ### dataset name will be supplied as an environment variable $SANOID_TARGET + post_snapshot_script = /path/to/script.sh + ### don't take an inconsistent snapshot + #no_inconsistent_snapshot = yes + ### run post_snapshot_script when pre_snapshot_script is failing + #force_post_snapshot_script = yes [template_ignore] autoprune = no diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index d86cc47..12a8049 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -15,6 +15,10 @@ path = recursive = use_template = process_children_only = +pre_snapshot_script = +post_snapshot_script = +no_inconsistent_snapshot = +force_post_snapshot_script = # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately # prune any of those type snapshots already present. From 84213216ec2d98182e31c7e43ecce48cd353404a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Belli?= Date: Tue, 2 Oct 2018 00:47:25 +0200 Subject: [PATCH 107/148] Expose snapshot name through ENV variable --- sanoid | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sanoid b/sanoid index e45133b..73efd5e 100755 --- a/sanoid +++ b/sanoid @@ -456,9 +456,11 @@ sub take_snapshots { if ( (scalar(@newsnaps)) > 0) { foreach my $snap ( @newsnaps ) { my $dataset = (split '@', $snap)[0]; + my $snapname = (split '@', $snap)[1]; my $presnapshotfailure = 0; if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) { $ENV{'SANOID_TARGET'} = $dataset; + $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; } if (system($config{$dataset}{'pre_snapshot_script'}) != 0) { warn "WARN: pre_snapshot_script failed, $?"; @@ -466,6 +468,7 @@ sub take_snapshots { $presnapshotfailure = 1; } delete $ENV{'SANOID_TARGET'}; + delete $ENV{'SANOID_SNAPNAME'}; } if ($args{'verbose'}) { print "taking snapshot $snap\n"; } if (!$args{'readonly'}) { @@ -477,12 +480,14 @@ sub take_snapshots { if ($config{$dataset}{'post_snapshot_script'} and !$args{'readonly'}) { if (!$presnapshotfailure or $config{$dataset}{'force_post_snapshot_script'}) { $ENV{'SANOID_TARGET'} = $dataset; + $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; } if (system($config{$dataset}{'post_snapshot_script'}) != 0) { warn "WARN: post_snapshot_script failed, $?"; $config{$dataset}{'no_inconsistent_snapshot'} and next; } delete $ENV{'SANOID_TARGET'}; + delete $ENV{'SANOID_SNAPNAME'}; } } } From fb6608bf47a9508fb3b661a364e597b94b517e5f Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 16 Oct 2018 10:48:04 +0200 Subject: [PATCH 108/148] implemented timeout for pre/post script execution and made sure environment is cleaned up after script failure --- sanoid | 50 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/sanoid b/sanoid index 73efd5e..c12dff8 100755 --- a/sanoid +++ b/sanoid @@ -458,17 +458,21 @@ sub take_snapshots { my $dataset = (split '@', $snap)[0]; my $snapname = (split '@', $snap)[1]; my $presnapshotfailure = 0; + my $timeout = 5; if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; } - if (system($config{$dataset}{'pre_snapshot_script'}) != 0) { - warn "WARN: pre_snapshot_script failed, $?"; + my $ret = runscript('pre_snapshot_script',$dataset,$timeout); + + delete $ENV{'SANOID_TARGET'}; + delete $ENV{'SANOID_SNAPNAME'}; + + if ($ret != 0) { + # warning was already thrown by runscript function $config{$dataset}{'no_inconsistent_snapshot'} and next; $presnapshotfailure = 1; } - delete $ENV{'SANOID_TARGET'}; - delete $ENV{'SANOID_SNAPNAME'}; } if ($args{'verbose'}) { print "taking snapshot $snap\n"; } if (!$args{'readonly'}) { @@ -482,10 +486,8 @@ sub take_snapshots { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; } - if (system($config{$dataset}{'post_snapshot_script'}) != 0) { - warn "WARN: post_snapshot_script failed, $?"; - $config{$dataset}{'no_inconsistent_snapshot'} and next; - } + runscript('post_snapshot_script',$dataset,$timeout); + delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; } @@ -1337,6 +1339,38 @@ sub removecachedsnapshots { undef %pruned; } +#######################################################################################################################3 +#######################################################################################################################3 +#######################################################################################################################3 + +sub runscript { + my $key=shift; + my $dataset=shift; + my $timeout=shift; + + my $ret; + eval { + local $SIG{ALRM} = sub { die "alarm\n" }; + alarm $timeout; + $ret = system($config{$dataset}{$key}); + alarm 0; + }; + if ($@) { + if ($@ eq "alarm\n") { + warn "WARN: $key didn't finish in the allowed time!"; + } else { + warn "CRITICAL ERROR: $@"; + } + return -1; + } else { + if ($ret != 0) { + warn "WARN: $key failed, $?"; + } + } + + return $ret; +} + __END__ =head1 NAME From 0a7fdcb232d5e75eed388f016060cd4736c6185a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Belli?= Date: Sun, 14 Oct 2018 16:28:24 +0200 Subject: [PATCH 109/148] Add pruning hooks --- sanoid | 11 +++++++++++ sanoid.conf | 2 ++ sanoid.defaults.conf | 1 + 3 files changed, 14 insertions(+) diff --git a/sanoid b/sanoid index c12dff8..866eef0 100755 --- a/sanoid +++ b/sanoid @@ -299,6 +299,17 @@ sub prune_snapshots { if (! $args{'readonly'}) { if (system($zfs, "destroy", $snap) == 0) { $pruned{$snap} = 1; + my $dataset = (split '@', $snap)[0]; + my $snapname = (split '@', $snap)[1]; + if ($config{$dataset}{'pruning_script'}) { + $ENV{'SANOID_TARGET'} = $dataset; + $ENV{'SANOID_SNAPNAME'} = $snapname; + if ($args{'verbose'}) { print "executing pruning_script '".$config{$dataset}{'pruning_script'}."' on dataset '$dataset'\n"; } + system($config{$dataset}{'pruning_script'}) == 0 + or warn "WARN: pruning_script failed, $?"; + delete $ENV{'SANOID_TARGET'}; + delete $ENV{'SANOID_SNAPNAME'}; + } } else { warn "could not remove $snap : $?"; } diff --git a/sanoid.conf b/sanoid.conf index 218a492..e684614 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -78,6 +78,8 @@ #no_inconsistent_snapshot = yes ### run post_snapshot_script when pre_snapshot_script is failing #force_post_snapshot_script = yes + ### dataset name will be supplied as an environment variable $SANOID_TARGET + pruning_script = /path/to/script.sh [template_ignore] autoprune = no diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 12a8049..d4dd19e 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -19,6 +19,7 @@ pre_snapshot_script = post_snapshot_script = no_inconsistent_snapshot = force_post_snapshot_script = +pruning_script = # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately # prune any of those type snapshots already present. From a7b7fe8d15adc826d78f914f962b0b6929df88cb Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 16 Oct 2018 11:58:25 +0200 Subject: [PATCH 110/148] let pruning script timeout so it doesn't hang sanoid --- sanoid | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index 866eef0..7c213e4 100755 --- a/sanoid +++ b/sanoid @@ -302,11 +302,12 @@ sub prune_snapshots { my $dataset = (split '@', $snap)[0]; my $snapname = (split '@', $snap)[1]; if ($config{$dataset}{'pruning_script'}) { + my $timeout = 5; $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pruning_script '".$config{$dataset}{'pruning_script'}."' on dataset '$dataset'\n"; } - system($config{$dataset}{'pruning_script'}) == 0 - or warn "WARN: pruning_script failed, $?"; + my $ret = runscript('pruning_script',$dataset,$timeout); + delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; } From a8d5c5652a82c505b117ccd02a39962be240bd10 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 16 Oct 2018 17:54:37 +0200 Subject: [PATCH 111/148] make script timeout configureable --- sanoid | 16 +++++++++------- sanoid.conf | 6 ++++-- sanoid.defaults.conf | 3 ++- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/sanoid b/sanoid index 7c213e4..9c0e54d 100755 --- a/sanoid +++ b/sanoid @@ -302,11 +302,10 @@ sub prune_snapshots { my $dataset = (split '@', $snap)[0]; my $snapname = (split '@', $snap)[1]; if ($config{$dataset}{'pruning_script'}) { - my $timeout = 5; $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pruning_script '".$config{$dataset}{'pruning_script'}."' on dataset '$dataset'\n"; } - my $ret = runscript('pruning_script',$dataset,$timeout); + my $ret = runscript('pruning_script',$dataset); delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; @@ -475,7 +474,7 @@ sub take_snapshots { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; } - my $ret = runscript('pre_snapshot_script',$dataset,$timeout); + my $ret = runscript('pre_snapshot_script',$dataset); delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; @@ -498,7 +497,7 @@ sub take_snapshots { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; } - runscript('post_snapshot_script',$dataset,$timeout); + runscript('post_snapshot_script',$dataset); delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; @@ -1358,12 +1357,15 @@ sub removecachedsnapshots { sub runscript { my $key=shift; my $dataset=shift; - my $timeout=shift; + + my $timeout=$config{$dataset}{'script_timeout'}; my $ret; eval { - local $SIG{ALRM} = sub { die "alarm\n" }; - alarm $timeout; + if ($timeout gt 0) { + local $SIG{ALRM} = sub { die "alarm\n" }; + alarm $timeout; + } $ret = system($config{$dataset}{$key}); alarm 0; }; diff --git a/sanoid.conf b/sanoid.conf index e684614..db468e2 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -74,12 +74,14 @@ ### run script after snapshot ### dataset name will be supplied as an environment variable $SANOID_TARGET post_snapshot_script = /path/to/script.sh + ### dataset name will be supplied as an environment variable $SANOID_TARGET + pruning_script = /path/to/script.sh ### don't take an inconsistent snapshot #no_inconsistent_snapshot = yes ### run post_snapshot_script when pre_snapshot_script is failing #force_post_snapshot_script = yes - ### dataset name will be supplied as an environment variable $SANOID_TARGET - pruning_script = /path/to/script.sh + ### limit allowed execution time of scripts before continuing (<= 0 -> infinite) + script_timeout = 5 [template_ignore] autoprune = no diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index d4dd19e..d8e428a 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -17,9 +17,10 @@ use_template = process_children_only = pre_snapshot_script = post_snapshot_script = +pruning_script = +script_timeout = 5 no_inconsistent_snapshot = force_post_snapshot_script = -pruning_script = # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately # prune any of those type snapshots already present. From 6968e441468355a03bd1d6fdd39c5c93d2409d36 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 16 Oct 2018 18:10:57 +0200 Subject: [PATCH 112/148] updated documentation regarding pre/post/prun scripts --- sanoid | 1 - sanoid.conf | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/sanoid b/sanoid index 9c0e54d..69562f7 100755 --- a/sanoid +++ b/sanoid @@ -469,7 +469,6 @@ sub take_snapshots { my $dataset = (split '@', $snap)[0]; my $snapname = (split '@', $snap)[1]; my $presnapshotfailure = 0; - my $timeout = 5; if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; diff --git a/sanoid.conf b/sanoid.conf index db468e2..feb2237 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -68,19 +68,19 @@ daily_crit = 60 [template_scripts] + ### dataset and snapshot name will be supplied as environment variables + ### for all pre/post/prune scripts ($SANOID_TARGET, $SANOID_SNAPNAME) ### run script before snapshot - ### dataset name will be supplied as an environment variable $SANOID_TARGET pre_snapshot_script = /path/to/script.sh ### run script after snapshot - ### dataset name will be supplied as an environment variable $SANOID_TARGET post_snapshot_script = /path/to/script.sh - ### dataset name will be supplied as an environment variable $SANOID_TARGET + ### run script after pruning snapshot pruning_script = /path/to/script.sh - ### don't take an inconsistent snapshot + ### don't take an inconsistent snapshot (skip if pre script fails) #no_inconsistent_snapshot = yes ### run post_snapshot_script when pre_snapshot_script is failing #force_post_snapshot_script = yes - ### limit allowed execution time of scripts before continuing (<= 0 -> infinite) + ### limit allowed execution time of scripts before continuing (<= 0: infinite) script_timeout = 5 [template_ignore] From 8e929a331dda6a00ab77208745b7642a2fab7d43 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 9 Nov 2018 07:31:51 +0100 Subject: [PATCH 113/148] removed sleeping between snapshot taking --- sanoid | 2 -- 1 file changed, 2 deletions(-) diff --git a/sanoid b/sanoid index 7ae1b5b..57acc7b 100755 --- a/sanoid +++ b/sanoid @@ -459,8 +459,6 @@ sub take_snapshots { if (!$args{'readonly'}) { system($zfs, "snapshot", "$snap") == 0 or warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; - # make sure we don't end up with multiple snapshots with the same ctime - sleep 1; } } $forcecacheupdate = 1; From fa3c511dc106008bd03066059ed819f231734438 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 14 Nov 2018 18:26:11 +0100 Subject: [PATCH 114/148] split snapshot taking/pruning into seperate units for debian package to prevent pruning blocking snapshot taking --- packages/debian/rules | 12 +++++++++++- packages/debian/sanoid-prune.service | 13 +++++++++++++ packages/debian/sanoid.service | 2 +- 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 packages/debian/sanoid-prune.service diff --git a/packages/debian/rules b/packages/debian/rules index 83eb475..ddd77b0 100755 --- a/packages/debian/rules +++ b/packages/debian/rules @@ -16,4 +16,14 @@ override_dh_auto_install: @mkdir -p $(DESTDIR)/usr/share/doc/sanoid; \ cp sanoid.conf $(DESTDIR)/usr/share/doc/sanoid/sanoid.conf.example; @mkdir -p $(DESTDIR)/lib/systemd/system; \ - cp debian/sanoid.timer $(DESTDIR)/lib/systemd/system; + cp debian/sanoid-prune.service $(DESTDIR)/lib/systemd/system; + +override_dh_installinit: + dh_installinit --noscripts + +override_dh_systemd_enable: + dh_systemd_enable sanoid.timer + dh_systemd_enable sanoid-prune.service + +override_dh_systemd_start: + dh_systemd_start sanoid.timer diff --git a/packages/debian/sanoid-prune.service b/packages/debian/sanoid-prune.service new file mode 100644 index 0000000..c956bd5 --- /dev/null +++ b/packages/debian/sanoid-prune.service @@ -0,0 +1,13 @@ +[Unit] +Description=Cleanup ZFS Pool +Requires=zfs.target +After=zfs.target sanoid.service +ConditionFileNotEmpty=/etc/sanoid/sanoid.conf + +[Service] +Environment=TZ=UTC +Type=oneshot +ExecStart=/usr/sbin/sanoid --prune-snapshots + +[Install] +WantedBy=sanoid.service diff --git a/packages/debian/sanoid.service b/packages/debian/sanoid.service index 2d01bbf..e146354 100644 --- a/packages/debian/sanoid.service +++ b/packages/debian/sanoid.service @@ -7,4 +7,4 @@ ConditionFileNotEmpty=/etc/sanoid/sanoid.conf [Service] Environment=TZ=UTC Type=oneshot -ExecStart=/usr/sbin/sanoid --cron +ExecStart=/usr/sbin/sanoid --take-snapshots From 2796e22dbf8a6eaea8b4d1d27b04f9bea4636ddb Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 21 Nov 2018 00:34:21 +0100 Subject: [PATCH 115/148] added option to defer pruning based on the available pool capacity --- sanoid | 68 +++++++++++++++++++++++++++++++++++++++++--- sanoid.defaults.conf | 4 ++- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/sanoid b/sanoid index 7ae1b5b..34445aa 100755 --- a/sanoid +++ b/sanoid @@ -31,6 +31,7 @@ if (keys %args < 2) { my $pscmd = '/bin/ps'; my $zfs = '/sbin/zfs'; +my $zpool = '/sbin/zpool'; my $conf_file = "$args{'configdir'}/sanoid.conf"; my $default_conf_file = "$args{'configdir'}/sanoid.defaults.conf"; @@ -44,6 +45,7 @@ my $cache = '/var/cache/sanoidsnapshots.txt'; my $cacheTTL = 900; # 15 minutes my %snaps = getsnaps( \%config, $cacheTTL, $forcecacheupdate ); my %pruned; +my %capacitycache; my %snapsbytype = getsnapsbytype( \%config, \%snaps ); @@ -254,6 +256,10 @@ sub prune_snapshots { my $path = $config{$section}{'path'}; my $period = 0; + if (check_prune_defer($config, $section)) { + if ($args{'verbose'}) { print "INFO: deferring snapshot pruning ($section)...\n"; } + next; + } foreach my $type (keys %{ $config{$section} }){ unless ($type =~ /ly$/) { next; } @@ -872,7 +878,7 @@ sub check_zpool() { exit $ERRORS{$state}; } - my $statcommand="/sbin/zpool list -o name,size,cap,health,free $pool"; + my $statcommand="$zpool list -o name,size,cap,health,free $pool"; if (! open STAT, "$statcommand|") { print ("$state '$statcommand' command returns no result! NOTE: This plugin needs OS support for ZFS, and execution with root privileges.\n"); @@ -920,7 +926,7 @@ sub check_zpool() { ## flag to detect section of zpool status involving our zpool my $poolfind=0; - $statcommand="/sbin/zpool status $pool"; + $statcommand="$zpool status $pool"; if (! open STAT, "$statcommand|") { $state = 'CRITICAL'; print ("$state '$statcommand' command returns no result! NOTE: This plugin needs OS support for ZFS, and execution with root privileges.\n"); @@ -1028,7 +1034,7 @@ sub check_zpool() { return ($ERRORS{$state},$msg); } # end check_zpool() -sub check_capacity_limit() { +sub check_capacity_limit { my $value = shift; if (!defined($value) || $value !~ /^\d+\z/) { @@ -1051,7 +1057,7 @@ sub check_zpool_capacity() { my $capacitylimitsref=shift; my %capacitylimits=%$capacitylimitsref; - my $statcommand="/sbin/zpool list -H -o cap $pool"; + my $statcommand="$zpool list -H -o cap $pool"; if (! open STAT, "$statcommand|") { print ("$state '$statcommand' command returns no result!\n"); @@ -1096,6 +1102,60 @@ sub check_zpool_capacity() { return ($ERRORS{$state},$msg); } # end check_zpool_capacity() +sub check_prune_defer { + my ($config, $section) = @_; + + my $limit = $config{$section}{"prune_defer"}; + + if (!check_capacity_limit($limit)) { + die "ERROR: invalid prune_defer limit!\n"; + } + + if ($limit eq 0) { + return 0; + } + + my @parts = split /\//, $section, 2; + my $pool = $parts[0]; + + if (exists $capacitycache{$pool}) { + } else { + $capacitycache{$pool} = get_zpool_capacity($pool); + } + + if ($limit < $capacitycache{$pool}) { + return 0; + } + + return 1; +} + +sub get_zpool_capacity { + my $pool = shift; + + my $statcommand="$zpool list -H -o cap $pool"; + + if (! open STAT, "$statcommand|") { + die "ERROR: '$statcommand' command returns no result!\n"; + } + + my $line = ; + close(STAT); + + chomp $line; + my @row = split(/ +/, $line); + my $cap=$row[0]; + + ## check for valid capacity value + if ($cap !~ m/^[0-9]{1,3}%$/ ) { + die "ERROR: '$statcommand' command returned invalid capacity value ($cap)!\n"; + } + + $cap =~ s/\D//g; + + return $cap; +} + ###################################################################################################### ###################################################################################################### ###################################################################################################### diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index d86cc47..0797fa8 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -26,7 +26,9 @@ hourly = 48 daily = 90 monthly = 6 yearly = 0 -min_percent_free = 10 +# pruning can be skipped based on the used capacity of the pool +# (0: always prune, 1-100: only prune if used capacity is greater than this value) +prune_defer = 0 # We will automatically take snapshots if autosnap is on, at the desired times configured # below (or immediately, if we don't have one since the last preferred time for that type). From c8b880c5e2ebb4771fa845af8e4cd006751d8970 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 21 Nov 2018 18:01:40 +0100 Subject: [PATCH 116/148] implemented clone handling (try to recreate on target instead of full replication) --- syncoid | 116 +++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 93 insertions(+), 23 deletions(-) diff --git a/syncoid b/syncoid index 30aef49..e9e3f27 100755 --- a/syncoid +++ b/syncoid @@ -104,17 +104,59 @@ my $exitcode = 0; ## replication ## if (!defined $args{'recursive'}) { - syncdataset($sourcehost, $sourcefs, $targethost, $targetfs); + syncdataset($sourcehost, $sourcefs, $targethost, $targetfs, undef); } else { if ($debug) { print "DEBUG: recursive sync of $sourcefs.\n"; } my @datasets = getchilddatasets($sourcehost, $sourcefs, $sourceisroot); - foreach my $dataset(@datasets) { + + my @deferred; + + foreach my $datasetProperties(@datasets) { + my $dataset = $datasetProperties->{'name'}; + my $origin = $datasetProperties->{'origin'}; + if ($origin eq "-") { + $origin = undef; + } else { + # check if clone source is replicated too + my @values = split(/@/, $origin, 2); + my $srcdataset = $values[0]; + + my $found = 0; + foreach my $datasetProperties(@datasets) { + if ($datasetProperties->{'name'} eq $srcdataset) { + $found = 1; + last; + } + } + + if ($found == 0) { + # clone source is not replicated, do a full replication + $origin = undef; + } else { + # clone source is replicated, defer until all non clones are replicated + push @deferred, $datasetProperties; + next; + } + } + $dataset =~ s/\Q$sourcefs\E//; chomp $dataset; my $childsourcefs = $sourcefs . $dataset; my $childtargetfs = $targetfs . $dataset; # print "syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs); \n"; - syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs); + syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs, $origin); + } + + # replicate cloned datasets and if this is the initial run, recreate them on the target + foreach my $datasetProperties(@deferred) { + my $dataset = $datasetProperties->{'name'}; + my $origin = $datasetProperties->{'origin'}; + + $dataset =~ s/\Q$sourcefs\E//; + chomp $dataset; + my $childsourcefs = $sourcefs . $dataset; + my $childtargetfs = $targetfs . $dataset; + syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs, $origin); } } @@ -147,37 +189,51 @@ sub getchilddatasets { $fsescaped = escapeshellparam($fsescaped); } - my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name -t filesystem,volume -Hr $fsescaped |"; + my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name,origin -t filesystem,volume -Hr $fsescaped |"; if ($debug) { print "DEBUG: getting list of child datasets on $fs using $getchildrencmd...\n"; } - open FH, $getchildrencmd; - my @children = ; - close FH; - - if (defined $args{'skip-parent'}) { - # parent dataset is the first element - shift @children; + if (! open FH, $getchildrencmd) { + die "ERROR: list command failed!\n"; } - if (defined $args{'exclude'}) { - my $excludes = $args{'exclude'}; - foreach (@$excludes) { - for my $i ( 0 .. $#children ) { - if ($children[$i] =~ /$_/) { - if ($debug) { print "DEBUG: excluded $children[$i] because of $_\n"; } - undef $children[$i] + my @children; + my $first = 1; + + DATASETS: while() { + chomp; + + if (defined $args{'skip-parent'} && $first eq 1) { + # parent dataset is the first element + $first = 0; + next; + } + + my ($dataset, $origin) = /^([^\t]+)\t([^\t]+)/; + + if (defined $args{'exclude'}) { + my $excludes = $args{'exclude'}; + foreach (@$excludes) { + print("$dataset\n"); + if ($dataset =~ /$_/) { + if ($debug) { print "DEBUG: excluded $dataset because of $_\n"; } + next DATASETS; } } - - @children = grep{ defined }@children; } + + my %properties; + $properties{'name'} = $dataset; + $properties{'origin'} = $origin; + + push @children, \%properties; } + close FH; return @children; } sub syncdataset { - my ($sourcehost, $sourcefs, $targethost, $targetfs) = @_; + my ($sourcehost, $sourcefs, $targethost, $targetfs, $origin) = @_; my $sourcefsescaped = escapeshellparam($sourcefs); my $targetfsescaped = escapeshellparam($targetfs); @@ -305,11 +361,25 @@ sub syncdataset { my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped"; my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; - my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); + my $pvsize; + if (defined $origin) { + my $originescaped = escapeshellparam($origin); + $sendcmd = "$sourcesudocmd $zfscmd send -i $originescaped $sourcefsescaped\@$oldestsnapescaped"; + my $streamargBackup = $args{'streamarg'}; + $args{'streamarg'} = "-i"; + $pvsize = getsendsize($sourcehost,$origin,"$sourcefs\@$oldestsnap",$sourceisroot); + $args{'streamarg'} = $streamargBackup; + } else { + $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); + } + my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = 'UNKNOWN'; } my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); if (!$quiet) { + if (defined $origin) { + print "INFO: Clone is recreated on target $targetfs based on $origin\n"; + } if (!defined ($args{'no-stream'}) ) { print "INFO: Sending oldest full snapshot $sourcefs\@$oldestsnap (~ $disp_pvsize) to new target filesystem:\n"; } else { @@ -396,7 +466,7 @@ sub syncdataset { # a resumed transfer will only be done to the next snapshot, # so do an normal sync cycle - return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs); + return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs, undef); } # find most recent matching snapshot and do an -I From 9d6cb42f4d2927c34d9d6344fbadb57f9f72caa1 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 21 Nov 2018 18:08:38 +0100 Subject: [PATCH 117/148] added option to disable smart clone handling --- README.md | 5 +++++ syncoid | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b833dec..ed7a107 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,11 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This argument tells syncoid to not use resumeable zfs send/receive streams. ++ --no-clone-handling + + This argument tells syncoid to not recreate clones on the targe on initial sync and doing a normal replication instead. + + + --dumpsnaps This prints a list of snapshots during the run. diff --git a/syncoid b/syncoid index e9e3f27..c54c915 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,8 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", + "no-clone-handling") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -114,7 +115,7 @@ if (!defined $args{'recursive'}) { foreach my $datasetProperties(@datasets) { my $dataset = $datasetProperties->{'name'}; my $origin = $datasetProperties->{'origin'}; - if ($origin eq "-") { + if ($origin eq "-" || defined $args{'no-clone-handling'}) { $origin = undef; } else { # check if clone source is replicated too @@ -1320,3 +1321,4 @@ Options: --dumpsnaps Dumps a list of snapshots during the run --no-command-checks Do not check command existence before attempting transfer. Not recommended --no-resume Don't use the ZFS resume feature if available + --no-clone-handling Don't try to recreate clones on target From f153810d08f86e804f7e43febb812ddabe810e92 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 21 Nov 2018 23:14:30 +0100 Subject: [PATCH 118/148] check for valid estimated send size to prevent a perl warning on systems which doesn't output size informations --- syncoid | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/syncoid b/syncoid index 30aef49..1f759e2 100755 --- a/syncoid +++ b/syncoid @@ -1154,6 +1154,11 @@ sub getsendsize { } chomp $sendsize; + # check for valid value + if ($sendsize !~ /^\d+$/) { + $sendsize = ''; + } + # to avoid confusion with a zero size pv, give sendsize # a minimum 4K value - or if empty, make sure it reads UNKNOWN if ($debug) { print "DEBUG: sendsize = $sendsize\n"; } From a25ec83812ace12c26ba01296e278a2799d04d77 Mon Sep 17 00:00:00 2001 From: Rodger Donaldson Date: Sun, 25 Nov 2018 07:14:59 +1300 Subject: [PATCH 119/148] Add a dependency for the configini patch The RPM will install but fail to run if the perl-Config-IniFiles rpm is not also installed; this adds as a Requires: so that yum/dnf can find and install. --- packages/rhel/sanoid.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index 3a9412f..7d4995d 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -14,7 +14,7 @@ License: GPLv3 URL: https://github.com/jimsalterjrs/sanoid Source0: https://github.com/jimsalterjrs/%{name}/archive/%{git_tag}/%{name}-%{version}.tar.gz -Requires: perl, mbuffer, lzop, pv +Requires: perl, mbuffer, lzop, pv, perl-Config-IniFiles %if 0%{?_with_systemd} Requires: systemd >= 212 From ea55308dfcdce7614306d8a4b7187ff61d43c4b8 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 4 Dec 2018 15:51:51 +0100 Subject: [PATCH 120/148] implemented support for excluding children of a specific dataset --- sanoid | 18 ++++++++++++++---- sanoid.defaults.conf | 1 + 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/sanoid b/sanoid index 7ae1b5b..8f390ba 100755 --- a/sanoid +++ b/sanoid @@ -661,7 +661,7 @@ sub init { tie my %ini, 'Config::IniFiles', ( -file => $conf_file ) or die "FATAL: cannot load $conf_file - please create a valid local config file before running sanoid!"; # we'll use these later to normalize potentially true and false values on any toggle keys - my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only'); + my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only','skip_children'); my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON"); my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF"); @@ -718,7 +718,7 @@ sub init { # override with any locally set values in the module itself foreach my $key (keys %{$ini{$section}} ) { - if (! ($key =~ /template|recursive/)) { + if (! ($key =~ /template|recursive|skip_children/)) { if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value directly set in module.\n"; } $config{$section}{$key} = $ini{$section}{$key}; } @@ -743,10 +743,17 @@ sub init { # how 'bout some recursion? =) my @datasets; - if ($ini{$section}{'recursive'}) { + if ($ini{$section}{'recursive'} || $ini{$section}{'skip_children'}) { @datasets = getchilddatasets($config{$section}{'path'}); - foreach my $dataset(@datasets) { + DATASETS: foreach my $dataset(@datasets) { chomp $dataset; + + if ($ini{$section}{'skip_children'}) { + if ($args{'debug'}) { print "DEBUG: ignoring $dataset.\n"; } + delete $config{$dataset}; + next DATASETS; + } + foreach my $key (keys %{$config{$section}} ) { if (! ($key =~ /template|recursive|children_only/)) { if ($args{'debug'}) { print "DEBUG: recursively setting $key from $section to $dataset.\n"; } @@ -1257,6 +1264,9 @@ sub getchilddatasets { my @children = ; close FH; + # parent dataset is the first element + shift @children; + return @children; } diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index d86cc47..0c9037a 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -15,6 +15,7 @@ path = recursive = use_template = process_children_only = +skip_children = # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately # prune any of those type snapshots already present. From a0b983ee6ef791449fb631353e26b4f05a5ab71a Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 4 Dec 2018 21:38:15 +0100 Subject: [PATCH 121/148] warn if unknown interval type is used --- sanoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanoid b/sanoid index cb43066..5c81893 100755 --- a/sanoid +++ b/sanoid @@ -471,7 +471,7 @@ sub take_snapshots { $lastpreferred = timelocal(@preferredtime); if ($lastpreferred > time()) { $lastpreferred -= 60*60*24*31*365.25; } # preferred time is later this year - so look at last year } else { - # unknown type + warn "WARN: unknown interval type $type in config!"; next; } From 7d742914e53591cfb2d13ad9dc22df55e9350446 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Wed, 23 Aug 2017 19:23:22 -0400 Subject: [PATCH 122/148] Add '--no-privilege-elevation' option to bypass root check. --- README.md | 4 ++++ syncoid | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c537540..e86d13a 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This prints a list of snapshots during the run. ++ --no-privilege-elevation + + Bypass the root check and assume syncoid has the necessary permissions (for use with ZFS permission delegation). + + --sshport Allow sync to/from boxes running SSH on non-standard ports. diff --git a/syncoid b/syncoid index e960d53..ccc5861 100755 --- a/syncoid +++ b/syncoid @@ -20,7 +20,7 @@ my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", - "no-clone-handling") or pod2usage(2); + "no-clone-handling", "no-privilege-elevation") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -1186,7 +1186,7 @@ sub getssh { $rhost =~ s/:\Q$fs\E$//; my $remoteuser = $rhost; $remoteuser =~ s/\@.*$//; - if ($remoteuser eq 'root') { $isroot = 1; } else { $isroot = 0; } + if ($remoteuser eq 'root' || $args{'no-privilege-elevation'}) { $isroot = 1; } else { $isroot = 0; } # now we need to establish a persistent master SSH connection $socket = "/tmp/syncoid-$remoteuser-$rhost-" . time(); open FH, "$sshcmd -M -S $socket -o ControlPersist=1m $args{'sshport'} $rhost exit |"; @@ -1194,7 +1194,7 @@ sub getssh { $rhost = "-S $socket $rhost"; } else { my $localuid = $<; - if ($localuid == 0) { $isroot = 1; } else { $isroot = 0; } + if ($localuid == 0 || $args{'no-privilege-elevation'}) { $isroot = 1; } else { $isroot = 0; } } # if ($isroot) { print "this user is root.\n"; } else { print "this user is not root.\n"; } return ($rhost,$fs,$isroot); @@ -1455,3 +1455,4 @@ Options: --no-command-checks Do not check command existence before attempting transfer. Not recommended --no-resume Don't use the ZFS resume feature if available --no-clone-handling Don't try to recreate clones on target + --no-privilege-elevation Bypass the root check, for use with ZFS permission delegation From 23a0ce2e06a8bf091335aa8fb6ffb3356bc549a2 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Tue, 4 Dec 2018 16:33:03 -0500 Subject: [PATCH 123/148] Update CHANGELIST --- CHANGELIST | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/CHANGELIST b/CHANGELIST index 515d05d..5c53199 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,29 @@ +current PRERELEASE: + #140 - --no-privilege-elevation option to bypass root checks entirely + #157 - configurable frequent snapshot period + #163 - weekly snapshot period + #247 - implement support for zfs bookmarks in syncoid + #248 - allow forced target snapshot deletion in syncoid with --force-delete, for those who like to live dangerously + #254 - add xz compression option to syncoid + #261 - add WARNings for unsupported parameters in templates + #262 - documentation typo fix + #264 - add clean WARN error for unknown snapshot type values + #265 - add support for lz4 compression to syncoid + #266 - squash uninitialized value perl warning on first sanoid run (no snapshots) + #267 - improve boolean handling for recursive parameter + #268 - update README + #269 - don't alarm for lack of hourly/daily/etc if hourly/daily/etc=0 + #277 - update automated test script for DST + #279 - FreeBSD pkg typo fix + #280 - add pre,post,prune snapshot script hooks + #286 - remove 1s sleep interval between snapshots + #287 - run snapshot prune, snapshot take as separate systemd services in Debian package + #289 - optional defer pruning until low %FREE reached + #290 - replicate clone structure from source to target + #292 - fix sendsize estimation to prevent perl warnings on some systems + #294 - add dependency for Perl::Config::Ini + #298 - exclude recursion for children of a dataset with skip-children = yes + 1.4.18 implemented special character handling and support of ZFS resume/receive tokens by default in syncoid, thank you @phreaker0! From 00a9c6f6b551ef1d19c97ecba102f9a47388fd5d Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 4 Dec 2018 23:12:16 +0100 Subject: [PATCH 124/148] added changed from recently merged PR's --- CHANGELIST | 60 +++++++++++++++++++++++-------------- packages/debian/changelog | 62 ++++++++++++++++++++++++--------------- packages/rhel/sanoid.spec | 2 +- 3 files changed, 78 insertions(+), 46 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 3ef74ae..9172b2d 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,25 +1,41 @@ -1.4.19 [sanoid] monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0) - [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie) - [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0) - [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0) - [syncoid] make local source bwlimit work (@phreaker0) - [syncoid] fix 'resume support' detection on FreeBSD (@pit3k) - [sanoid] updated INSTALL with missing dependency - [sanoid] fixed monitor-health command for pools containing cache and log devices (@phreaker0) - [sanoid] quiet flag suppresses all info output (@martinvw) - [sanoid] check for empty lockfile which lead to sanoid failing on start (@jasonblewis) - [sanoid] added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0) - [sanoid] cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0) - [sanoid] implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0) - [syncoid] Added support for ZStandard compression.(@danielewood) - [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0) - [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0) - [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0) - [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0) - [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0) - [sanoid] use UTC by default in unit template and documentation (@phreaker0) - [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0) - [syncoid] documented compatibility issues with (t)csh shells (@ecoutu) +1.4.19 [overall] documentation updates, small fixes, more warnings (@sparky3387, @ljwobker, @phreaker0) + [syncoid] added force delete flag (@phreaker0) + [sanoid] removed sleeping between snapshot taking (@phreaker0) + [syncoid] added '--no-privilege-elevation' option to bypass root check (@lopsided98) + [sanoid] implemented weekly period (@phreaker0) + [syncoid] implemented support for zfs bookmarks as fallback (@phreaker0) + [sanoid] support for pre, post and prune snapshot scripts (@jouir, @darkbasic, @phreaker0) + [sanoid] ignore snapshots types that are set to 0 (@muff1nman) + [packaging] split snapshot taking/pruning into separate systemd units for debian package (@phreaker0) + [syncoid] replicate clones (@phreaker0) + [syncoid] added compression algorithms: lz4, xz (@spheenik, @phreaker0) + [sanoid] added option to defer pruning based on the available pool capacity (@phreaker0) + [sanoid] implemented frequent snapshots with configurable period (@phreaker0) + [syncoid] prevent a perl warning on systems which doesn't output estimated send size information (@phreaker0) + [packaging] dependency fixes (@rodgerd, mabushey) + [syncoid] implemented support for excluding children of a specific dataset (@phreaker0) + [sanoid] monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0) + [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie) + [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0) + [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0) + [syncoid] make local source bwlimit work (@phreaker0) + [syncoid] fix 'resume support' detection on FreeBSD (@pit3k) + [sanoid] updated INSTALL with missing dependency + [sanoid] fixed monitor-health command for pools containing cache and log devices (@phreaker0) + [sanoid] quiet flag suppresses all info output (@martinvw) + [sanoid] check for empty lockfile which lead to sanoid failing on start (@jasonblewis) + [sanoid] added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0) + [sanoid] cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0) + [sanoid] implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0) + [syncoid] Added support for ZStandard compression.(@danielewood) + [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0) + [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0) + [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0) + [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0) + [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0) + [sanoid] use UTC by default in unit template and documentation (@phreaker0) + [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0) + [syncoid] documented compatibility issues with (t)csh shells (@ecoutu) 1.4.18 implemented special character handling and support of ZFS resume/receive tokens by default in syncoid, thank you @phreaker0! diff --git a/packages/debian/changelog b/packages/debian/changelog index 67cca4d..34909b6 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -1,29 +1,45 @@ sanoid (1.4.19) unstable; urgency=medium - [sanoid] monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0) - [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie) - [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0) - [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0) - [syncoid] make local source bwlimit work (@phreaker0) - [syncoid] fix 'resume support' detection on FreeBSD (@pit3k) - [sanoid] updated INSTALL with missing dependency - [sanoid] fixed monitor-health command for pools containing cache and log devices (@phreaker0) - [sanoid] quiet flag suppresses all info output (@martinvw) - [sanoid] check for empty lockfile which lead to sanoid failing on start (@jasonblewis) - [sanoid] added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0) - [sanoid] cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0) - [sanoid] implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0) - [syncoid] Added support for ZStandard compression.(@danielewood) - [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0) - [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0) - [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0) - [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0) - [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0) - [sanoid] use UTC by default in unit template and documentation (@phreaker0) - [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0) - [syncoid] documented compatibility issues with (t)csh shells (@ecoutu) + [overall] documentation updates, small fixes, more warnings (@sparky3387, @ljwobker, @phreaker0) + [syncoid] added force delete flag (@phreaker0) + [sanoid] removed sleeping between snapshot taking (@phreaker0) + [syncoid] added '--no-privilege-elevation' option to bypass root check (@lopsided98) + [sanoid] implemented weekly period (@phreaker0) + [syncoid] implemented support for zfs bookmarks as fallback (@phreaker0) + [sanoid] support for pre, post and prune snapshot scripts (@jouir, @darkbasic, @phreaker0) + [sanoid] ignore snapshots types that are set to 0 (@muff1nman) + [packaging] split snapshot taking/pruning into separate systemd units for debian package (@phreaker0) + [syncoid] replicate clones (@phreaker0) + [syncoid] added compression algorithms: lz4, xz (@spheenik, @phreaker0) + [sanoid] added option to defer pruning based on the available pool capacity (@phreaker0) + [sanoid] implemented frequent snapshots with configurable period (@phreaker0) + [syncoid] prevent a perl warning on systems which doesn't output estimated send size information (@phreaker0) + [packaging] dependency fixes (@rodgerd, mabushey) + [syncoid] implemented support for excluding children of a specific dataset (@phreaker0) + [sanoid] monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0) + [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie) + [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0) + [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0) + [syncoid] make local source bwlimit work (@phreaker0) + [syncoid] fix 'resume support' detection on FreeBSD (@pit3k) + [sanoid] updated INSTALL with missing dependency + [sanoid] fixed monitor-health command for pools containing cache and log devices (@phreaker0) + [sanoid] quiet flag suppresses all info output (@martinvw) + [sanoid] check for empty lockfile which lead to sanoid failing on start (@jasonblewis) + [sanoid] added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0) + [sanoid] cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0) + [sanoid] implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0) + [syncoid] Added support for ZStandard compression.(@danielewood) + [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0) + [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0) + [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0) + [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0) + [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0) + [sanoid] use UTC by default in unit template and documentation (@phreaker0) + [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0) + [syncoid] documented compatibility issues with (t)csh shells (@ecoutu) - -- Jim Salter Wed, 05 Sep 2018 04:00:00 -0400 + -- Jim Salter Wed, 04 Dec 2018 18:10:00 -0400 sanoid (1.4.18) unstable; urgency=medium diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index b5d7172..1579a27 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -111,7 +111,7 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name} %endif %changelog -* Wed Sep 05 2018 Christoph Klaffl - 1.4.19 +* Wed Dec 04 2018 Christoph Klaffl - 1.4.19 - Bump to 1.4.19 * Sat Apr 28 2018 Dominic Robinson - 1.4.18-1 - Bump to 1.4.18 From 3a1ffe8554802dfc2459032b3beba5f5cf360640 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 4 Dec 2018 23:33:18 +0100 Subject: [PATCH 125/148] fixed a regression which causes perl warnings --- sanoid | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index fbb94b6..dc71b9d 100755 --- a/sanoid +++ b/sanoid @@ -843,13 +843,15 @@ sub init { } # how 'bout some recursion? =) + my $recursive = $ini{$section}{'recursive'} && grep( /^$ini{$section}{'recursive'}$/, @istrue ); + my $skipChildren = $ini{$section}{'skip_children'} && grep( /^$ini{$section}{'skip_children'}$/, @istrue ); my @datasets; - if (grep( /^$ini{$section}{'recursive'}$/, @istrue ) || grep( /^$ini{$section}{'skip_children'}$/, @istrue )) { + if ($recursive || $skipChildren) { @datasets = getchilddatasets($config{$section}{'path'}); DATASETS: foreach my $dataset(@datasets) { chomp $dataset; - if (grep( /^$ini{$section}{'skip_children'}$/, @istrue )) { + if ($skipChildren) { if ($args{'debug'}) { print "DEBUG: ignoring $dataset.\n"; } delete $config{$dataset}; next DATASETS; From 1ed37e9891400755311429a4a21e5102c87cd393 Mon Sep 17 00:00:00 2001 From: Gionatan Danti Date: Thu, 6 Dec 2018 09:26:22 +0100 Subject: [PATCH 126/148] Resolve a conflict --- syncoid | 9 --------- 1 file changed, 9 deletions(-) diff --git a/syncoid b/syncoid index 3ad1622..1bb4c80 100755 --- a/syncoid +++ b/syncoid @@ -583,14 +583,6 @@ sub syncdataset { system ("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped"); } } -<<<<<<< HEAD - my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped"; - my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); - my $disp_pvsize = readablebytes($pvsize); - if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } - my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); -======= my $nextsnapshot = 0; @@ -608,7 +600,6 @@ sub syncdataset { } } } ->>>>>>> e186f3c66e9c757fa62c4eaa8a1c05bc49dbcff1 # bookmark stream size can't be determined my $pvsize = 0; From 210e0aae640481cc846609a9947c66d7a5b2b82f Mon Sep 17 00:00:00 2001 From: Gionatan Danti Date: Fri, 7 Dec 2018 17:38:45 +0100 Subject: [PATCH 127/148] Introduced (un)forced recv when using bookmarks also --- syncoid | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index 1bb4c80..f599f96 100755 --- a/syncoid +++ b/syncoid @@ -609,7 +609,7 @@ sub syncdataset { my $nextsnapshotescaped = escapeshellparam($nextsnapshot); my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$nextsnapshotescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped"; my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $nextsnapshot (~ $disp_pvsize):\n"; } @@ -624,7 +624,7 @@ sub syncdataset { $matchingsnapescaped = escapeshellparam($matchingsnap); } else { my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$newsyncsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped"; my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $newsyncsnap (~ $disp_pvsize):\n"; } @@ -641,7 +641,7 @@ sub syncdataset { # bookmark replication was only done to the next oldest snapshot if (!$bookmark || $nextsnapshot) { my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } From 8568ac3e1030751ce5726bd4950c3e7e20f4b8f2 Mon Sep 17 00:00:00 2001 From: Gionatan Danti Date: Mon, 10 Dec 2018 11:21:43 +0100 Subject: [PATCH 128/148] Fix missing coma --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index f599f96..d3e678f 100755 --- a/syncoid +++ b/syncoid @@ -20,7 +20,7 @@ my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", - "no-clone-handling", "no-privilege-elevation", "force-delete" "no-clone-rollback", "no-rollback") or pod2usage(2); + "no-clone-handling", "no-privilege-elevation", "force-delete", "no-clone-rollback", "no-rollback") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set From dc4df15e2ee8988449788861b7bd8f0260ea94f3 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Fri, 14 Dec 2018 10:44:45 -0500 Subject: [PATCH 129/148] fix broken monthly_warn monthly_crit in sanoid.defaults.conf --- sanoid.defaults.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 4139393..96be95c 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -100,8 +100,8 @@ daily_warn = 28 daily_crit = 32 weekly_warn = 0 weekly_crit = 0 -monthly_warn = 5 -monthly_crit = 6 +monthly_warn = 32 +monthly_crit = 40 yearly_warn = 0 yearly_crit = 0 From 2ece13eccf0bd4f5ce0aaf15c2bfdffed37b9d41 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 16 Dec 2018 21:54:43 +0100 Subject: [PATCH 130/148] allow time units to be used for monitoring warn/crit values --- sanoid | 48 ++++++++++++++++++++++++++++++++++++++++---- sanoid.defaults.conf | 12 +++++------ 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/sanoid b/sanoid index ddd457f..3aa57a3 100755 --- a/sanoid +++ b/sanoid @@ -143,8 +143,8 @@ sub monitor_snapshots { my $typewarn = $type . '_warn'; my $typecrit = $type . '_crit'; - my $warn = $config{$section}{$typewarn} * $smallerperiod; - my $crit = $config{$section}{$typecrit} * $smallerperiod; + my $warn = convertTimePeriod($config{$section}{$typewarn}, $smallerperiod); + my $crit = convertTimePeriod($config{$section}{$typecrit}, $smallerperiod); my $elapsed = -1; if (defined $snapsbytype{$path}{$type}{'newest'}) { $elapsed = $snapsbytype{$path}{$type}{'newest'}; @@ -153,7 +153,7 @@ sub monitor_snapshots { my $dispwarn = displaytime($warn); my $dispcrit = displaytime($crit); if ( $elapsed > $crit || $elapsed == -1) { - if ($config{$section}{$typecrit} > 0) { + if ($crit > 0) { if (! $config{$section}{'monitor_dont_crit'}) { $errorlevel = 2; } if ($elapsed == -1) { push @msgs, "CRIT: $path has no $type snapshots at all!"; @@ -162,7 +162,7 @@ sub monitor_snapshots { } } } elsif ($elapsed > $warn) { - if ($config{$section}{$typewarn} > 0) { + if ($warn > 0) { if (! $config{$section}{'monitor_dont_warn'} && ($errorlevel < 2) ) { $errorlevel = 1; } push @msgs, "WARN: $path\'s newest $type snapshot is $dispelapsed old (should be < $dispwarn)"; } @@ -1511,6 +1511,46 @@ sub runscript { return $ret; } +#######################################################################################################################3 +#######################################################################################################################3 +#######################################################################################################################3 + +sub convertTimePeriod { + my $value=shift; + my $period=shift; + + if ($value =~ /^\d+Y$/) { + $period = 60*60*24*31*365; + chop $value; + } elsif ($value =~ /^\d+M$/) { + $period = 60*60*24*31; + chop $value; + } elsif ($value =~ /^\d+W$/) { + $period = 60*60*24*7; + chop $value; + } elsif ($value =~ /^\d+D$/) { + $period = 60*60*24; + chop $value; + } elsif ($value =~ /^\d+h$/) { + $period = 60*60; + chop $value; + } elsif ($value =~ /^\d+m$/) { + $period = 60; + chop $value; + } elsif ($value =~ /^\d+s$/) { + $period = 1; + chop $value; + } elsif ($value =~ /^\d+$/) { + # no unit, provided fallback period is used + } else { + # invalid value, return smallest valid value as fallback + # (will trigger a warning message for monitoring for sure) + return 1; + } + + return $value * $period; +} + __END__ =head1 NAME diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 96be95c..8785e7c 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -94,14 +94,14 @@ monitor_dont_warn = no monitor_dont_crit = no frequently_warn = 0 frequently_crit = 0 -hourly_warn = 90 -hourly_crit = 360 -daily_warn = 28 -daily_crit = 32 +hourly_warn = 90m +hourly_crit = 360m +daily_warn = 28h +daily_crit = 32h weekly_warn = 0 weekly_crit = 0 -monthly_warn = 32 -monthly_crit = 40 +monthly_warn = 32D +monthly_crit = 40D yearly_warn = 0 yearly_crit = 0 From cfab4eafdf2e8512f787db9e854835872493533f Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 16 Dec 2018 22:02:14 +0100 Subject: [PATCH 131/148] added/fixed documentation --- sanoid.defaults.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 8785e7c..6649c2e 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -83,7 +83,8 @@ yearly_min = 0 # monitoring plugin - define warn / crit levels for each snapshot type by age, in units of one period down # example hourly_warn = 90 means issue WARNING if most recent hourly snapshot is not less than 90 minutes old, # daily_crit = 36 means issue CRITICAL if most recent daily snapshot is not less than 36 hours old, -# monthly_warn = 36 means issue WARNING if most recent monthly snapshot is not less than 36 days old... etc. +# monthly_warn = 5 means issue WARNING if most recent monthly snapshot is not less than 5 weeks old... etc. +# the following time suffixes can also be used: Y = years, M = months, W = weeks, D = days, h = hours, m = minutes, s = seconds # # monitor_dont_warn = yes will cause the monitoring service to report warnings as text, but with status OK. # monitor_dont_crit = yes will cause the monitoring service to report criticals as text, but with status OK. From ac80a753157dc26005c976675336f1e3fe3e1ad9 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 19 Dec 2018 00:36:10 +0100 Subject: [PATCH 132/148] made time units case insensitive and removed monthlies --- sanoid | 15 ++++++--------- sanoid.defaults.conf | 7 ++++--- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/sanoid b/sanoid index 3aa57a3..ce8207c 100755 --- a/sanoid +++ b/sanoid @@ -1519,25 +1519,22 @@ sub convertTimePeriod { my $value=shift; my $period=shift; - if ($value =~ /^\d+Y$/) { + if ($value =~ /^\d+[yY]$/) { $period = 60*60*24*31*365; chop $value; - } elsif ($value =~ /^\d+M$/) { - $period = 60*60*24*31; - chop $value; - } elsif ($value =~ /^\d+W$/) { + } elsif ($value =~ /^\d+[wW]$/) { $period = 60*60*24*7; chop $value; - } elsif ($value =~ /^\d+D$/) { + } elsif ($value =~ /^\d+[dD]$/) { $period = 60*60*24; chop $value; - } elsif ($value =~ /^\d+h$/) { + } elsif ($value =~ /^\d+[hH]$/) { $period = 60*60; chop $value; - } elsif ($value =~ /^\d+m$/) { + } elsif ($value =~ /^\d+[mM]$/) { $period = 60; chop $value; - } elsif ($value =~ /^\d+s$/) { + } elsif ($value =~ /^\d+[sS]$/) { $period = 1; chop $value; } elsif ($value =~ /^\d+$/) { diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 6649c2e..a9ca382 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -84,7 +84,8 @@ yearly_min = 0 # example hourly_warn = 90 means issue WARNING if most recent hourly snapshot is not less than 90 minutes old, # daily_crit = 36 means issue CRITICAL if most recent daily snapshot is not less than 36 hours old, # monthly_warn = 5 means issue WARNING if most recent monthly snapshot is not less than 5 weeks old... etc. -# the following time suffixes can also be used: Y = years, M = months, W = weeks, D = days, h = hours, m = minutes, s = seconds +# the following time case insensitive suffixes can also be used: +# y = years, w = weeks, d = days, h = hours, m = minutes, s = seconds # # monitor_dont_warn = yes will cause the monitoring service to report warnings as text, but with status OK. # monitor_dont_crit = yes will cause the monitoring service to report criticals as text, but with status OK. @@ -101,8 +102,8 @@ daily_warn = 28h daily_crit = 32h weekly_warn = 0 weekly_crit = 0 -monthly_warn = 32D -monthly_crit = 40D +monthly_warn = 32d +monthly_crit = 40d yearly_warn = 0 yearly_crit = 0 From 8d88c4743e9e0a5bcb66a2530f9c2b1950f3fc65 Mon Sep 17 00:00:00 2001 From: Ben Yanke Date: Wed, 3 Oct 2018 16:20:58 -0500 Subject: [PATCH 133/148] improved documentation on --no-command-checks --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b833dec..99d8db1 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. -

sanoid rollback demo
(Real time demo: rolling back a full-scale cryptomalware infection in seconds!)

+

sanoid rollback demo
(Real time demo: rolling back a full-scale cryptomalware infection in seconds!)

More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: ``` @@ -180,7 +181,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --no-command-checks - Do not check the existance of commands before attempting the transfer. It assumes all programs are available. This should never be used. + Does not check the existence of commands before attempting the transfer, providing administrators a way to run the tool with minimal overhead and maximum speed, at risk of potentially failed replication, or other possible edge cases. It assumes all programs are available, and should not be used in most situations. This is an not an officially supported run mode. + --no-stream From 1605a60c624b586243afa6a5d803b746a83b3b17 Mon Sep 17 00:00:00 2001 From: Sam Allred Date: Tue, 19 Dec 2017 15:47:40 -0700 Subject: [PATCH 134/148] CLIMATE-1151: added config option to use zfs recursion --- sanoid | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/sanoid b/sanoid index ce8207c..30363c5 100755 --- a/sanoid +++ b/sanoid @@ -502,7 +502,14 @@ sub take_snapshots { # update to most current possible datestamp %datestamp = get_date(); # print "we should have had a $type snapshot of $path $maxage seconds ago; most recent is $newestage seconds old.\n"; - push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}${dateSuffix}_$type"); + + # use zfs recursion if specified in config + if ($config{$section}{'zfs_recursion'}) { + push(@newsnaps, "-r $path\@autosnap_$datestamp{'sortable'}_$type"); + + }else{ + push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}_$type"); + } } } } @@ -530,7 +537,7 @@ sub take_snapshots { } if ($args{'verbose'}) { print "taking snapshot $snap\n"; } if (!$args{'readonly'}) { - system($zfs, "snapshot", "$snap") == 0 + system("$zfs snapshot $snap") == 0 or warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; } if ($config{$dataset}{'post_snapshot_script'} and !$args{'readonly'}) { @@ -846,7 +853,10 @@ sub init { my $recursive = $ini{$section}{'recursive'} && grep( /^$ini{$section}{'recursive'}$/, @istrue ); my $skipChildren = $ini{$section}{'skip_children'} && grep( /^$ini{$section}{'skip_children'}$/, @istrue ); my @datasets; - if ($recursive || $skipChildren) { + + if($ini{$section}{'recursive'} =~ /zfs/i) { + $config{$section}{'zfs_recursion'} = 1; + }elsif ($ini{$section}{'recursive'}) { @datasets = getchilddatasets($config{$section}{'path'}); DATASETS: foreach my $dataset(@datasets) { chomp $dataset; From 4daafca9cf5386ac33bb0cec6e29c2c0e9ab2c38 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 20 Dec 2018 01:33:35 +0100 Subject: [PATCH 135/148] prevent problems with shell expansion and codestyle --- sanoid | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/sanoid b/sanoid index 30363c5..9fcbb88 100755 --- a/sanoid +++ b/sanoid @@ -503,12 +503,11 @@ sub take_snapshots { %datestamp = get_date(); # print "we should have had a $type snapshot of $path $maxage seconds ago; most recent is $newestage seconds old.\n"; - # use zfs recursion if specified in config + # use zfs (atomic) recursion if specified in config if ($config{$section}{'zfs_recursion'}) { - push(@newsnaps, "-r $path\@autosnap_$datestamp{'sortable'}_$type"); - - }else{ - push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}_$type"); + push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}${dateSuffix}_$type\@"); + } else { + push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}${dateSuffix}_$type"); } } } @@ -517,8 +516,14 @@ sub take_snapshots { if ( (scalar(@newsnaps)) > 0) { foreach my $snap ( @newsnaps ) { - my $dataset = (split '@', $snap)[0]; - my $snapname = (split '@', $snap)[1]; + my @split = split '@', $snap, -1; + my $recursiveFlag = 0; + if (scalar(@split) == 3) { + $recursiveFlag = 1; + chop $snap; + } + my $dataset = $split[0]; + my $snapname = $split[1]; my $presnapshotfailure = 0; if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) { $ENV{'SANOID_TARGET'} = $dataset; @@ -537,8 +542,13 @@ sub take_snapshots { } if ($args{'verbose'}) { print "taking snapshot $snap\n"; } if (!$args{'readonly'}) { - system("$zfs snapshot $snap") == 0 - or warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; + if ($recursiveFlag) { + system($zfs, "snapshot", "-r", "$snap") == 0 + or warn "CRITICAL ERROR: $zfs snapshot -r $snap failed, $?"; + } else { + system($zfs, "snapshot", "$snap") == 0 + or warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; + } } if ($config{$dataset}{'post_snapshot_script'} and !$args{'readonly'}) { if (!$presnapshotfailure or $config{$dataset}{'force_post_snapshot_script'}) { @@ -853,10 +863,9 @@ sub init { my $recursive = $ini{$section}{'recursive'} && grep( /^$ini{$section}{'recursive'}$/, @istrue ); my $skipChildren = $ini{$section}{'skip_children'} && grep( /^$ini{$section}{'skip_children'}$/, @istrue ); my @datasets; - - if($ini{$section}{'recursive'} =~ /zfs/i) { + if ($ini{$section}{'recursive'} =~ /zfs/i) { $config{$section}{'zfs_recursion'} = 1; - }elsif ($ini{$section}{'recursive'}) { + } elsif ($recursive || $skipChildren) { @datasets = getchilddatasets($config{$section}{'path'}); DATASETS: foreach my $dataset(@datasets) { chomp $dataset; From 7cf64be7029a985bd8dbde04688bb947861a15ef Mon Sep 17 00:00:00 2001 From: matveevandrey Date: Thu, 20 Dec 2018 22:53:57 +0300 Subject: [PATCH 136/148] pre/post snapshot script calls should be also dumped in --readonly mode --- sanoid | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/sanoid b/sanoid index ce8207c..03aefc5 100755 --- a/sanoid +++ b/sanoid @@ -513,11 +513,15 @@ sub take_snapshots { my $dataset = (split '@', $snap)[0]; my $snapname = (split '@', $snap)[1]; my $presnapshotfailure = 0; - if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) { + my $ret = 0; + if ($config{$dataset}{'pre_snapshot_script'}) { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; } - my $ret = runscript('pre_snapshot_script',$dataset); + + if (!$args{'readonly'}) { + $ret = runscript('pre_snapshot_script',$dataset); + } delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; @@ -533,12 +537,15 @@ sub take_snapshots { system($zfs, "snapshot", "$snap") == 0 or warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; } - if ($config{$dataset}{'post_snapshot_script'} and !$args{'readonly'}) { + if ($config{$dataset}{'post_snapshot_script'}) { if (!$presnapshotfailure or $config{$dataset}{'force_post_snapshot_script'}) { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; } - runscript('post_snapshot_script',$dataset); + + if (!$args{'readonly'}) { + runscript('post_snapshot_script',$dataset); + } delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; From f8e0e006ab92344b62a640988c668631c243107b Mon Sep 17 00:00:00 2001 From: matveevandrey Date: Thu, 20 Dec 2018 23:52:40 +0300 Subject: [PATCH 137/148] preserve taking snapshots order always from yearly to frequently very simple patch (not best way in terms of programming but minimal change requirements achieved) --- sanoid | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index 03aefc5..d30dcba 100755 --- a/sanoid +++ b/sanoid @@ -377,9 +377,12 @@ sub take_snapshots { if ($config{$section}{'process_children_only'}) { next; } my $path = $config{$section}{'path'}; + my @types = ('yearly','monthly','weekly','daily','hourly','frequently'); - foreach my $type (keys %{ $config{$section} }){ - unless ($type =~ /ly$/) { next; } + foreach my $type (@types) { + + foreach my $_type (keys %{ $config{$section} }){ + unless ($type eq $_type) { next; } if ($config{$section}{$type} > 0) { my $newestage; # in seconds @@ -506,6 +509,7 @@ sub take_snapshots { } } } + } } if ( (scalar(@newsnaps)) > 0) { From c35c953e548a922e57edfaa046313a5fafe271a6 Mon Sep 17 00:00:00 2001 From: matveevandrey Date: Fri, 21 Dec 2018 12:56:10 +0300 Subject: [PATCH 138/148] remove redundant loop --- sanoid | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sanoid b/sanoid index d30dcba..82cfce9 100755 --- a/sanoid +++ b/sanoid @@ -380,9 +380,6 @@ sub take_snapshots { my @types = ('yearly','monthly','weekly','daily','hourly','frequently'); foreach my $type (@types) { - - foreach my $_type (keys %{ $config{$section} }){ - unless ($type eq $_type) { next; } if ($config{$section}{$type} > 0) { my $newestage; # in seconds @@ -509,7 +506,6 @@ sub take_snapshots { } } } - } } if ( (scalar(@newsnaps)) > 0) { From ca76f4268b27f8330f91b1e1421853765c326047 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 23 Dec 2018 09:18:49 -0500 Subject: [PATCH 139/148] syncoid: add '--mbuffer-size' option --- syncoid | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index d3e678f..1085e36 100755 --- a/syncoid +++ b/syncoid @@ -14,13 +14,16 @@ use Pod::Usage; use Time::Local; use Sys::Hostname; +my $mbuffer_size = "16M"; + # Blank defaults to use ssh client's default # TODO: Merge into a single "sshflags" option? my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", - "no-clone-handling", "no-privilege-elevation", "force-delete", "no-clone-rollback", "no-rollback") or pod2usage(2); + "no-clone-handling", "no-privilege-elevation", "force-delete", "no-clone-rollback", "no-rollback", + "mbuffer-size=s" => \$mbuffer_size) or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -56,7 +59,7 @@ my $pscmd = '/bin/ps'; my $pvcmd = '/usr/bin/pv'; my $mbuffercmd = '/usr/bin/mbuffer'; my $sudocmd = '/usr/bin/sudo'; -my $mbufferoptions = '-q -s 128k -m 16M 2>/dev/null'; +my $mbufferoptions = "-q -s 128k -m $mbuffer_size 2>/dev/null"; # currently using ls to check for file existence because we aren't depending on perl # being present on remote machines. my $lscmd = '/bin/ls'; @@ -1489,6 +1492,7 @@ Options: --skip-parent Skips syncing of the parent dataset. Does nothing without '--recursive' option. --source-bwlimit= Bandwidth limit on the source transfer --target-bwlimit= Bandwidth limit on the target transfer + --mbuffer-size=VALUE Specify the mbuffer size (default: 16M), please refer to mbuffer(1) manual page. --no-stream Replicates using newest snapshot instead of intermediates --no-sync-snap Does not create new snapshot, only transfers existing --no-clone-rollback Does not rollback clones on target From bd4eb491d8d101333381340382d1f1ab3a88d4d5 Mon Sep 17 00:00:00 2001 From: Matthew Swabey Date: Wed, 12 Sep 2018 22:22:03 -0400 Subject: [PATCH 140/148] Added in --sendoptions=OPTIONS and --recvoptions=OPTIONS to inject OPTIONS enabling things like syncoid --sendoptions=-Lcep and syncoid --recvoptions="-x property" Co-authored-by: Clint Armstrong --- syncoid | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/syncoid b/syncoid index d3e678f..009f10e 100755 --- a/syncoid +++ b/syncoid @@ -17,13 +17,24 @@ use Sys::Hostname; # Blank defaults to use ssh client's default # TODO: Merge into a single "sshflags" option? my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); -GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", +GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "sendoptions=s", "recvoptions=s", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", "no-clone-handling", "no-privilege-elevation", "force-delete", "no-clone-rollback", "no-rollback") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set +my $sendoptions = ''; +if (length $args{'sendoptions'}) { + $sendoptions = $args{'sendoptions'} +} + +my $recvoptions = ''; +if (length $args{'recvoptions'}) { + $recvoptions = $args{'recvoptions'} +} + + # TODO Expand to accept multiple sources? if (scalar(@ARGV) != 2) { print("Source or target not found!\n"); @@ -365,13 +376,13 @@ sub syncdataset { if (defined $args{'no-stream'}) { $oldestsnap = getnewestsnapshot(\%snaps); } my $oldestsnapescaped = escapeshellparam($oldestsnap); - my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped"; + my $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions $sourcefsescaped\@$oldestsnapescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped"; my $pvsize; if (defined $origin) { my $originescaped = escapeshellparam($origin); - $sendcmd = "$sourcesudocmd $zfscmd send -i $originescaped $sourcefsescaped\@$oldestsnapescaped"; + $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions -i $originescaped $sourcefsescaped\@$oldestsnapescaped"; my $streamargBackup = $args{'streamarg'}; $args{'streamarg'} = "-i"; $pvsize = getsendsize($sourcehost,$origin,"$sourcefs\@$oldestsnap",$sourceisroot); @@ -419,7 +430,7 @@ sub syncdataset { # $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); - $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; + $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions $args{'streamarg'} $sourcefsescaped\@$oldestsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap","$sourcefs\@$newsyncsnap",$sourceisroot); $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -456,8 +467,8 @@ sub syncdataset { # and because this will ony resume the receive to the next # snapshot, do a normal sync after that if (defined($receivetoken)) { - my $sendcmd = "$sourcesudocmd $zfscmd send -t $receivetoken"; - my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped"; + my $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions -t $receivetoken"; + my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"","",$sourceisroot,$receivetoken); my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -608,8 +619,8 @@ sub syncdataset { if ($nextsnapshot) { my $nextsnapshotescaped = escapeshellparam($nextsnapshot); - my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$nextsnapshotescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped"; + my $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$nextsnapshotescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped"; my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $nextsnapshot (~ $disp_pvsize):\n"; } @@ -623,8 +634,8 @@ sub syncdataset { $matchingsnap = $nextsnapshot; $matchingsnapescaped = escapeshellparam($matchingsnap); } else { - my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$newsyncsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped"; + my $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$newsyncsnapescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped"; my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $newsyncsnap (~ $disp_pvsize):\n"; } @@ -640,8 +651,8 @@ sub syncdataset { # do a normal replication if bookmarks aren't used or if previous # bookmark replication was only done to the next oldest snapshot if (!$bookmark || $nextsnapshot) { - my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped"; + my $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -1384,7 +1395,7 @@ sub getsendsize { $snaps = "-t $receivetoken"; } - my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send -nP $snaps"; + my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send $sendoptions -nP $snaps"; if ($debug) { print "DEBUG: getting estimated transfer size from source $sourcehost using \"$getsendsizecmd 2>&1 |\"...\n"; } open FH, "$getsendsizecmd 2>&1 |"; @@ -1494,6 +1505,8 @@ Options: --no-clone-rollback Does not rollback clones on target --no-rollback Does not rollback clones or snapshots on target (it probably requires a readonly target) --exclude=REGEX Exclude specific datasets which match the given regular expression. Can be specified multiple times + --sendoptions=OPTIONS DANGER: Inject OPTIONS into zfs send, e.g. syncoid --sendoptions="-Lce" sets zfs send -Lce ... + --recvoptions=OPTIONS DANGER: Inject OPTIONS into zfs received, e.g. syncoid --recvoptions="-x property" sets zfs receive -x property ... --sshkey=FILE Specifies a ssh public key to use to connect --sshport=PORT Connects to remote on a particular port --sshcipher|c=CIPHER Passes CIPHER to ssh to use a particular cipher set From f941d7f78244f2be1f9b3f7a3d826feccae993b4 Mon Sep 17 00:00:00 2001 From: WG Dev Date: Sun, 6 Jan 2019 10:46:24 +0100 Subject: [PATCH 141/148] fix for #316 - CRITICAL ERROR: bookmarks couldn't be listed for --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index d3e678f..b0a8ec8 100755 --- a/syncoid +++ b/syncoid @@ -1316,7 +1316,7 @@ sub getbookmarks() { close FH or $error = 1; if ($error == 1) { - if ($rawbookmarks[0] =~ /invalid type/) { + if ($rawbookmarks[0] =~ /invalid type/ or $rawbookmarks[0] =~ /operation not applicable to datasets of this type/) { # no support for zfs bookmarks, return empty hash return %bookmarks; } From 165faee70600bd866ea037c6c4fd123a6fd227d2 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Sun, 6 Jan 2019 15:29:29 -0500 Subject: [PATCH 142/148] remove spurious line break from HTML --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 99d8db1..4a57c32 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,7 @@ Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. -

sanoid rollback demo
(Real time demo: rolling back a full-scale cryptomalware infection in seconds!)

+

sanoid rollback demo
(Real time demo: rolling back a full-scale cryptomalware infection in seconds!)

More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: ``` From 50a237402156f816b7dbfba50f29a4e44e66dfd1 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Sun, 6 Jan 2019 15:41:18 -0500 Subject: [PATCH 143/148] Revert "Zfs Recursion" --- sanoid | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/sanoid b/sanoid index 9fcbb88..ce8207c 100755 --- a/sanoid +++ b/sanoid @@ -502,13 +502,7 @@ sub take_snapshots { # update to most current possible datestamp %datestamp = get_date(); # print "we should have had a $type snapshot of $path $maxage seconds ago; most recent is $newestage seconds old.\n"; - - # use zfs (atomic) recursion if specified in config - if ($config{$section}{'zfs_recursion'}) { - push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}${dateSuffix}_$type\@"); - } else { - push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}${dateSuffix}_$type"); - } + push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}${dateSuffix}_$type"); } } } @@ -516,14 +510,8 @@ sub take_snapshots { if ( (scalar(@newsnaps)) > 0) { foreach my $snap ( @newsnaps ) { - my @split = split '@', $snap, -1; - my $recursiveFlag = 0; - if (scalar(@split) == 3) { - $recursiveFlag = 1; - chop $snap; - } - my $dataset = $split[0]; - my $snapname = $split[1]; + my $dataset = (split '@', $snap)[0]; + my $snapname = (split '@', $snap)[1]; my $presnapshotfailure = 0; if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) { $ENV{'SANOID_TARGET'} = $dataset; @@ -542,13 +530,8 @@ sub take_snapshots { } if ($args{'verbose'}) { print "taking snapshot $snap\n"; } if (!$args{'readonly'}) { - if ($recursiveFlag) { - system($zfs, "snapshot", "-r", "$snap") == 0 - or warn "CRITICAL ERROR: $zfs snapshot -r $snap failed, $?"; - } else { - system($zfs, "snapshot", "$snap") == 0 - or warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; - } + system($zfs, "snapshot", "$snap") == 0 + or warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; } if ($config{$dataset}{'post_snapshot_script'} and !$args{'readonly'}) { if (!$presnapshotfailure or $config{$dataset}{'force_post_snapshot_script'}) { @@ -863,9 +846,7 @@ sub init { my $recursive = $ini{$section}{'recursive'} && grep( /^$ini{$section}{'recursive'}$/, @istrue ); my $skipChildren = $ini{$section}{'skip_children'} && grep( /^$ini{$section}{'skip_children'}$/, @istrue ); my @datasets; - if ($ini{$section}{'recursive'} =~ /zfs/i) { - $config{$section}{'zfs_recursion'} = 1; - } elsif ($recursive || $skipChildren) { + if ($recursive || $skipChildren) { @datasets = getchilddatasets($config{$section}{'path'}); DATASETS: foreach my $dataset(@datasets) { chomp $dataset; From ea482ce7b6d7f662701874c2ef889054831f36d5 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 8 Jan 2019 20:10:38 +0100 Subject: [PATCH 144/148] make tests work on FreeBSD --- tests/1_one_year/run.sh | 2 +- tests/2_dst_handling/run.sh | 2 +- tests/common/lib.sh | 14 +++++++++++++- tests/run-tests.sh | 2 +- .../1_bookmark_replication_intermediate/run.sh | 4 ++-- .../2_bookmark_replication_no_intermediate/run.sh | 4 ++-- tests/syncoid/3_force_delete/run.sh | 4 ++-- 7 files changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/1_one_year/run.sh b/tests/1_one_year/run.sh index 1cae7b4..88100da 100755 --- a/tests/1_one_year/run.sh +++ b/tests/1_one_year/run.sh @@ -39,7 +39,7 @@ function cleanUp { trap cleanUp EXIT while [ $timestamp -le $END ]; do - date --utc --set @$timestamp; date; "${SANOID}" --cron --verbose + setdate $timestamp; date; "${SANOID}" --cron --verbose timestamp=$((timestamp+3600)) done diff --git a/tests/2_dst_handling/run.sh b/tests/2_dst_handling/run.sh index eba21ed..7d7774e 100755 --- a/tests/2_dst_handling/run.sh +++ b/tests/2_dst_handling/run.sh @@ -42,7 +42,7 @@ function cleanUp { trap cleanUp EXIT while [ $timestamp -le $END ]; do - date --utc --set @$timestamp; date; "${SANOID}" --cron --verbose + setdate $timestamp; date; "${SANOID}" --cron --verbose timestamp=$((timestamp+900)) done diff --git a/tests/common/lib.sh b/tests/common/lib.sh index 78f128b..3aee40d 100644 --- a/tests/common/lib.sh +++ b/tests/common/lib.sh @@ -1,5 +1,7 @@ #!/bin/bash +unamestr="$(uname)" + function setup { export LANG=C export LANGUAGE=C @@ -90,7 +92,7 @@ function verifySnapshotList { message="${message}monthly snapshot count is wrong: ${monthly_count}\n" fi - checksum=$(sha256sum "${RESULT}" | cut -d' ' -f1) + checksum=$(shasum -a 256 "${RESULT}" | cut -d' ' -f1) if [ "${checksum}" != "${CHECKSUM}" ]; then failed=1 message="${message}result checksum mismatch\n" @@ -105,3 +107,13 @@ function verifySnapshotList { exit 1 } + +function setdate { + TIMESTAMP="$1" + + if [ "$unamestr" == 'FreeBSD' ]; then + date -u -f '%s' "${TIMESTAMP}" + else + date --utc --set "@${TIMESTAMP}" + fi +} diff --git a/tests/run-tests.sh b/tests/run-tests.sh index a8469e9..38054b0 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -15,7 +15,7 @@ for test in */; do echo -n "Running test ${testName} ... " cd "${test}" - echo | bash run.sh > "${LOGFILE}" 2>&1 + echo -n y | bash run.sh > "${LOGFILE}" 2>&1 if [ $? -eq 0 ]; then echo "[PASS]" diff --git a/tests/syncoid/1_bookmark_replication_intermediate/run.sh b/tests/syncoid/1_bookmark_replication_intermediate/run.sh index 11edb04..66af442 100755 --- a/tests/syncoid/1_bookmark_replication_intermediate/run.sh +++ b/tests/syncoid/1_bookmark_replication_intermediate/run.sh @@ -46,8 +46,8 @@ zfs snapshot "${POOL_NAME}"/src@snap5 ../../../syncoid --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1 # verify -output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name) -checksum=$(echo "${output}" | grep -v syncoid_ | sha256sum) +output=$(zfs list -t snapshot -r -H -o name "${POOL_NAME}") +checksum=$(echo "${output}" | grep -v syncoid_ | shasum -a 256) if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then exit 1 diff --git a/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh b/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh index 94ac690..f6c1755 100755 --- a/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh +++ b/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh @@ -46,8 +46,8 @@ zfs snapshot "${POOL_NAME}"/src@snap5 ../../../syncoid --no-stream --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1 # verify -output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name) -checksum=$(echo "${output}" | sha256sum) +output=$(zfs list -t snapshot -r -H -o name "${POOL_NAME}") +checksum=$(echo "${output}" | shasum -a 256) if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then exit 1 diff --git a/tests/syncoid/3_force_delete/run.sh b/tests/syncoid/3_force_delete/run.sh index 03ad9fa..25044cb 100755 --- a/tests/syncoid/3_force_delete/run.sh +++ b/tests/syncoid/3_force_delete/run.sh @@ -37,8 +37,8 @@ sleep 1 ../../../syncoid -r --force-delete --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1 # verify -output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name | sed 's/@syncoid_.*$'/@syncoid_/) -checksum=$(echo "${output}" | sha256sum) +output=$(zfs list -t snapshot -r -H -o name "${POOL_NAME}" | sed 's/@syncoid_.*$'/@syncoid_/) +checksum=$(echo "${output}" | shasum -a 256) if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then exit 1 From c3e20fdefcd0a61976155831c6b2b23ac7aa035f Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 9 Jan 2019 17:41:16 +0100 Subject: [PATCH 145/148] FreeBSD sed needs a different syntax --- tests/common/lib.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/common/lib.sh b/tests/common/lib.sh index 3aee40d..b070da2 100644 --- a/tests/common/lib.sh +++ b/tests/common/lib.sh @@ -60,7 +60,11 @@ function saveSnapshotList { zfs list -t snapshot -o name -Hr "${POOL_NAME}" | sort > "${RESULT}" # clear the seconds for comparing - sed -i 's/\(autosnap_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9]:[0-9][0-9]:\)[0-9][0-9]_/\100_/g' "${RESULT}" + if [ "$unamestr" == 'FreeBSD' ]; then + sed -i '' 's/\(autosnap_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9]:[0-9][0-9]:\)[0-9][0-9]_/\100_/g' "${RESULT}" + else + sed -i 's/\(autosnap_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9]:[0-9][0-9]:\)[0-9][0-9]_/\100_/g' "${RESULT}" + fi } function verifySnapshotList { @@ -109,11 +113,11 @@ function verifySnapshotList { } function setdate { - TIMESTAMP="$1" + TIMESTAMP="$1" - if [ "$unamestr" == 'FreeBSD' ]; then - date -u -f '%s' "${TIMESTAMP}" - else - date --utc --set "@${TIMESTAMP}" - fi + if [ "$unamestr" == 'FreeBSD' ]; then + date -u -f '%s' "${TIMESTAMP}" + else + date --utc --set "@${TIMESTAMP}" + fi } From 7ef435c5e31c00d47c8b4442334eed5df4ca175c Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Fri, 25 Jan 2019 14:48:17 -0500 Subject: [PATCH 146/148] add "hotspare" template (for local, hourly replication targets) --- sanoid.conf | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/sanoid.conf b/sanoid.conf index 9f13105..6bd5c62 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -69,6 +69,27 @@ daily_warn = 48 daily_crit = 60 +[template_hotspare] + autoprune = yes + frequently = 0 + hourly = 30 + daily = 90 + monthly = 3 + yearly = 0 + + ### don't take new snapshots - snapshots on backup + ### datasets are replicated in from source, not + ### generated locally + autosnap = no + + ### monitor hourlies and dailies, but don't warn or + ### crit until they're over 4h old, since replication + ### is typically hourly only + hourly_warn = 4h + hourly_crit = 6h + daily_warn = 2d + daily_crit = 4d + [template_scripts] ### dataset and snapshot name will be supplied as environment variables ### for all pre/post/prune scripts ($SANOID_TARGET, $SANOID_SNAPNAME) From d3f19c839782a112c03bed4d18cb13eff66d91a4 Mon Sep 17 00:00:00 2001 From: Simon Alman Date: Sun, 17 Feb 2019 07:17:03 +0000 Subject: [PATCH 147/148] Sanoid ebuild for Gentoo --- packages/gentoo/sys-fs/Manifest | 4 +++ packages/gentoo/sys-fs/files/sanoid.cron | 1 + packages/gentoo/sys-fs/sanoid-2.0.1.ebuild | 36 ++++++++++++++++++++ packages/gentoo/sys-fs/sanoid-9999.ebuild | 38 ++++++++++++++++++++++ 4 files changed, 79 insertions(+) create mode 100644 packages/gentoo/sys-fs/Manifest create mode 100644 packages/gentoo/sys-fs/files/sanoid.cron create mode 100644 packages/gentoo/sys-fs/sanoid-2.0.1.ebuild create mode 100644 packages/gentoo/sys-fs/sanoid-9999.ebuild diff --git a/packages/gentoo/sys-fs/Manifest b/packages/gentoo/sys-fs/Manifest new file mode 100644 index 0000000..4c629f9 --- /dev/null +++ b/packages/gentoo/sys-fs/Manifest @@ -0,0 +1,4 @@ +AUX sanoid.cron 45 BLAKE2B 3f6294bbbf485dc21a565cd2c8da05a42fb21cdaabdf872a21500f1a7338786c60d4a1fd188bbf81ce85f06a376db16998740996f47c049707a5109bdf02c052 SHA512 7676b32f21e517e8c84a097c7934b54097cf2122852098ea756093ece242125da3f6ca756a6fbb82fc348f84b94bfd61639e86e0bfa4bbe7abf94a8a4c551419 +DIST sanoid-2.0.1.tar.gz 106981 BLAKE2B 824b7271266ac9f9bf1fef5374a442215c20a4f139081f77d5d8db2ec7db9b8b349d9d0394c76f9d421a957853af64ff069097243f69e7e4b83a804f5ba992a6 SHA512 9d999b0f071bc3c3ca956df11e1501fd72a842f7d3315ede3ab3b5e0a36351100b6edbab8448bba65a2e187e4e8f77ff24671ed33b28f2fca9bb6ad0801aba9d +EBUILD sanoid-2.0.1.ebuild 772 BLAKE2B befbc479b5c79faa88ae21649ed31d1af70dbecb60416e8c879fffd9a3cdf9f3f508e12d8edc9f4e0afbf0e6ab0491a36fdae2af995a1984072dc5bffd63fe1d SHA512 d90a8b8ae40634e2f2e1fa11ba787cfcb461b75fa65b19c0d9a34eb458f07f510bbb1992f4a0e7a0e4aa5f55a5acdc064779c9a4f993b30eb5cbf39037f97858 +EBUILD sanoid-9999.ebuild 752 BLAKE2B 073533436c6f5c47b9e8410c898bf86b605d61c9b16a08b57253f5a87ad583e00d935ae9ea90f98b42c20dc1fbda0b9f1a8a7bf5be1cf3daf20afc640f1428ca SHA512 40ad34230fdb538bbdcda2d8149f37eac2a0e2accce5f79f7ba77d8e62e3fd78e997d8143baa0e050f548f90ce1cb6827e50b536b5e3acc444c6032f170251be diff --git a/packages/gentoo/sys-fs/files/sanoid.cron b/packages/gentoo/sys-fs/files/sanoid.cron new file mode 100644 index 0000000..09169ad --- /dev/null +++ b/packages/gentoo/sys-fs/files/sanoid.cron @@ -0,0 +1 @@ +* * * * * root TZ=UTC /usr/bin/sanoid --cron diff --git a/packages/gentoo/sys-fs/sanoid-2.0.1.ebuild b/packages/gentoo/sys-fs/sanoid-2.0.1.ebuild new file mode 100644 index 0000000..5a8d67e --- /dev/null +++ b/packages/gentoo/sys-fs/sanoid-2.0.1.ebuild @@ -0,0 +1,36 @@ +# Copyright 2019 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +EAPI=7 + +DESCRIPTION="Policy-driven snapshot management and replication tools for ZFS" +HOMEPAGE="https://github.com/jimsalterjrs/sanoid" +SRC_URI="https://github.com/jimsalterjrs/${PN}/archive/v${PV}.tar.gz -> ${P}.tar.gz" + +LICENSE="GPL-3.0" +SLOT="0" +KEYWORDS="~x86 ~amd64" +IUSE="" + +DEPEND="app-arch/lzop + dev-perl/Config-IniFiles + sys-apps/pv + sys-block/mbuffer + virtual/perl-Data-Dumper" +RDEPEND="${DEPEND}" +BDEPEND="" + +DOCS=( README.md ) + +src_install() { + dobin findoid + dobin sanoid + dobin sleepymutex + dobin syncoid + keepdir /etc/${PN} + insinto /etc/${PN} + doins sanoid.conf + doins sanoid.defaults.conf + insinto /etc/cron.d + newins "${FILESDIR}/${PN}.cron" ${PN} +} diff --git a/packages/gentoo/sys-fs/sanoid-9999.ebuild b/packages/gentoo/sys-fs/sanoid-9999.ebuild new file mode 100644 index 0000000..7eaf509 --- /dev/null +++ b/packages/gentoo/sys-fs/sanoid-9999.ebuild @@ -0,0 +1,38 @@ +# Copyright 2019 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +EAPI=7 + +EGIT_REPO_URI="https://github.com/jimsalterjrs/${PN}.git" +inherit git-r3 + +DESCRIPTION="Policy-driven snapshot management and replication tools for ZFS" +HOMEPAGE="https://github.com/jimsalterjrs/sanoid" + +LICENSE="GPL-3.0" +SLOT="0" +KEYWORDS="**" +IUSE="" + +DEPEND="app-arch/lzop + dev-perl/Config-IniFiles + sys-apps/pv + sys-block/mbuffer + virtual/perl-Data-Dumper" +RDEPEND="${DEPEND}" +BDEPEND="" + +DOCS=( README.md ) + +src_install() { + dobin findoid + dobin sanoid + dobin sleepymutex + dobin syncoid + keepdir /etc/${PN} + insinto /etc/${PN} + doins sanoid.conf + doins sanoid.defaults.conf + insinto /etc/cron.d + newins "${FILESDIR}/${PN}.cron" ${PN} +} From eefc659c0396a90b6afa5524c3b903092771e29c Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 20 Feb 2019 08:15:45 +0100 Subject: [PATCH 148/148] update the install instruction with recent changes --- INSTALL.md | 55 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 94a9f2e..3a941a6 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -5,7 +5,7 @@ - [Installation](#installation) - - [Ubuntu](#ubuntu) + - [Debian/Ubuntu](#debianubuntu) - [CentOS](#centos) - [FreeBSD](#freebsd) - [Other OSes](#other-oses) @@ -15,7 +15,7 @@ -## Ubuntu +## Debian/Ubuntu Install prerequisite software: @@ -23,6 +23,24 @@ Install prerequisite software: apt install libconfig-inifiles-perl pv lzop mbuffer ``` +Clone this repo, build the debian package and install it (alternatively you can skip the package and do it manually like described below for CentOS): + +```bash +# Download the repo as root to avoid changing permissions later +sudo git clone https://github.com/jimsalterjrs/sanoid.git +cd sanoid +ln -s packages/debian . +dpkg-buildpackage -uc -us +apt install ../sanoid_*_all.deb +``` + +Enable sanoid timer: +```bash +# enable and start the sanoid timer +sudo systemctl enable sanoid.timer +sudo systemctl start sanoid.timer +``` + ## CentOS Install prerequisite software: @@ -60,23 +78,42 @@ cat << "EOF" | sudo tee /etc/systemd/system/sanoid.service Description=Snapshot ZFS Pool Requires=zfs.target After=zfs.target +ConditionFileNotEmpty=/etc/sanoid/sanoid.conf [Service] +Environment=TZ=UTC Type=oneshot -ExecStart=/usr/sbin/sanoid --cron +ExecStart=/usr/sbin/sanoid --take-snapshots +EOF + +cat << "EOF" | sudo tee /etc/systemd/system/sanoid-prune.service +[Unit] +Description=Cleanup ZFS Pool +Requires=zfs.target +After=zfs.target sanoid.service +ConditionFileNotEmpty=/etc/sanoid/sanoid.conf + +[Service] +Environment=TZ=UTC +Type=oneshot +ExecStart=/usr/sbin/sanoid --prune-snapshots + +[Install] +WantedBy=sanoid.service EOF ``` -And a systemd timer that will execute **Sanoid** once per minute: +And a systemd timer that will execute **Sanoid** once per quarter hour +(Decrease the interval as suitable for configuration): ```bash cat << "EOF" | sudo tee /etc/systemd/system/sanoid.timer [Unit] -Description=Run Sanoid Every Minute +Description=Run Sanoid Every 15 Minutes Requires=sanoid.service [Timer] -OnCalendar=*:0/1 +OnCalendar=*:0/15 Persistent=true [Install] @@ -100,7 +137,7 @@ Now, proceed to configure [**Sanoid**](#configuration) Install prerequisite software: ```bash -pkg install p5-Config-Inifiles pv lzop +pkg install p5-Config-Inifiles pv mbuffer lzop ``` **Additional notes:** @@ -109,6 +146,8 @@ pkg install p5-Config-Inifiles pv lzop * Simplest path workaround is symlinks, eg `ln -s /usr/local/bin/lzop /usr/bin/lzop` or similar, as appropriate to create links in **/usr/bin** to wherever the utilities actually are on your system. +* See note about mbuffer and other things in FREEBSD.readme + ## Other OSes **Sanoid** depends on the Perl module Config::IniFiles and will not operate without it. Config::IniFiles may be installed from CPAN, though the project strongly recommends using your distribution's repositories instead. @@ -130,4 +169,4 @@ pkg install p5-Config-Inifiles pv lzop ## Sanoid -Instructions on how to set up `sanoid.conf`. Maybe just copy/paste the example `sanoid.conf` file in here but clean it up a little bit. +Take a look at the files `sanoid.defaults.conf` and` sanoid.conf.example` for all possible configuration options. Also have a look at the README.md