From d8613d13797db640f9cf7432f29677056bddeb43 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 7 Nov 2017 08:03:53 +0100 Subject: [PATCH 01/97] implemented frequent snapshots with configurable period Fixes #75 --- sanoid | 21 +++++++++++++++++---- sanoid.conf | 2 ++ sanoid.defaults.conf | 7 +++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/sanoid b/sanoid index d6e58ce..e8756db 100755 --- a/sanoid +++ b/sanoid @@ -121,12 +121,13 @@ sub monitor_snapshots { my $path = $config{$section}{'path'}; push @paths, $path; - my @types = ('yearly','monthly','daily','hourly'); + my @types = ('yearly','monthly','daily','hourly','frequently'); foreach my $type (@types) { my $smallerperiod = 0; # we need to set the period length in seconds first - if ($type eq 'hourly') { $smallerperiod = 60; } + if ($type eq 'frequently') { $smallerperiod = 1; } + elsif ($type eq 'hourly') { $smallerperiod = 60; } elsif ($type eq 'daily') { $smallerperiod = 60*60; } elsif ($type eq 'monthly') { $smallerperiod = 60*60*24; } elsif ($type eq 'yearly') { $smallerperiod = 60*60*24; } @@ -200,7 +201,8 @@ sub prune_snapshots { unless ($type =~ /ly$/) { next; } # we need to set the period length in seconds first - if ($type eq 'hourly') { $period = 60*60; } + if ($type eq 'frequently') { $period = 60 * $config{$section}{'frequent_period'}; } + elsif ($type eq 'hourly') { $period = 60*60; } elsif ($type eq 'daily') { $period = 60*60*24; } elsif ($type eq 'monthly') { $period = 60*60*24*31; } elsif ($type eq 'yearly') { $period = 60*60*24*365.25; } @@ -291,7 +293,18 @@ sub take_snapshots { my @preferredtime; my $lastpreferred; - if ($type eq 'hourly') { + if ($type eq 'frequently') { + my $frequentslice = int($datestamp{'min'} / $config{$section}{'frequent_period'}); + + push @preferredtime,0; # try to hit 0 seconds + push @preferredtime,$frequentslice * $config{$section}{'frequent_period'}; + push @preferredtime,$datestamp{'hour'}; + push @preferredtime,$datestamp{'mday'}; + push @preferredtime,($datestamp{'mon'}-1); # january is month 0 + push @preferredtime,$datestamp{'year'}; + $lastpreferred = timelocal(@preferredtime); + if ($lastpreferred > time()) { $lastpreferred -= 60 * $config{$section}{'frequent_period'}; } # preferred time is later this frequent period - so look at last frequent period + } elsif ($type eq 'hourly') { push @preferredtime,0; # try to hit 0 seconds push @preferredtime,$config{$section}{'hourly_min'}; push @preferredtime,$datestamp{'hour'}; diff --git a/sanoid.conf b/sanoid.conf index 9b1f19d..b999634 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -40,6 +40,7 @@ daily = 60 [template_production] + frequently = 0 hourly = 36 daily = 30 monthly = 3 @@ -49,6 +50,7 @@ [template_backup] autoprune = yes + frequently = 0 hourly = 30 daily = 90 monthly = 12 diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 35c804d..b5e4e63 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -16,12 +16,17 @@ recursive = use_template = process_children_only = +# The period in minutes for frequent snapshots, +# should be in the range of 1-30 and divide an hour without remainder +frequent_period = 15 + # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately # prune any of those type snapshots already present. # # Otherwise, if autoprune is set, we will prune any snapshots of that type which are older # than (setting * periodicity) - so if daily = 90, we'll prune any dailies older than 90 days. autoprune = yes +frequently = 0 hourly = 48 daily = 90 monthly = 6 @@ -62,6 +67,8 @@ yearly_min = 0 monitor = yes monitor_dont_warn = no monitor_dont_crit = no +frequently_warn = 2000 +frequently_crit = 8000 hourly_warn = 90 hourly_crit = 360 daily_warn = 28 From 21a6881c4bbf97ebd21c3da44bdb7dd8b5192eb3 Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Wed, 8 Nov 2017 16:19:41 +0000 Subject: [PATCH 02/97] Unrevert syncoid --- syncoid | 349 ++++++++++++++++++++++++++------------------------------ 1 file changed, 162 insertions(+), 187 deletions(-) diff --git a/syncoid b/syncoid index d927cba..8694561 100755 --- a/syncoid +++ b/syncoid @@ -4,25 +4,43 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.16'; +$::VERSION = '1.4.16'; use strict; use warnings; use Data::Dumper; +use Getopt::Long qw(:config auto_version auto_help); +use Pod::Usage; use Time::Local; use Sys::Hostname; -my %args = getargs(@ARGV); +# Blank defaults to use ssh client's default +# TODO: Merge into a single "sshflags" option? +my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); +GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", + "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", + "debug", "quiet", "no-stream", "no-sync-snap") or pod2usage(2); -if ($args{'version'}) { - print "Syncoid version: $version\n"; - exit 0; +$args{'compress'} = compressargset($args{'compress'} || 'default'); # Can't be done with GetOptions arg, as default still needs to be set + +# TODO Expand to accept multiple sources? +if (scalar(@ARGV) != 2) { + print("Source or target not found!\n"); + pod2usage(2); + exit 127; +} else { + $args{'source'} = $ARGV[0]; + $args{'target'} = $ARGV[1]; } -if (!(defined $args{'source'} && defined $args{'target'})) { - print 'usage: syncoid [src_user@src_host:]src_pool/src_dataset [dst_user@dst_host:]dst_pool/dst_dataset'."\n"; - exit 127; +# Could possibly merge these into an options function +if (length $args{'source-bwlimit'}) { + $args{'source-bwlimit'} = "-R $args{'source-bwlimit'}"; } +if (length $args{'target-bwlimit'}) { + $args{'target-bwlimit'} = "-r $args{'target-bwlimit'}"; +} +$args{'streamarg'} = (defined $args{'no-stream'} ? '-i' : '-I'); my $rawsourcefs = $args{'source'}; my $rawtargetfs = $args{'target'}; @@ -32,28 +50,7 @@ my $quiet = $args{'quiet'}; my $zfscmd = '/sbin/zfs'; my $sshcmd = '/usr/bin/ssh'; my $pscmd = '/bin/ps'; -my $sshcipher; -if (defined $args{'c'}) { - $sshcipher = "-c $args{'c'}"; -} else { - # default to no cipher specified now that SSH - # has not defaulted to a cripplingly slow cipher - # in a very long time - $sshcipher = ''; -} -my $sshport = '-p 22'; -my $sshoption; -if (defined $args{'o'}) { - my @options = split(',', $args{'o'}); - foreach my $option (@options) { - $sshoption .= " -o $option"; - if ($option eq "NoneSwitch=yes") { - $sshcipher = ""; - } - } -} else { - $sshoption = ""; -} + my $pvcmd = '/usr/bin/pv'; my $mbuffercmd = '/usr/bin/mbuffer'; my $sudocmd = '/usr/bin/sudo'; @@ -62,23 +59,25 @@ my $mbufferoptions = '-q -s 128k -m 16M 2>/dev/null'; # being present on remote machines. my $lscmd = '/bin/ls'; -if ( $args{'sshport'} ) { - $sshport = "-p $args{'sshport'}"; +if (length $args{'sshcipher'}) { + $args{'sshcipher'} = "-c $args{'sshcipher'}"; } +if (length $args{'sshport'}) { + $args{'sshport'} = "-p $args{'sshport'}"; +} +if (length $args{'sshkey'}) { + $args{'sshkey'} = "-i $args{'sshkey'}"; +} +my $sshoptions = join " ", map { "-o " . $_ } @{$args{'sshoption'}}; # deref required + # figure out if source and/or target are remote. -if ( $args{'sshkey'} ) { - $sshcmd = "$sshcmd $sshoption $sshcipher $sshport -i $args{'sshkey'}"; -} -else { - $sshcmd = "$sshcmd $sshoption $sshcipher $sshport"; -} +$sshcmd = "$sshcmd $args{'sshcipher'} $sshoptions $args{'sshport'} $args{'sshkey'}"; +if ($debug) { print "DEBUG: SSHCMD: $sshcmd\n"; } my ($sourcehost,$sourcefs,$sourceisroot) = getssh($rawsourcefs); my ($targethost,$targetfs,$targetisroot) = getssh($rawtargetfs); -my $sourcesudocmd; -my $targetsudocmd; -if ($sourceisroot) { $sourcesudocmd = ''; } else { $sourcesudocmd = $sudocmd; } -if ($targetisroot) { $targetsudocmd = ''; } else { $targetsudocmd = $sudocmd; } +my $sourcesudocmd = $sourceisroot ? '' : $sudocmd; +my $targetsudocmd = $targetisroot ? '' : $sudocmd; # figure out whether compression, mbuffering, pv # are available on source, target, local machines. @@ -91,7 +90,7 @@ my %snaps; ## can loop across children separately, for recursive ## ## replication ## -if (! $args{'recursive'}) { +if (!defined $args{'recursive'}) { syncdataset($sourcehost, $sourcefs, $targethost, $targetfs); } else { if ($debug) { print "DEBUG: recursive sync of $sourcefs.\n"; } @@ -164,17 +163,21 @@ sub syncdataset { %snaps = (%sourcesnaps, %targetsnaps); } - if ($args{'dumpsnaps'}) { print "merged snapshot list: \n"; dumphash(\%snaps); print "\n\n\n"; } + if (defined $args{'dumpsnaps'}) { + print "merged snapshot list of $targetfs: \n"; + dumphash(\%snaps); + print "\n\n\n"; + } # create a new syncoid snapshot on the source filesystem. my $newsyncsnap; - if (!defined ($args{'no-sync-snap'}) ) { + if (!defined $args{'no-sync-snap'}) { $newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot); } else { # we don't want sync snapshots created, so use the newest snapshot we can find. $newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot); if ($newsyncsnap eq 0) { - warn "CRITICAL: no snapshots exist on source, and you asked for --no-sync-snap.\n"; + warn "CRITICAL: no snapshots exist on source $sourcefs, and you asked for --no-sync-snap.\n"; return 0; } } @@ -285,7 +288,7 @@ sub syncdataset { my $targetsize = getzfsvalue($targethost,$targetfs,$targetisroot,'-p used'); - my $matchingsnap = getmatchingsnapshot($targetsize, \%snaps); + my $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, $targetsize, \%snaps); if (! $matchingsnap) { # no matching snapshot; we whined piteously already, but let's go ahead and return false # now in case more child datasets need replication. @@ -337,111 +340,54 @@ sub syncdataset { } # end syncdataset() +sub compressargset { + my ($value) = @_; + my $DEFAULT_COMPRESSION = 'lzo'; + my %COMPRESS_ARGS = ( + 'none' => { + rawcmd => '', + args => '', + decomrawcmd => '', + decomargs => '', + }, + 'gzip' => { + rawcmd => '/bin/gzip', + args => '-3', + decomrawcmd => '/bin/zcat', + decomargs => '', + }, + 'pigz-fast' => { + rawcmd => '/usr/bin/pigz', + args => '-3', + decomrawcmd => '/usr/bin/pigz', + decomargs => '-dc', + }, + 'pigz-slow' => { + rawcmd => '/usr/bin/pigz', + args => '-9', + decomrawcmd => '/usr/bin/pigz', + decomargs => '-dc', + }, + 'lzo' => { + rawcmd => '/usr/bin/lzop', + args => '', + decomrawcmd => '/usr/bin/lzop', + decomargs => '-dfc', + }, + ); -sub getargs { - my @args = @_; - my %args; - - my %novaluearg; - my %validarg; - push my @validargs, ('debug','nocommandchecks','version','monitor-version','compress','c','o','source-bwlimit','target-bwlimit','dumpsnaps','recursive','r','sshkey','sshport','quiet','no-stream','no-sync-snap'); - foreach my $item (@validargs) { $validarg{$item} = 1; } - push my @novalueargs, ('debug','nocommandchecks','version','monitor-version','dumpsnaps','recursive','r','quiet','no-stream','no-sync-snap'); - foreach my $item (@novalueargs) { $novaluearg{$item} = 1; } - - while (my $rawarg = shift(@args)) { - my $arg = $rawarg; - my $argvalue = ''; - if ($rawarg =~ /=/) { - # user specified the value for a CLI argument with = - # instead of with blank space. separate appropriately. - $argvalue = $arg; - $arg =~ s/=.*$//; - $argvalue =~ s/^.*=//; - } - if ($rawarg =~ /^--/) { - # doubledash arg - $arg =~ s/^--//; - if (! $validarg{$arg}) { die "ERROR: don't understand argument $rawarg.\n"; } - if ($novaluearg{$arg}) { - $args{$arg} = 1; - } else { - # if this CLI arg takes a user-specified value and - # we don't already have it, then the user must have - # specified with a space, so pull in the next value - # from the array as this value rather than as the - # next argument. - if ($argvalue eq '') { $argvalue = shift(@args); } - $args{$arg} = $argvalue; - } - } elsif ($arg =~ /^-/) { - # singledash arg - $arg =~ s/^-//; - if (! $validarg{$arg}) { die "ERROR: don't understand argument $rawarg.\n"; } - if ($novaluearg{$arg}) { - $args{$arg} = 1; - } else { - # if this CLI arg takes a user-specified value and - # we don't already have it, then the user must have - # specified with a space, so pull in the next value - # from the array as this value rather than as the - # next argument. - if ($argvalue eq '') { $argvalue = shift(@args); } - $args{$arg} = $argvalue; - } - } else { - # bare arg - if (defined $args{'source'}) { - if (! defined $args{'target'}) { - $args{'target'} = $arg; - } else { - die "ERROR: don't know what to do with third bare argument $rawarg.\n"; - } - } else { - $args{'source'} = $arg; - } - } + if ($value eq 'default') { + $value = $DEFAULT_COMPRESSION; + } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'lzo', 'default', 'none'))) { + warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION"; + $value = $DEFAULT_COMPRESSION; } - if (defined $args{'source-bwlimit'}) { $args{'source-bwlimit'} = "-R $args{'source-bwlimit'}"; } else { $args{'source-bwlimit'} = ''; } - if (defined $args{'target-bwlimit'}) { $args{'target-bwlimit'} = "-r $args{'target-bwlimit'}"; } else { $args{'target-bwlimit'} = ''; } - - if (defined $args{'no-stream'}) { $args{'streamarg'} = '-i'; } else { $args{'streamarg'} = '-I'; } - - if ($args{'r'}) { $args{'recursive'} = $args{'r'}; } - - if (!defined $args{'compress'}) { $args{'compress'} = 'default'; } - - if ($args{'compress'} eq 'gzip') { - $args{'rawcompresscmd'} = '/bin/gzip'; - $args{'compressargs'} = '-3'; - $args{'rawdecompresscmd'} = '/bin/zcat'; - $args{'decompressargs'} = ''; - } elsif ( ($args{'compress'} eq 'pigz-fast')) { - $args{'rawcompresscmd'} = '/usr/bin/pigz'; - $args{'compressargs'} = '-3'; - $args{'rawdecompresscmd'} = '/usr/bin/pigz'; - $args{'decompressargs'} = '-dc'; - } elsif ( ($args{'compress'} eq 'pigz-slow')) { - $args{'rawcompresscmd'} = '/usr/bin/pigz'; - $args{'compressargs'} = '-9'; - $args{'rawdecompresscmd'} = '/usr/bin/pigz'; - $args{'decompressargs'} = '-dc'; - } elsif ( ($args{'compress'} eq 'lzo') || ($args{'compress'} eq 'default') ) { - $args{'rawcompresscmd'} = '/usr/bin/lzop'; - $args{'compressargs'} = ''; - $args{'rawdecompresscmd'} = '/usr/bin/lzop'; - $args{'decompressargs'} = '-dfc'; - } else { - $args{'rawcompresscmd'} = ''; - $args{'compressargs'} = ''; - $args{'rawdecompresscmd'} = ''; - $args{'decompressargs'} = ''; - } - $args{'compresscmd'} = "$args{'rawcompresscmd'} $args{'compressargs'}"; - $args{'decompresscmd'} = "$args{'rawdecompresscmd'} $args{'decompressargs'}"; - - return %args; + my %comargs = %{$COMPRESS_ARGS{$value}}; # copy + $comargs{'compress'} = $value; + $comargs{'cmd'} = "$comargs{'rawcmd'} $comargs{'args'}"; + $comargs{'decomcmd'} = "$comargs{'decomrawcmd'} $comargs{'decomargs'}"; + return \%comargs; } sub checkcommands { @@ -471,24 +417,15 @@ sub checkcommands { # if raw compress command is null, we must have specified no compression. otherwise, # make sure that compression is available everywhere we need it - if ($args{'rawcompresscmd'} eq '') { - $avail{'sourcecompress'} = 0; - $avail{'sourcecompress'} = 0; - $avail{'localcompress'} = 0; - if ($args{'compress'} eq 'none' || - $args{'compress'} eq 'no' || - $args{'compress'} eq '0') { - if ($debug) { print "DEBUG: compression forced off from command line arguments.\n"; } - } else { - print "WARN: value $args{'compress'} for argument --compress not understood, proceeding without compression.\n"; - } + if ($args{'compress'}{'compress'} eq 'none') { + if ($debug) { print "DEBUG: compression forced off from command line arguments.\n"; } } else { - if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on source...\n"; } - $avail{'sourcecompress'} = `$sourcessh $lscmd $args{'rawcompresscmd'} 2>/dev/null`; - if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on target...\n"; } - $avail{'targetcompress'} = `$targetssh $lscmd $args{'rawcompresscmd'} 2>/dev/null`; - if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on local machine...\n"; } - $avail{'localcompress'} = `$lscmd $args{'rawcompresscmd'} 2>/dev/null`; + if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on source...\n"; } + $avail{'sourcecompress'} = `$sourcessh $lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`; + if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on target...\n"; } + $avail{'targetcompress'} = `$targetssh $lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`; + if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on local machine...\n"; } + $avail{'localcompress'} = `$lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`; } my ($s,$t); @@ -514,14 +451,14 @@ sub checkcommands { if ($avail{'sourcecompress'} eq '') { - if ($args{'rawcompresscmd'} ne '') { - print "WARN: $args{'compresscmd'} not available on source $s- sync will continue without compression.\n"; + if ($args{'compress'}{'rawcmd'} ne '') { + print "WARN: $args{'compress'}{'rawcmd'} not available on source $s- sync will continue without compression.\n"; } $avail{'compress'} = 0; } if ($avail{'targetcompress'} eq '') { - if ($args{'rawcompresscmd'} ne '') { - print "WARN: $args{'compresscmd'} not available on target $t - sync will continue without compression.\n"; + if ($args{'compress'}{'rawcmd'} ne '') { + print "WARN: $args{'compress'}{'rawcmd'} not available on target $t - sync will continue without compression.\n"; } $avail{'compress'} = 0; } @@ -533,8 +470,8 @@ sub checkcommands { # corner case - if source AND target are BOTH remote, we have to check for local compress too if ($sourcehost ne '' && $targethost ne '' && $avail{'localcompress'} eq '') { - if ($args{'rawcompresscmd'} ne '') { - print "WARN: $args{'compresscmd'} not available on local machine - sync will continue without compression.\n"; + if ($args{'compress'}{'rawcmd'} ne '') { + print "WARN: $args{'compress'}{'rawcmd'} not available on local machine - sync will continue without compression.\n"; } $avail{'compress'} = 0; } @@ -690,9 +627,9 @@ sub buildsynccmd { $synccmd = "$sendcmd |"; # avoid confusion - accept either source-bwlimit or target-bwlimit as the bandwidth limiting option here my $bwlimit = ''; - if (defined $args{'source-bwlimit'}) { + if (length $args{'bwlimit'}) { $bwlimit = $args{'source-bwlimit'}; - } elsif (defined $args{'target-bwlimit'}) { + } elsif (length $args{'target-bwlimit'}) { $bwlimit = $args{'target-bwlimit'}; } @@ -701,18 +638,18 @@ sub buildsynccmd { $synccmd .= " $recvcmd"; } elsif ($sourcehost eq '') { # local source, remote target. - #$synccmd = "$sendcmd | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'"; + #$synccmd = "$sendcmd | $pvcmd | $args{'compress'}{'cmd'} | $mbuffercmd | $sshcmd $targethost '$args{'compress'}{'decomcmd'} | $mbuffercmd | $recvcmd'"; $synccmd = "$sendcmd |"; if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; } - if ($avail{'compress'}) { $synccmd .= " $args{'compresscmd'} |"; } + if ($avail{'compress'}) { $synccmd .= " $args{'compress'}{'cmd'} |"; } if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; } $synccmd .= " $sshcmd $targethost '"; if ($avail{'targetmbuffer'}) { $synccmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; } - if ($avail{'compress'}) { $synccmd .= " $args{'decompresscmd'} |"; } + if ($avail{'compress'}) { $synccmd .= " $args{'compress'}{'decomcmd'} |"; } $synccmd .= " $recvcmd'"; } elsif ($targethost eq '') { # remote source, local target. - #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $mbuffercmd | $pvcmd | $recvcmd"; + #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compress'}{'cmd'} | $mbuffercmd' | $args{'decompress'}{'cmd'} | $mbuffercmd | $pvcmd | $recvcmd"; $synccmd = "$sshcmd $sourcehost '$sendcmd"; if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; } if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } @@ -723,18 +660,18 @@ sub buildsynccmd { $synccmd .= "$recvcmd"; } else { #remote source, remote target... weird, but whatever, I'm not here to judge you. - #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'"; + #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compress'}{'cmd'} | $mbuffercmd' | $args{'compress'}{'decomcmd'} | $pvcmd | $args{'compress'}{'cmd'} | $mbuffercmd | $sshcmd $targethost '$args{'compress'}{'decomcmd'} | $mbuffercmd | $recvcmd'"; $synccmd = "$sshcmd $sourcehost '$sendcmd"; - if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; } + if ($avail{'compress'}) { $synccmd .= " | $args{'compress'}{'cmd'}"; } if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } $synccmd .= "' | "; - if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'decomcmd'} | "; } if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } - if ($avail{'compress'}) { $synccmd .= "$args{'compresscmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'cmd'} | "; } if ($avail{'localmbuffer'}) { $synccmd .= "$mbuffercmd $mbufferoptions | "; } $synccmd .= "$sshcmd $targethost '"; if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } - if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'decomcmd'} | "; } $synccmd .= "$recvcmd'"; } return $synccmd; @@ -794,7 +731,7 @@ sub pruneoldsyncsnaps { } sub getmatchingsnapshot { - my ($targetsize, $snaps) = shift; + my ($sourcefs, $targetfs, $targetsize, $snaps) = @_; foreach my $snap ( sort { $snaps{'source'}{$b}{'creation'}<=>$snaps{'source'}{$a}{'creation'} } keys %{ $snaps{'source'} }) { if (defined $snaps{'target'}{$snap}{'guid'}) { if ($snaps{'source'}{$snap}{'guid'} == $snaps{'target'}{$snap}{'guid'}) { @@ -806,7 +743,7 @@ sub getmatchingsnapshot { # if we got this far, we failed to find a matching snapshot. print "\n"; - print "CRITICAL ERROR: Target exists but has no matching snapshots!\n"; + print "CRITICAL ERROR: Target $targetfs exists but has no snapshots matching with $sourcefs!\n"; print " Replication to target would require destroying existing\n"; print " target. Cowardly refusing to destroy your existing target.\n\n"; @@ -814,7 +751,7 @@ sub getmatchingsnapshot { # zfs create targetpool/targetsnap ; syncoid sourcepool/sourcesnap targetpool/targetsnap ... if ( $targetsize < (64*1024*1024) ) { - print " NOTE: Target dataset is < 64MB used - did you mistakenly run\n"; + print " NOTE: Target $targetfs dataset is < 64MB used - did you mistakenly run\n"; print " \`zfs create $args{'target'}\` on the target? ZFS initial\n"; print " replication must be to a NON EXISTENT DATASET, which will\n"; print " then be CREATED BY the initial replication process.\n\n"; @@ -868,7 +805,7 @@ sub getssh { if ($remoteuser eq 'root') { $isroot = 1; } else { $isroot = 0; } # now we need to establish a persistent master SSH connection $socket = "/tmp/syncoid-$remoteuser-$rhost-" . time(); - open FH, "$sshcmd -M -S $socket -o ControlPersist=1m $sshport $rhost exit |"; + open FH, "$sshcmd -M -S $socket -o ControlPersist=1m $args{'sshport'} $rhost exit |"; close FH; $rhost = "-S $socket $rhost"; } else { @@ -987,3 +924,41 @@ sub getdate { $date{'stamp'} = "$date{'year'}-$date{'mon'}-$date{'mday'}:$date{'hour'}:$date{'min'}:$date{'sec'}"; return %date; } + +__END__ + +=head1 NAME + +syncoid - ZFS snapshot replication tool + +=head1 SYNOPSIS + + syncoid [options]... SOURCE TARGET + or syncoid [options]... SOURCE [USER@]HOST:TARGET + or syncoid [options]... [USER@]HOST:SOURCE [TARGET] + or syncoid [options]... [USER@]HOST:SOURCE [USER@]HOST:TARGET + + SOURCE Source ZFS dataset. Can be either local or remote + TARGET Target ZFS dataset. Can be either local or remote + +Options: + + --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none + --recursive|r Also transfers child datasets + --source-bwlimit= Bandwidth limit on the source transfer + --target-bwlimit= Bandwidth limit on the target transfer + --no-stream Replicates using newest snapshot instead of intermediates + --no-sync-snap Does not create new snapshot, only transfers existing + + --sshkey=FILE Specifies a ssh public key to use to connect + --sshport=PORT Connects to remote on a particular port + --sshcipher|c=CIPHER Passes CIPHER to ssh to use a particular cipher set + --sshoption|o=OPTION Passes OPTION to ssh for remote usage. Can be specified multiple times + + --help Prints this helptext + --verbose Prints the version number + --debug Prints out a lot of additional information during a syncoid run + --monitor-version Currently does nothing + --quiet Suppresses non-error output + --dumpsnaps Dumps a list of snapshots during the run + --no-command-checks Do not check command existence before attempting transfer. Not recommended From 11da17fa4a82f0eb668131fe7d4587c25cfbeb05 Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Wed, 8 Nov 2017 16:19:59 +0000 Subject: [PATCH 03/97] Fix compress arg handling --- syncoid | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/syncoid b/syncoid index 8694561..3f07990 100755 --- a/syncoid +++ b/syncoid @@ -21,7 +21,7 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", "debug", "quiet", "no-stream", "no-sync-snap") or pod2usage(2); -$args{'compress'} = compressargset($args{'compress'} || 'default'); # Can't be done with GetOptions arg, as default still needs to be set +my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set # TODO Expand to accept multiple sources? if (scalar(@ARGV) != 2) { @@ -417,15 +417,15 @@ sub checkcommands { # if raw compress command is null, we must have specified no compression. otherwise, # make sure that compression is available everywhere we need it - if ($args{'compress'}{'compress'} eq 'none') { + if ($compressargs{'compress'} eq 'none') { if ($debug) { print "DEBUG: compression forced off from command line arguments.\n"; } } else { - if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on source...\n"; } - $avail{'sourcecompress'} = `$sourcessh $lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`; - if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on target...\n"; } - $avail{'targetcompress'} = `$targetssh $lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`; - if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on local machine...\n"; } - $avail{'localcompress'} = `$lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`; + if ($debug) { print "DEBUG: checking availability of $compressargs{'rawcmd'} on source...\n"; } + $avail{'sourcecompress'} = `$sourcessh $lscmd $compressargs{'rawcmd'} 2>/dev/null`; + if ($debug) { print "DEBUG: checking availability of $compressargs{'rawcmd'} on target...\n"; } + $avail{'targetcompress'} = `$targetssh $lscmd $compressargs{'rawcmd'} 2>/dev/null`; + if ($debug) { print "DEBUG: checking availability of $compressargs{'rawcmd'} on local machine...\n"; } + $avail{'localcompress'} = `$lscmd $compressargs{'rawcmd'} 2>/dev/null`; } my ($s,$t); @@ -451,14 +451,14 @@ sub checkcommands { if ($avail{'sourcecompress'} eq '') { - if ($args{'compress'}{'rawcmd'} ne '') { - print "WARN: $args{'compress'}{'rawcmd'} not available on source $s- sync will continue without compression.\n"; + if ($compressargs{'rawcmd'} ne '') { + print "WARN: $compressargs{'rawcmd'} not available on source $s- sync will continue without compression.\n"; } $avail{'compress'} = 0; } if ($avail{'targetcompress'} eq '') { - if ($args{'compress'}{'rawcmd'} ne '') { - print "WARN: $args{'compress'}{'rawcmd'} not available on target $t - sync will continue without compression.\n"; + if ($compressargs{'rawcmd'} ne '') { + print "WARN: $compressargs{'rawcmd'} not available on target $t - sync will continue without compression.\n"; } $avail{'compress'} = 0; } @@ -470,8 +470,8 @@ sub checkcommands { # corner case - if source AND target are BOTH remote, we have to check for local compress too if ($sourcehost ne '' && $targethost ne '' && $avail{'localcompress'} eq '') { - if ($args{'compress'}{'rawcmd'} ne '') { - print "WARN: $args{'compress'}{'rawcmd'} not available on local machine - sync will continue without compression.\n"; + if ($compressargs{'rawcmd'} ne '') { + print "WARN: $compressargs{'rawcmd'} not available on local machine - sync will continue without compression.\n"; } $avail{'compress'} = 0; } @@ -638,40 +638,40 @@ sub buildsynccmd { $synccmd .= " $recvcmd"; } elsif ($sourcehost eq '') { # local source, remote target. - #$synccmd = "$sendcmd | $pvcmd | $args{'compress'}{'cmd'} | $mbuffercmd | $sshcmd $targethost '$args{'compress'}{'decomcmd'} | $mbuffercmd | $recvcmd'"; + #$synccmd = "$sendcmd | $pvcmd | $compressargs{'cmd'} | $mbuffercmd | $sshcmd $targethost '$compressargs{'decomcmd'} | $mbuffercmd | $recvcmd'"; $synccmd = "$sendcmd |"; if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; } - if ($avail{'compress'}) { $synccmd .= " $args{'compress'}{'cmd'} |"; } + if ($avail{'compress'}) { $synccmd .= " $compressargs{'cmd'} |"; } if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; } $synccmd .= " $sshcmd $targethost '"; if ($avail{'targetmbuffer'}) { $synccmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; } - if ($avail{'compress'}) { $synccmd .= " $args{'compress'}{'decomcmd'} |"; } + if ($avail{'compress'}) { $synccmd .= " $compressargs{'decomcmd'} |"; } $synccmd .= " $recvcmd'"; } elsif ($targethost eq '') { # remote source, local target. - #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compress'}{'cmd'} | $mbuffercmd' | $args{'decompress'}{'cmd'} | $mbuffercmd | $pvcmd | $recvcmd"; + #$synccmd = "$sshcmd $sourcehost '$sendcmd | $compressargs{'cmd'} | $mbuffercmd' | $args{'decompress'}{'cmd'} | $mbuffercmd | $pvcmd | $recvcmd"; $synccmd = "$sshcmd $sourcehost '$sendcmd"; - if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; } + if ($avail{'compress'}) { $synccmd .= " | $compressargs{'compresscmd'}"; } if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } $synccmd .= "' | "; if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } - if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$compressargs{'decompresscmd'} | "; } if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } $synccmd .= "$recvcmd"; } else { #remote source, remote target... weird, but whatever, I'm not here to judge you. - #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compress'}{'cmd'} | $mbuffercmd' | $args{'compress'}{'decomcmd'} | $pvcmd | $args{'compress'}{'cmd'} | $mbuffercmd | $sshcmd $targethost '$args{'compress'}{'decomcmd'} | $mbuffercmd | $recvcmd'"; + #$synccmd = "$sshcmd $sourcehost '$sendcmd | $compressargs{'cmd'} | $mbuffercmd' | $compressargs{'decomcmd'} | $pvcmd | $compressargs{'cmd'} | $mbuffercmd | $sshcmd $targethost '$compressargs{'decomcmd'} | $mbuffercmd | $recvcmd'"; $synccmd = "$sshcmd $sourcehost '$sendcmd"; - if ($avail{'compress'}) { $synccmd .= " | $args{'compress'}{'cmd'}"; } + if ($avail{'compress'}) { $synccmd .= " | $compressargs{'cmd'}"; } if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } $synccmd .= "' | "; - if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'decomcmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$compressargs{'decomcmd'} | "; } if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } - if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'cmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$compressargs{'cmd'} | "; } if ($avail{'localmbuffer'}) { $synccmd .= "$mbuffercmd $mbufferoptions | "; } $synccmd .= "$sshcmd $targethost '"; if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } - if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'decomcmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$compressargs{'decomcmd'} | "; } $synccmd .= "$recvcmd'"; } return $synccmd; From 90e6a401da6eeaffa49223e9a6729095d564e2ee Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Wed, 8 Nov 2017 16:32:39 +0000 Subject: [PATCH 04/97] Missed some compression variable renames --- syncoid | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index 3f07990..7337f5b 100755 --- a/syncoid +++ b/syncoid @@ -651,11 +651,11 @@ sub buildsynccmd { # remote source, local target. #$synccmd = "$sshcmd $sourcehost '$sendcmd | $compressargs{'cmd'} | $mbuffercmd' | $args{'decompress'}{'cmd'} | $mbuffercmd | $pvcmd | $recvcmd"; $synccmd = "$sshcmd $sourcehost '$sendcmd"; - if ($avail{'compress'}) { $synccmd .= " | $compressargs{'compresscmd'}"; } + if ($avail{'compress'}) { $synccmd .= " | $compressargs{'cmd'}"; } if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } $synccmd .= "' | "; if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } - if ($avail{'compress'}) { $synccmd .= "$compressargs{'decompresscmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$compressargs{'decomcmd'} | "; } if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } $synccmd .= "$recvcmd"; } else { From c9adcdab1e7e2a7eaa04bdbc15a24881278cefdb Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 8 Nov 2017 20:25:07 +0100 Subject: [PATCH 05/97] hardcoded new defaults --- sanoid | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/sanoid b/sanoid index e8756db..6428cdf 100755 --- a/sanoid +++ b/sanoid @@ -553,6 +553,18 @@ sub getsnaps { #################################################################################### #################################################################################### +sub verify_option_existence { + my ($hash, $key, $default) = @_; + + if (! defined (%$hash{$key})) { + $hash->{$key} = $default; + } +} + +#################################################################################### +#################################################################################### +#################################################################################### + sub init { my ($conf_file, $default_conf_file) = @_; my %config; @@ -568,6 +580,12 @@ sub init { my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON"); my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF"); + # hardcoded defaults which may be missing from older default configuration file + verify_option_existence($defaults{'template_default'}, 'frequent_period', 15); + verify_option_existence($defaults{'template_default'}, 'frequently', 0); + verify_option_existence($defaults{'template_default'}, 'frequently_warn', 2000); + verify_option_existence($defaults{'template_default'}, 'frequently_crit', 8000); + foreach my $section (keys %ini) { # first up - die with honor if unknown parameters are set in any modules or templates by the user. From 8bd98f18008494e27c47c6d16cc1fbbaec9a33a6 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 8 Nov 2017 21:40:30 +0100 Subject: [PATCH 06/97] added more documentation for frequent snapshots --- README.md | 1 + sanoid.defaults.conf | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9c4e6ba..863b697 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ And its /etc/sanoid/sanoid.conf might look something like this: ############################# [template_production] + frequently = 0 hourly = 36 daily = 30 monthly = 3 diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index b5e4e63..e7c22a8 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -16,8 +16,16 @@ recursive = use_template = process_children_only = -# The period in minutes for frequent snapshots, -# should be in the range of 1-30 and divide an hour without remainder +# for snapshots shorter than one hour, the period duration must be defined +# in minutes. Because they are executed within a full hour, the selected +# value should divide 60 minutes without remainder so taken snapshots +# are apart in equal intervals. Values larger than 59 aren't practical +# as only one snapshot will be taken on each full hour in this case. +# examples: +# frequent_period = 15 -> four snapshot each hour 15 minutes apart +# frequent_period = 5 -> twelve snapshots each hour 5 minutes apart +# frequent_period = 45 -> two snapshots each hour with different time gaps +# between them: 45 minutes and 15 minutes in this case frequent_period = 15 # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately From bc2f8bd4e9b150294bad5c3150a682a3e857e3e0 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 8 Nov 2017 21:48:37 +0100 Subject: [PATCH 07/97] Revert "hardcoded new defaults" This reverts commit c9adcdab1e7e2a7eaa04bdbc15a24881278cefdb. --- sanoid | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/sanoid b/sanoid index 6428cdf..e8756db 100755 --- a/sanoid +++ b/sanoid @@ -553,18 +553,6 @@ sub getsnaps { #################################################################################### #################################################################################### -sub verify_option_existence { - my ($hash, $key, $default) = @_; - - if (! defined (%$hash{$key})) { - $hash->{$key} = $default; - } -} - -#################################################################################### -#################################################################################### -#################################################################################### - sub init { my ($conf_file, $default_conf_file) = @_; my %config; @@ -580,12 +568,6 @@ sub init { my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON"); my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF"); - # hardcoded defaults which may be missing from older default configuration file - verify_option_existence($defaults{'template_default'}, 'frequent_period', 15); - verify_option_existence($defaults{'template_default'}, 'frequently', 0); - verify_option_existence($defaults{'template_default'}, 'frequently_warn', 2000); - verify_option_existence($defaults{'template_default'}, 'frequently_crit', 8000); - foreach my $section (keys %ini) { # first up - die with honor if unknown parameters are set in any modules or templates by the user. From 293d83bdaa6155e765432a9a145a786d72afed9e Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 8 Nov 2017 22:08:02 +0100 Subject: [PATCH 08/97] versioning and compatibility check for default configuration file --- sanoid | 12 ++++++++++++ sanoid.defaults.conf | 2 ++ 2 files changed, 14 insertions(+) diff --git a/sanoid b/sanoid index e8756db..452f535 100755 --- a/sanoid +++ b/sanoid @@ -5,6 +5,7 @@ # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. $::VERSION = '1.4.17'; +my $MINIMUM_DEFAULTS_VERSION = 2; use strict; use warnings; @@ -568,6 +569,17 @@ sub init { my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON"); my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF"); + # check if default configuration file is up to date + my $defaults_version = 1; + if (defined $defaults{'version'}{'version'}) { + $defaults_version = $defaults{'version'}{'version'}; + delete $defaults{'version'}; + } + + if ($defaults_version < $MINIMUM_DEFAULTS_VERSION) { + die "FATAL: you're using sanoid.defaults.conf v$defaults_version, this version of sanoid requires a minimum sanoid.defaults.conf v$MINIMUM_DEFAULTS_VERSION"; + } + foreach my $section (keys %ini) { # first up - die with honor if unknown parameters are set in any modules or templates by the user. diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index e7c22a8..06fc714 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -5,6 +5,8 @@ # # # you have been warned. # ################################################################################### +[version] +version = 2 [template_default] From ac16b2128e6863a8af619d0bf9282bf92f6dfbc9 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 9 Nov 2017 17:44:04 +0100 Subject: [PATCH 09/97] disable monitoring of frequent snapshots --- sanoid.defaults.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 06fc714..a521401 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -77,8 +77,8 @@ yearly_min = 0 monitor = yes monitor_dont_warn = no monitor_dont_crit = no -frequently_warn = 2000 -frequently_crit = 8000 +frequently_warn = 0 +frequently_crit = 0 hourly_warn = 90 hourly_crit = 360 daily_warn = 28 From 4a3e93372c3502a7a4d9e2202d147ab00fd23559 Mon Sep 17 00:00:00 2001 From: Jason Lewis Date: Mon, 20 Nov 2017 15:16:43 +1100 Subject: [PATCH 10/97] check for emtpy lockfile --- sanoid | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index d6e58ce..889696f 100755 --- a/sanoid +++ b/sanoid @@ -930,13 +930,22 @@ sub checklock { # no lockfile return 1; } + # make sure lockfile contains something + if ( -z $lockfile) { + # zero size lockfile, something is wrong + die "ERROR: something is wrong! $lockfile is empty\n"; + } # lockfile exists. read pid and mutex from it. see if it's our pid. if not, see if # there's still a process running with that pid and with the same mutex. - open FH, "< $lockfile"; + open FH, "< $lockfile" or die "ERROR: unable to open $lockfile"; my @lock = ; close FH; + # if we didn't get exactly 2 items from the lock file there is a problem + if (scalar(@lock) != 2) { + die "ERROR: $lockfile is invalid.\n" + } my $lockmutex = pop(@lock); my $lockpid = pop(@lock); @@ -948,7 +957,6 @@ sub checklock { # we own the lockfile. no need to check any further. return 2; } - open PL, "$pscmd -p $lockpid -o args= |"; my @processlist = ; close PL; From 31da53140fda7c5bfc79d063e3c6da8d8f49300c Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 19:00:44 +0100 Subject: [PATCH 11/97] implemented a simple test which will take snapshots over a whole year and checks the resulting snapshot list --- tests/1_one_year/run.sh | 55 ++++++++++++++++++ tests/1_one_year/sanoid.conf | 10 ++++ tests/common/lib.sh | 106 +++++++++++++++++++++++++++++++++++ tests/run-tests.sh | 27 +++++++++ 4 files changed, 198 insertions(+) create mode 100755 tests/1_one_year/run.sh create mode 100644 tests/1_one_year/sanoid.conf create mode 100644 tests/common/lib.sh create mode 100755 tests/run-tests.sh diff --git a/tests/1_one_year/run.sh b/tests/1_one_year/run.sh new file mode 100755 index 0000000..7cec813 --- /dev/null +++ b/tests/1_one_year/run.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -x + +# this test will take hourly, daily and monthly snapshots +# for the whole year of 2017 in the timezone Europe/Vienna +# sanoid is run hourly and no snapshots are pruned + +. ../common/lib.sh + +POOL_NAME="sanoid-test-1" +POOL_TARGET="" # root +RESULT="/tmp/sanoid_test_result" +RESULT_CHECKSUM="aa15e5595b0ed959313289ecb70323dad9903328ac46e881da5c4b0f871dd7cf" + +# UTC timestamp of start and end +START="1483225200" +END="1514761199" + +# prepare +setup +checkEnvironment +disableTimeSync + +# set timezone +ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime + +timestamp=$START + +mkdir -p "${POOL_TARGET}" +truncate -s 5120M "${POOL_TARGET}"/zpool.img + +zpool create -f "${POOL_NAME}" "${POOL_TARGET}"/zpool.img + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +while [ $timestamp -le $END ]; do + date --utc --set @$timestamp; date; "${SANOID}" --cron --verbose + timestamp=$((timestamp+3600)) +done + +saveSnapshotList "${POOL_NAME}" "${RESULT}" + +# hourly daily monthly +verifySnapshotList "${RESULT}" 8759 366 12 "${RESULT_CHECKSUM}" + +# hourly count should be 8760 but one hour get's lost because of DST + +# daily count should be 365 but one additional daily is taken +# because the DST change leads to a day with 25 hours +# which will trigger an additional daily snapshot diff --git a/tests/1_one_year/sanoid.conf b/tests/1_one_year/sanoid.conf new file mode 100644 index 0000000..f5692f0 --- /dev/null +++ b/tests/1_one_year/sanoid.conf @@ -0,0 +1,10 @@ +[sanoid-test-1] + use_template = production + +[template_production] + hourly = 36 + daily = 30 + monthly = 3 + yearly = 0 + autosnap = yes + autoprune = no diff --git a/tests/common/lib.sh b/tests/common/lib.sh new file mode 100644 index 0000000..2c15e9b --- /dev/null +++ b/tests/common/lib.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +function setup { + export LANG=C + export LANGUAGE=C + export LC_ALL=C + + export SANOID="../../sanoid" + + # make sure that there is no cache file + rm -f /var/cache/sanoidsnapshots.txt + + # install needed sanoid configuration files + [ -f sanoid.conf ] && cp sanoid.conf /etc/sanoid/sanoid.conf + cp ../../sanoid.defaults.conf /etc/sanoid/sanoid.defaults.conf +} + +function checkEnvironment { + ASK=1 + + which systemd-detect-virt > /dev/null + if [ $? -eq 0 ]; then + systemd-detect-virt --vm > /dev/null + if [ $? -eq 0 ]; then + # we are in a vm + ASK=0 + fi + fi + + if [ $ASK -eq 1 ]; then + set +x + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "you should be running this test in a" + echo "dedicated vm, as it will mess with your system!" + echo "Are you sure you wan't to continue? (y)" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + set -x + + read -n 1 c + if [ "$c" != "y" ]; then + exit 1 + fi + fi +} + +function disableTimeSync { + # disable ntp sync + which timedatectl > /dev/null + if [ $? -eq 0 ]; then + timedatectl set-ntp 0 + fi +} + +function saveSnapshotList { + POOL_NAME="$1" + RESULT="$2" + + zfs list -t snapshot -o name -Hr "${POOL_NAME}" | sort > "${RESULT}" + + # clear the seconds for comparing + sed -i 's/\(autosnap_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9]:[0-9][0-9]:\)[0-9][0-9]_/\100_/g' "${RESULT}" +} + +function verifySnapshotList { + RESULT="$1" + HOURLY_COUNT=$2 + DAILY_COUNT=$3 + MONTHLY_COUNT=$4 + CHECKSUM="$5" + + failed=0 + message="" + + hourly_count=$(grep -c "autosnap_.*_hourly" < "${RESULT}") + daily_count=$(grep -c "autosnap_.*_daily" < "${RESULT}") + monthly_count=$(grep -c "autosnap_.*_monthly" < "${RESULT}") + + if [ "${hourly_count}" -ne "${HOURLY_COUNT}" ]; then + failed=1 + message="${message}hourly snapshot count is wrong: ${hourly_count}\n" + fi + + if [ "${daily_count}" -ne "${DAILY_COUNT}" ]; then + failed=1 + message="${message}daily snapshot count is wrong: ${daily_count}\n" + fi + + if [ "${monthly_count}" -ne "${MONTHLY_COUNT}" ]; then + failed=1 + message="${message}monthly snapshot count is wrong: ${monthly_count}\n" + fi + + checksum=$(sha256sum "${RESULT}" | cut -d' ' -f1) + if [ "${checksum}" != "${CHECKSUM}" ]; then + failed=1 + message="${message}result checksum mismatch\n" + fi + + if [ "${failed}" -eq 0 ]; then + exit 0 + fi + + echo "TEST FAILED:" >&2 + echo -n -e "${message}" >&2 + +} diff --git a/tests/run-tests.sh b/tests/run-tests.sh new file mode 100755 index 0000000..a8469e9 --- /dev/null +++ b/tests/run-tests.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# run's all the available tests + +for test in */; do + if [ ! -x "${test}/run.sh" ]; then + continue + fi + + testName="${test%/}" + + LOGFILE=/tmp/sanoid_test_run_"${testName}".log + + pushd . > /dev/null + + echo -n "Running test ${testName} ... " + cd "${test}" + echo | bash run.sh > "${LOGFILE}" 2>&1 + + if [ $? -eq 0 ]; then + echo "[PASS]" + else + echo "[FAILED] (see ${LOGFILE})" + fi + + popd > /dev/null +done From 9a6cdb85438eb730348a35e23fd4fdf7b41b60f3 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 20:15:29 +0100 Subject: [PATCH 12/97] handle DST (daylight saving times) properly for hourly and daily snapshots, fixes #155 --- sanoid | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index d6e58ce..030fea2 100755 --- a/sanoid +++ b/sanoid @@ -268,6 +268,19 @@ sub take_snapshots { my @newsnaps; + # get utc timestamp of the current day for DST check + my $daystartUtc = timelocal(0, 0, 0, $datestamp{'mday'}, ($datestamp{'mon'}-1), $datestamp{'year'}); + my ($isdst) = (localtime($daystartUtc))[8]; + my $dstOffset = 0; + + if ($isdst ne $datestamp{'isdst'}) { + # current dst is different then at the beginning og the day + if ($isdst) { + # DST ended in the current day + $dstOffset = 60*60; + } + } + if ($args{'verbose'}) { print "INFO: taking snapshots...\n"; } foreach my $section (keys %config) { if ($section =~ /^template/) { next; } @@ -291,6 +304,9 @@ sub take_snapshots { my @preferredtime; my $lastpreferred; + # to avoid duplicates with DST + my $dateSuffix = ""; + if ($type eq 'hourly') { push @preferredtime,0; # try to hit 0 seconds push @preferredtime,$config{$section}{'hourly_min'}; @@ -299,6 +315,13 @@ sub take_snapshots { push @preferredtime,($datestamp{'mon'}-1); # january is month 0 push @preferredtime,$datestamp{'year'}; $lastpreferred = timelocal(@preferredtime); + + if ($dstOffset ne 0) { + # timelocal doesn't take DST into account + $lastpreferred += $dstOffset; + # DST ended, avoid duplicates + $dateSuffix = "_y"; + } if ($lastpreferred > time()) { $lastpreferred -= 60*60; } # preferred time is later this hour - so look at last hour's } elsif ($type eq 'daily') { push @preferredtime,0; # try to hit 0 seconds @@ -308,7 +331,29 @@ sub take_snapshots { push @preferredtime,($datestamp{'mon'}-1); # january is month 0 push @preferredtime,$datestamp{'year'}; $lastpreferred = timelocal(@preferredtime); - if ($lastpreferred > time()) { $lastpreferred -= 60*60*24; } # preferred time is later today - so look at yesterday's + + # timelocal doesn't take DST into account + $lastpreferred += $dstOffset; + + # check if the planned time has different DST flag than the current + my ($isdst) = (localtime($lastpreferred))[8]; + if ($isdst ne $datestamp{'isdst'}) { + if (!$isdst) { + # correct DST difference + $lastpreferred -= 60*60; + } + } + + if ($lastpreferred > time()) { + $lastpreferred -= 60*60*24; + + if ($dstOffset ne 0) { + # because we are going back one day + # the DST difference has to be accounted + # for in reverse now + $lastpreferred -= 2*$dstOffset; + } + } # preferred time is later today - so look at yesterday's } elsif ($type eq 'monthly') { push @preferredtime,0; # try to hit 0 seconds push @preferredtime,$config{$section}{'monthly_min'}; @@ -336,7 +381,7 @@ sub take_snapshots { # update to most current possible datestamp %datestamp = get_date(); # print "we should have had a $type snapshot of $path $maxage seconds ago; most recent is $newestage seconds old.\n"; - push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}_$type"); + push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}_${dateSuffix}_$type"); } } } From c1f7cd4241cefcafc1951b1e005b4c0a566ac062 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 20:23:52 +0100 Subject: [PATCH 13/97] fixed snapshot suffix --- sanoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 030fea2..1c470c3 100755 --- a/sanoid +++ b/sanoid @@ -381,7 +381,7 @@ sub take_snapshots { # update to most current possible datestamp %datestamp = get_date(); # print "we should have had a $type snapshot of $path $maxage seconds ago; most recent is $newestage seconds old.\n"; - push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}_${dateSuffix}_$type"); + push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}${dateSuffix}_$type"); } } } From e61ccf1c9dcccb1e156ed684413e0c0a5671664a Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 20:24:54 +0100 Subject: [PATCH 14/97] added test for checking for correct DST handling behaviour --- tests/2_dst_handling/run.sh | 54 ++++++++++++++++++++++++++++++++ tests/2_dst_handling/sanoid.conf | 10 ++++++ 2 files changed, 64 insertions(+) create mode 100755 tests/2_dst_handling/run.sh create mode 100644 tests/2_dst_handling/sanoid.conf diff --git a/tests/2_dst_handling/run.sh b/tests/2_dst_handling/run.sh new file mode 100755 index 0000000..eba21ed --- /dev/null +++ b/tests/2_dst_handling/run.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -x + +# this test will check the behaviour arround a date where DST ends +# with hourly, daily and monthly snapshots checked in a 15 minute interval + +# Daylight saving time 2017 in Europe/Vienna began at 02:00 on Sunday, 26 March +# and ended at 03:00 on Sunday, 29 October. All times are in +# Central European Time. + +. ../common/lib.sh + +POOL_NAME="sanoid-test-2" +POOL_TARGET="" # root +RESULT="/tmp/sanoid_test_result" +RESULT_CHECKSUM="a916d9cd46f4b80f285d069f3497d02671bbb1bfd12b43ef93531cbdaf89d55c" + +# UTC timestamp of start and end +START="1509141600" +END="1509400800" + +# prepare +setup +checkEnvironment +disableTimeSync + +# set timezone +ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime + +timestamp=$START + +mkdir -p "${POOL_TARGET}" +truncate -s 512M "${POOL_TARGET}"/zpool2.img + +zpool create -f "${POOL_NAME}" "${POOL_TARGET}"/zpool2.img + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +while [ $timestamp -le $END ]; do + date --utc --set @$timestamp; date; "${SANOID}" --cron --verbose + timestamp=$((timestamp+900)) +done + +saveSnapshotList "${POOL_NAME}" "${RESULT}" + +# hourly daily monthly +verifySnapshotList "${RESULT}" 73 3 1 "${RESULT_CHECKSUM}" + +# one more hour because of DST diff --git a/tests/2_dst_handling/sanoid.conf b/tests/2_dst_handling/sanoid.conf new file mode 100644 index 0000000..7ded3f8 --- /dev/null +++ b/tests/2_dst_handling/sanoid.conf @@ -0,0 +1,10 @@ +[sanoid-test-2] + use_template = production + +[template_production] + hourly = 36 + daily = 30 + monthly = 3 + yearly = 0 + autosnap = yes + autoprune = no From 8d4484a2d1641789c1e667c03e69773e96cde6ea Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 23:24:34 +0100 Subject: [PATCH 15/97] exit with error code upon failure --- debian/changelog | 4 ++++ tests/common/lib.sh | 1 + 2 files changed, 5 insertions(+) diff --git a/debian/changelog b/debian/changelog index ab530b0..beb6584 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,7 @@ +sanoid (1.4.17-SNAPSHOT) unstable; urgency=medium + +-- Jim Salter Wed, 9 Aug 2017 12:28:49 -0400 + sanoid (1.4.16) unstable; urgency=medium * merged @hrast01's extended fix to support -o option1=val,option2=val passthrough to SSH. merged @JakobR's diff --git a/tests/common/lib.sh b/tests/common/lib.sh index 2c15e9b..78f128b 100644 --- a/tests/common/lib.sh +++ b/tests/common/lib.sh @@ -103,4 +103,5 @@ function verifySnapshotList { echo "TEST FAILED:" >&2 echo -n -e "${message}" >&2 + exit 1 } From 53894b2855016e6398609995ee383c173fc2ff99 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 23:36:13 +0100 Subject: [PATCH 16/97] indentation fix --- sanoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 1c470c3..12d746d 100755 --- a/sanoid +++ b/sanoid @@ -317,7 +317,7 @@ sub take_snapshots { $lastpreferred = timelocal(@preferredtime); if ($dstOffset ne 0) { - # timelocal doesn't take DST into account + # timelocal doesn't take DST into account $lastpreferred += $dstOffset; # DST ended, avoid duplicates $dateSuffix = "_y"; From ceb1397ef084941b92bc2348d85b39020cac49a7 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 23:40:23 +0100 Subject: [PATCH 17/97] Revert "exit with error code upon failure" This reverts commit 8d4484a2d1641789c1e667c03e69773e96cde6ea. --- debian/changelog | 4 ---- tests/common/lib.sh | 1 - 2 files changed, 5 deletions(-) diff --git a/debian/changelog b/debian/changelog index beb6584..ab530b0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,3 @@ -sanoid (1.4.17-SNAPSHOT) unstable; urgency=medium - --- Jim Salter Wed, 9 Aug 2017 12:28:49 -0400 - sanoid (1.4.16) unstable; urgency=medium * merged @hrast01's extended fix to support -o option1=val,option2=val passthrough to SSH. merged @JakobR's diff --git a/tests/common/lib.sh b/tests/common/lib.sh index 78f128b..2c15e9b 100644 --- a/tests/common/lib.sh +++ b/tests/common/lib.sh @@ -103,5 +103,4 @@ function verifySnapshotList { echo "TEST FAILED:" >&2 echo -n -e "${message}" >&2 - exit 1 } From 371f8ff318ad749bcb9374b8acc723ac77da841d Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 6 Dec 2017 23:41:15 +0100 Subject: [PATCH 18/97] exit with error code upon failure --- tests/common/lib.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/common/lib.sh b/tests/common/lib.sh index 2c15e9b..78f128b 100644 --- a/tests/common/lib.sh +++ b/tests/common/lib.sh @@ -103,4 +103,5 @@ function verifySnapshotList { echo "TEST FAILED:" >&2 echo -n -e "${message}" >&2 + exit 1 } From e260f9095f7ac6d6666c22e9b8b312a0ac7ff6e6 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 10 Dec 2017 16:05:08 +0100 Subject: [PATCH 19/97] escape filesystem names as needed to avoid interpreting special characters like whitespace and stop interpreting metacharacters in fs names for some regular expressions, fixes #40 --- syncoid | 191 +++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 135 insertions(+), 56 deletions(-) diff --git a/syncoid b/syncoid index d927cba..3d58863 100755 --- a/syncoid +++ b/syncoid @@ -97,7 +97,7 @@ if (! $args{'recursive'}) { if ($debug) { print "DEBUG: recursive sync of $sourcefs.\n"; } my @datasets = getchilddatasets($sourcehost, $sourcefs, $sourceisroot); foreach my $dataset(@datasets) { - $dataset =~ s/$sourcefs//; + $dataset =~ s/\Q$sourcefs\E//; chomp $dataset; my $childsourcefs = $sourcefs . $dataset; my $childtargetfs = $targetfs . $dataset; @@ -126,11 +126,16 @@ exit 0; sub getchilddatasets { my ($rhost,$fs,$isroot,%snaps) = @_; my $mysudocmd; + my $fsescaped = escapeshellparam($fs); if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } - my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name -t filesystem,volume -Hr $fs |"; + my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name -t filesystem,volume -Hr $fsescaped |"; if ($debug) { print "DEBUG: getting list of child datasets on $fs using $getchildrencmd...\n"; } open FH, $getchildrencmd; my @children = ; @@ -143,6 +148,9 @@ sub syncdataset { my ($sourcehost, $sourcefs, $targethost, $targetfs) = @_; + my $sourcefsescaped = escapeshellparam($sourcefs); + my $targetfsescaped = escapeshellparam($targetfs); + if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; } # make sure target is not currently in receive. @@ -209,8 +217,8 @@ sub syncdataset { # if --no-stream is specified, our full needs to be the newest snapshot, not the oldest. if (defined $args{'no-stream'}) { $oldestsnap = getnewestsnapshot(\%snaps); } - my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefs\@$oldestsnap"; - my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfs"; + my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnap"; + my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); my $disp_pvsize = readablebytes($pvsize); @@ -245,7 +253,7 @@ sub syncdataset { # $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); - $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefs\@$oldestsnap $sourcefs\@$newsyncsnap"; + $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnap $sourcefsescaped\@$newsyncsnap"; $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap","$sourcefs\@$newsyncsnap",$sourceisroot); $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -305,15 +313,15 @@ sub syncdataset { # rollback target to matchingsnap if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; } if ($targethost ne '') { - if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap\n"; } - system ("$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap"); + if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap\n"; } + system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap")); } else { - if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap\n"; } - system ("$targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap"); + if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap\n"; } + system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap"); } - my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefs\@$matchingsnap $sourcefs\@$newsyncsnap"; - my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfs"; + my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnap $sourcefsescaped\@$newsyncsnap"; + my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -590,7 +598,7 @@ sub iszfsbusy { foreach my $process (@processes) { # if ($debug) { print "DEBUG: checking process $process...\n"; } - if ($process =~ /zfs *(receive|recv).*$fs/) { + if ($process =~ /zfs *(receive|recv).*\Q$fs\E/) { # there's already a zfs receive process for our target filesystem - return true if ($debug) { print "DEBUG: process $process matches target $fs!\n"; } return 1; @@ -603,24 +611,40 @@ sub iszfsbusy { sub setzfsvalue { my ($rhost,$fs,$isroot,$property,$value) = @_; - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + + my $fsescaped = escapeshellparam($fs); + + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } + if ($debug) { print "DEBUG: setting $property to $value on $fs...\n"; } my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } - if ($debug) { print "$rhost $mysudocmd $zfscmd set $property=$value $fs\n"; } - system("$rhost $mysudocmd $zfscmd set $property=$value $fs") == 0 - or warn "WARNING: $rhost $mysudocmd $zfscmd set $property=$value $fs died: $?, proceeding anyway.\n"; + if ($debug) { print "$rhost $mysudocmd $zfscmd set $property=$value $fsescaped\n"; } + system("$rhost $mysudocmd $zfscmd set $property=$value $fsescaped") == 0 + or warn "WARNING: $rhost $mysudocmd $zfscmd set $property=$value $fsescaped died: $?, proceeding anyway.\n"; return; } sub getzfsvalue { my ($rhost,$fs,$isroot,$property) = @_; - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + + my $fsescaped = escapeshellparam($fs); + + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } + if ($debug) { print "DEBUG: getting current value of $property on $fs...\n"; } my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } - if ($debug) { print "$rhost $mysudocmd $zfscmd get -H $property $fs\n"; } - open FH, "$rhost $mysudocmd $zfscmd get -H $property $fs |"; + if ($debug) { print "$rhost $mysudocmd $zfscmd get -H $property $fsescaped\n"; } + open FH, "$rhost $mysudocmd $zfscmd get -H $property $fsescaped |"; my $value = ; close FH; my @values = split(/\s/,$value); @@ -706,17 +730,24 @@ sub buildsynccmd { if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; } if ($avail{'compress'}) { $synccmd .= " $args{'compresscmd'} |"; } if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; } - $synccmd .= " $sshcmd $targethost '"; - if ($avail{'targetmbuffer'}) { $synccmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; } - if ($avail{'compress'}) { $synccmd .= " $args{'decompresscmd'} |"; } - $synccmd .= " $recvcmd'"; + $synccmd .= " $sshcmd $targethost "; + + my $remotecmd = ""; + if ($avail{'targetmbuffer'}) { $remotecmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; } + if ($avail{'compress'}) { $remotecmd .= " $args{'decompresscmd'} |"; } + $remotecmd .= " $recvcmd"; + + $synccmd .= escapeshellparam($remotecmd); } elsif ($targethost eq '') { # remote source, local target. #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $mbuffercmd | $pvcmd | $recvcmd"; - $synccmd = "$sshcmd $sourcehost '$sendcmd"; - if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; } - if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } - $synccmd .= "' | "; + + my $remotecmd = $sendcmd; + if ($avail{'compress'}) { $remotecmd .= " | $args{'compresscmd'}"; } + if ($avail{'sourcembuffer'}) { $remotecmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } + + $synccmd = "$sshcmd $sourcehost " . escapeshellparam($remotecmd); + $synccmd .= " | "; if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } @@ -724,25 +755,37 @@ sub buildsynccmd { } else { #remote source, remote target... weird, but whatever, I'm not here to judge you. #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'"; - $synccmd = "$sshcmd $sourcehost '$sendcmd"; - if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; } - if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } - $synccmd .= "' | "; + + my $remotecmd = $sendcmd; + if ($avail{'compress'}) { $remotecmd .= " | $args{'compresscmd'}"; } + if ($avail{'sourcembuffer'}) { $remotecmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } + + $synccmd = "$sshcmd $sourcehost " . escapeshellparam($remotecmd); + $synccmd .= " | "; + if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } if ($avail{'compress'}) { $synccmd .= "$args{'compresscmd'} | "; } if ($avail{'localmbuffer'}) { $synccmd .= "$mbuffercmd $mbufferoptions | "; } - $synccmd .= "$sshcmd $targethost '"; - if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } - if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } - $synccmd .= "$recvcmd'"; + $synccmd .= "$sshcmd $targethost "; + + $remotecmd = ""; + if ($avail{'targetmbuffer'}) { $remotecmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; } + if ($avail{'compress'}) { $remotecmd .= " $args{'decompresscmd'} |"; } + $remotecmd .= " $recvcmd"; + + $synccmd .= escapeshellparam($remotecmd); } return $synccmd; } sub pruneoldsyncsnaps { my ($rhost,$fs,$newsyncsnap,$isroot,@snaps) = @_; + + my $fsescaped = escapeshellparam($fs); + if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + my $hostid = hostname(); my $mysudocmd; @@ -752,7 +795,7 @@ sub pruneoldsyncsnaps { # only prune snaps beginning with syncoid and our own hostname foreach my $snap(@snaps) { - if ($snap =~ /^syncoid_$hostid/) { + if ($snap =~ /^syncoid_\Q$hostid\E/) { # no matter what, we categorically refuse to # prune the new sync snap we created for this run if ($snap ne $newsyncsnap) { @@ -768,12 +811,14 @@ sub pruneoldsyncsnaps { my $prunecmd; foreach my $snap(@prunesnaps) { $counter ++; - $prunecmd .= "$mysudocmd $zfscmd destroy $fs\@$snap; "; + $prunecmd .= "$mysudocmd $zfscmd destroy $fsescaped\@$snap; "; if ($counter > $maxsnapspercmd) { $prunecmd =~ s/\; $//; - if ($rhost ne '') { $prunecmd = '"' . $prunecmd . '"'; } if ($debug) { print "DEBUG: pruning up to $maxsnapspercmd obsolete sync snapshots...\n"; } if ($debug) { print "DEBUG: $rhost $prunecmd\n"; } + if ($rhost ne '') { + $prunecmd = escapeshellparam($prunecmd); + } system("$rhost $prunecmd") == 0 or warn "CRITICAL ERROR: $rhost $prunecmd failed: $?"; $prunecmd = ''; @@ -784,9 +829,11 @@ sub pruneoldsyncsnaps { # the loop, commit 'em now if ($counter) { $prunecmd =~ s/\; $//; - if ($rhost ne '') { $prunecmd = '"' . $prunecmd . '"'; } if ($debug) { print "DEBUG: pruning up to $maxsnapspercmd obsolete sync snapshots...\n"; } if ($debug) { print "DEBUG: $rhost $prunecmd\n"; } + if ($rhost ne '') { + $prunecmd = escapeshellparam($prunecmd); + } system("$rhost $prunecmd") == 0 or warn "WARNING: $rhost $prunecmd failed: $?"; } @@ -824,13 +871,18 @@ sub getmatchingsnapshot { sub newsyncsnap { my ($rhost,$fs,$isroot) = @_; - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + my $fsescaped = escapeshellparam($fs); + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } my $hostid = hostname(); my %date = getdate(); my $snapname = "syncoid\_$hostid\_$date{'stamp'}"; - my $snapcmd = "$rhost $mysudocmd $zfscmd snapshot $fs\@$snapname\n"; + my $snapcmd = "$rhost $mysudocmd $zfscmd snapshot $fsescaped\@$snapname\n"; system($snapcmd) == 0 or die "CRITICAL ERROR: $snapcmd failed: $?"; return $snapname; @@ -838,16 +890,21 @@ sub newsyncsnap { sub targetexists { my ($rhost,$fs,$isroot) = @_; - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + my $fsescaped = escapeshellparam($fs); + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } - my $checktargetcmd = "$rhost $mysudocmd $zfscmd get -H name $fs"; + my $checktargetcmd = "$rhost $mysudocmd $zfscmd get -H name $fsescaped"; if ($debug) { print "DEBUG: checking to see if target filesystem exists using \"$checktargetcmd 2>&1 |\"...\n"; } open FH, "$checktargetcmd 2>&1 |"; my $targetexists = ; close FH; my $exit = $?; - $targetexists = ( $targetexists =~ /^$fs/ && $exit == 0 ); + $targetexists = ( $targetexists =~ /^\Q$fs\E/ && $exit == 0 ); return $targetexists; } @@ -888,11 +945,16 @@ sub dumphash() { sub getsnaps() { my ($type,$rhost,$fs,$isroot,%snaps) = @_; my $mysudocmd; + my $fsescaped = escapeshellparam($fs); if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } - my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t snapshot guid,creation $fs |"; + my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t snapshot guid,creation $fsescaped |"; if ($debug) { print "DEBUG: getting list of snapshots on $fs using $getsnapcmd...\n"; } open FH, $getsnapcmd; my @rawsnaps = ; @@ -903,24 +965,24 @@ sub getsnaps() { foreach my $line (@rawsnaps) { # only import snap guids from the specified filesystem - if ($line =~ /$fs\@.*guid/) { + if ($line =~ /\Q$fs\E\@.*guid/) { chomp $line; my $guid = $line; $guid =~ s/^.*\sguid\s*(\d*).*/$1/; my $snap = $line; - $snap =~ s/^\S*\@(\S*)\s*guid.*$/$1/; + $snap =~ s/^.*\@(\S*)\s*guid.*$/$1/; $snaps{$type}{$snap}{'guid'}=$guid; } } foreach my $line (@rawsnaps) { # only import snap creations from the specified filesystem - if ($line =~ /$fs\@.*creation/) { + if ($line =~ /\Q$fs\E\@.*creation/) { chomp $line; my $creation = $line; $creation =~ s/^.*\screation\s*(\d*).*/$1/; my $snap = $line; - $snap =~ s/^\S*\@(\S*)\s*creation.*$/$1/; + $snap =~ s/^.*\@(\S*)\s*creation.*$/$1/; $snaps{$type}{$snap}{'creation'}=$creation; } } @@ -932,21 +994,30 @@ sub getsnaps() { sub getsendsize { my ($sourcehost,$snap1,$snap2,$isroot) = @_; + my $snap1escaped = escapeshellparam($snap1); + my $snap2escaped = escapeshellparam($snap2); + my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } + my $sourcessh; + if ($sourcehost ne '') { + $sourcessh = "$sshcmd $sourcehost"; + $snap1escaped = escapeshellparam($snap1escaped); + $snap2escaped = escapeshellparam($snap2escaped); + } else { + $sourcessh = ''; + } + my $snaps; if ($snap2) { # if we got a $snap2 argument, we want an incremental send estimate from $snap1 to $snap2. - $snaps = "$args{'streamarg'} $snap1 $snap2"; + $snaps = "$args{'streamarg'} $snap1escaped $snap2escaped"; } else { # if we didn't get a $snap2 arg, we want a full send estimate for $snap1. - $snaps = "$snap1"; + $snaps = "$snap1escaped"; } - my $sourcessh; - if ($sourcehost ne '') { $sourcessh = "$sshcmd $sourcehost"; } else { $sourcessh = ''; } - my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send -nP $snaps"; if ($debug) { print "DEBUG: getting estimated transfer size from source $sourcehost using \"$getsendsizecmd 2>&1 |\"...\n"; } @@ -987,3 +1058,11 @@ sub getdate { $date{'stamp'} = "$date{'year'}-$date{'mon'}-$date{'mday'}:$date{'hour'}:$date{'min'}:$date{'sec'}"; return %date; } + +sub escapeshellparam { + my ($par) = @_; + # "escape" all single quotes + $par =~ s/'/'"'"'/g; + # single-quote entire string + return "'$par'"; + } From adca6230b735b973d34988068c0a3643aff9a760 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 10 Dec 2017 16:10:49 +0100 Subject: [PATCH 20/97] missed one regular expression --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 3d58863..ed5cce9 100755 --- a/syncoid +++ b/syncoid @@ -919,7 +919,7 @@ sub getssh { if ($fs =~ /\@/) { $rhost = $fs; $fs =~ s/^\S*\@\S*://; - $rhost =~ s/:$fs$//; + $rhost =~ s/:\Q$fs\E$//; my $remoteuser = $rhost; $remoteuser =~ s/\@.*$//; if ($remoteuser eq 'root') { $isroot = 1; } else { $isroot = 0; } From 3027831095d51e4e53f28bb82e337d8c11606cac Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 10 Dec 2017 16:32:49 +0100 Subject: [PATCH 21/97] escape all remaining snapshot names which could have special characters --- syncoid | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/syncoid b/syncoid index ed5cce9..797b5c0 100755 --- a/syncoid +++ b/syncoid @@ -216,8 +216,9 @@ sub syncdataset { # if --no-stream is specified, our full needs to be the newest snapshot, not the oldest. if (defined $args{'no-stream'}) { $oldestsnap = getnewestsnapshot(\%snaps); } + my $oldestsnapescaped = escapeshellparam($oldestsnap); - my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnap"; + my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped"; my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); @@ -253,7 +254,7 @@ sub syncdataset { # $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); - $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnap $sourcefsescaped\@$newsyncsnap"; + $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnapescaped $sourcefsescaped\@$newsyncsnap"; $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap","$sourcefs\@$newsyncsnap",$sourceisroot); $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -310,17 +311,18 @@ sub syncdataset { # barf some text but don't touch the filesystem if (!$quiet) { print "INFO: no snapshots on source newer than $newsyncsnap on target. Nothing to do, not syncing.\n"; } } else { + my $matchingsnapescaped = escapeshellparam($matchingsnap); # rollback target to matchingsnap if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; } if ($targethost ne '') { - if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap\n"; } - system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap")); + if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped\n"; } + system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped")); } else { - if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap\n"; } - system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap"); + if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped\n"; } + system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped"); } - my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnap $sourcefsescaped\@$newsyncsnap"; + my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnap"; my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); From 85cc99c9e6b4c507a1559903ecc7718685b0a77c Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 10 Dec 2017 16:56:56 +0100 Subject: [PATCH 22/97] accidentally removed helptext --- syncoid | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/syncoid b/syncoid index 86810d4..8825dcb 100755 --- a/syncoid +++ b/syncoid @@ -1005,3 +1005,41 @@ sub escapeshellparam { # single-quote entire string return "'$par'"; } + +__END__ + +=head1 NAME + +syncoid - ZFS snapshot replication tool + +=head1 SYNOPSIS + + syncoid [options]... SOURCE TARGET + or syncoid [options]... SOURCE [USER@]HOST:TARGET + or syncoid [options]... [USER@]HOST:SOURCE [TARGET] + or syncoid [options]... [USER@]HOST:SOURCE [USER@]HOST:TARGET + + SOURCE Source ZFS dataset. Can be either local or remote + TARGET Target ZFS dataset. Can be either local or remote + +Options: + + --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none + --recursive|r Also transfers child datasets + --source-bwlimit= Bandwidth limit on the source transfer + --target-bwlimit= Bandwidth limit on the target transfer + --no-stream Replicates using newest snapshot instead of intermediates + --no-sync-snap Does not create new snapshot, only transfers existing + + --sshkey=FILE Specifies a ssh public key to use to connect + --sshport=PORT Connects to remote on a particular port + --sshcipher|c=CIPHER Passes CIPHER to ssh to use a particular cipher set + --sshoption|o=OPTION Passes OPTION to ssh for remote usage. Can be specified multiple times + + --help Prints this helptext + --verbose Prints the version number + --debug Prints out a lot of additional information during a syncoid run + --monitor-version Currently does nothing + --quiet Suppresses non-error output + --dumpsnaps Dumps a list of snapshots during the run + --no-command-checks Do not check command existence before attempting transfer. Not recommended From 742db32e686520dc22ea11a8f831ed15618fdf9c Mon Sep 17 00:00:00 2001 From: Martin van Wingerden Date: Fri, 29 Dec 2017 20:02:01 +0100 Subject: [PATCH 23/97] Made two INFO prints mutable Added a quiet check for two non-controllable INFO prints Fixes: #182 Signed-off-by: Martin van Wingerden --- sanoid | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index d6e58ce..b995924 100755 --- a/sanoid +++ b/sanoid @@ -235,7 +235,7 @@ sub prune_snapshots { foreach my $snap( @prunesnaps ){ if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; } if (iszfsbusy($path)) { - print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; + if ($args{'verbose'}) { print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; } } else { if (! $args{'readonly'}) { system($zfs, "destroy",$snap) == 0 or warn "could not remove $snap : $?"; } } @@ -244,7 +244,7 @@ sub prune_snapshots { $forcecacheupdate = 1; %snaps = getsnaps(%config,$cacheTTL,$forcecacheupdate); } else { - print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n"; + if ($args{'verbose'}) { print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n"; } } } } From 8a2a673c58ddbc1b222ec65a75c4db77b9e8b4e4 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Sat, 6 Jan 2018 11:01:44 +0000 Subject: [PATCH 24/97] fixed loud 'NEWEST SNAPSHOT' message --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 7337f5b..d039d15 100755 --- a/syncoid +++ b/syncoid @@ -597,7 +597,7 @@ sub getnewestsnapshot { my $snaps = shift; foreach my $snap ( sort { $snaps{'source'}{$b}{'creation'}<=>$snaps{'source'}{$a}{'creation'} } keys %{ $snaps{'source'} }) { # return on first snap found - it's the newest - print "NEWEST SNAPSHOT: $snap\n"; + if (!$quiet) { print "NEWEST SNAPSHOT: $snap\n"; } return $snap; } # must not have had any snapshots on source - looks like we'd better create one! From e902df1ef2347aa08aeb20478a2b988695dd5d20 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Sat, 6 Jan 2018 11:03:02 +0000 Subject: [PATCH 25/97] also made warnings quiet --- syncoid | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/syncoid b/syncoid index d039d15..aec99cd 100755 --- a/syncoid +++ b/syncoid @@ -452,13 +452,13 @@ sub checkcommands { if ($avail{'sourcecompress'} eq '') { if ($compressargs{'rawcmd'} ne '') { - print "WARN: $compressargs{'rawcmd'} not available on source $s- sync will continue without compression.\n"; + if (!$quiet) { print "WARN: $compressargs{'rawcmd'} not available on source $s- sync will continue without compression.\n"; } } $avail{'compress'} = 0; } if ($avail{'targetcompress'} eq '') { if ($compressargs{'rawcmd'} ne '') { - print "WARN: $compressargs{'rawcmd'} not available on target $t - sync will continue without compression.\n"; + if (!$quiet) { print "WARN: $compressargs{'rawcmd'} not available on target $t - sync will continue without compression.\n"; } } $avail{'compress'} = 0; } @@ -471,7 +471,7 @@ sub checkcommands { # corner case - if source AND target are BOTH remote, we have to check for local compress too if ($sourcehost ne '' && $targethost ne '' && $avail{'localcompress'} eq '') { if ($compressargs{'rawcmd'} ne '') { - print "WARN: $compressargs{'rawcmd'} not available on local machine - sync will continue without compression.\n"; + if (!$quiet) { print "WARN: $compressargs{'rawcmd'} not available on local machine - sync will continue without compression.\n"; } } $avail{'compress'} = 0; } @@ -479,7 +479,7 @@ sub checkcommands { if ($debug) { print "DEBUG: checking availability of $mbuffercmd on source...\n"; } $avail{'sourcembuffer'} = `$sourcessh $lscmd $mbuffercmd 2>/dev/null`; if ($avail{'sourcembuffer'} eq '') { - print "WARN: $mbuffercmd not available on source $s - sync will continue without source buffering.\n"; + if (!$quiet) { print "WARN: $mbuffercmd not available on source $s - sync will continue without source buffering.\n"; } $avail{'sourcembuffer'} = 0; } else { $avail{'sourcembuffer'} = 1; @@ -488,7 +488,7 @@ sub checkcommands { if ($debug) { print "DEBUG: checking availability of $mbuffercmd on target...\n"; } $avail{'targetmbuffer'} = `$targetssh $lscmd $mbuffercmd 2>/dev/null`; if ($avail{'targetmbuffer'} eq '') { - print "WARN: $mbuffercmd not available on target $t - sync will continue without target buffering.\n"; + if (!$quiet) { print "WARN: $mbuffercmd not available on target $t - sync will continue without target buffering.\n"; } $avail{'targetmbuffer'} = 0; } else { $avail{'targetmbuffer'} = 1; @@ -500,14 +500,14 @@ sub checkcommands { $avail{'localmbuffer'} = `$lscmd $mbuffercmd 2>/dev/null`; if ($avail{'localmbuffer'} eq '') { $avail{'localmbuffer'} = 0; - print "WARN: $mbuffercmd not available on local machine - sync will continue without local buffering.\n"; + if (!$quiet) { print "WARN: $mbuffercmd not available on local machine - sync will continue without local buffering.\n"; } } } if ($debug) { print "DEBUG: checking availability of $pvcmd on local machine...\n"; } $avail{'localpv'} = `$lscmd $pvcmd 2>/dev/null`; if ($avail{'localpv'} eq '') { - print "WARN: $pvcmd not available on local machine - sync will continue without progress bar.\n"; + if (!$quiet) { print "WARN: $pvcmd not available on local machine - sync will continue without progress bar.\n"; } $avail{'localpv'} = 0; } else { $avail{'localpv'} = 1; From d5f4b5abba08a597108538cadbd09961afa80363 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 13 Feb 2018 18:47:55 +0100 Subject: [PATCH 26/97] support resumable zfs send/receive --- syncoid | 127 +++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 98 insertions(+), 29 deletions(-) diff --git a/syncoid b/syncoid index 8825dcb..8a31f37 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "resume") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -161,35 +161,55 @@ sub syncdataset { # does the target filesystem exist yet? my $targetexists = targetexists($targethost,$targetfs,$targetisroot); - # build hashes of the snaps on the source and target filesystems. + my $receiveextraargs = ""; + my $receivetoken; + if (defined $args{'resume'}) { + # save state of interrupted receive stream + $receiveextraargs = "-s"; - %snaps = getsnaps('source',$sourcehost,$sourcefs,$sourceisroot); + if ($targetexists) { + # check remote dataset for receive resume token (interrupted receive) + $receivetoken = getreceivetoken($targethost,$targetfs,$targetisroot); - if ($targetexists) { - my %targetsnaps = getsnaps('target',$targethost,$targetfs,$targetisroot); - my %sourcesnaps = %snaps; - %snaps = (%sourcesnaps, %targetsnaps); - } - - if (defined $args{'dumpsnaps'}) { - print "merged snapshot list of $targetfs: \n"; - dumphash(\%snaps); - print "\n\n\n"; - } - - # create a new syncoid snapshot on the source filesystem. - my $newsyncsnap; - if (!defined $args{'no-sync-snap'}) { - $newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot); - } else { - # we don't want sync snapshots created, so use the newest snapshot we can find. - $newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot); - if ($newsyncsnap eq 0) { - warn "CRITICAL: no snapshots exist on source $sourcefs, and you asked for --no-sync-snap.\n"; - return 0; + if ($debug && defined($receivetoken)) { + print "DEBUG: got receive resume token: $receivetoken: \n"; + } } } + my $newsyncsnap; + + # skip snapshot checking/creation in case of resumed receive + if (!defined($receivetoken)) { + # build hashes of the snaps on the source and target filesystems. + + %snaps = getsnaps('source',$sourcehost,$sourcefs,$sourceisroot); + + if ($targetexists) { + my %targetsnaps = getsnaps('target',$targethost,$targetfs,$targetisroot); + my %sourcesnaps = %snaps; + %snaps = (%sourcesnaps, %targetsnaps); + } + + if (defined $args{'dumpsnaps'}) { + print "merged snapshot list of $targetfs: \n"; + dumphash(\%snaps); + print "\n\n\n"; + } + + if (!defined $args{'no-sync-snap'}) { + # create a new syncoid snapshot on the source filesystem. + $newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot); + } else { + # we don't want sync snapshots created, so use the newest snapshot we can find. + $newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot); + if ($newsyncsnap eq 0) { + warn "CRITICAL: no snapshots exist on source $sourcefs, and you asked for --no-sync-snap.\n"; + return 0; + } + } + } + # there is currently (2014-09-01) a bug in ZFS on Linux # that causes readonly to always show on if it's EVER # been turned on... even when it's off... unless and @@ -222,7 +242,7 @@ sub syncdataset { my $oldestsnapescaped = escapeshellparam($oldestsnap); my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); my $disp_pvsize = readablebytes($pvsize); @@ -286,6 +306,27 @@ sub syncdataset { # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly); } } else { + # resume interrupted receive if there is a valid resume $token + # and because this will ony resume the receive to the next + # snapshot, do a normal sync after that + if (defined($receivetoken)) { + my $sendcmd = "$sourcesudocmd $zfscmd send -t $receivetoken"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $pvsize = getsendsize($sourcehost,"","",$sourceisroot,$receivetoken); + my $disp_pvsize = readablebytes($pvsize); + if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } + my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + + if (!$quiet) { print "Resuming interrupted zfs send/receive from $sourcefs to $targetfs (~ $disp_pvsize remaining):\n"; } + if ($debug) { print "DEBUG: $synccmd\n"; } + system("$synccmd") == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; + + # a resumed transfer will only be done to the next snapshot, + # so do an normal sync cycle + return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs); + } + # find most recent matching snapshot and do an -I # to the new snapshot @@ -326,7 +367,7 @@ sub syncdataset { } my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnap"; - my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -931,7 +972,7 @@ sub getsnaps() { sub getsendsize { - my ($sourcehost,$snap1,$snap2,$isroot) = @_; + my ($sourcehost,$snap1,$snap2,$isroot,$receivetoken) = @_; my $snap1escaped = escapeshellparam($snap1); my $snap2escaped = escapeshellparam($snap2); @@ -957,6 +998,12 @@ sub getsendsize { $snaps = "$snap1escaped"; } + # in case of a resumed receive, get the remaining + # size based on the resume token + if (defined($receivetoken)) { + $snaps = "-t $receivetoken"; + } + my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send -nP $snaps"; if ($debug) { print "DEBUG: getting estimated transfer size from source $sourcehost using \"$getsendsizecmd 2>&1 |\"...\n"; } @@ -969,7 +1016,13 @@ sub getsendsize { # size of proposed xfer in bytes, but we need to remove # human-readable crap from it my $sendsize = pop(@rawsize); - $sendsize =~ s/^size\s*//; + # the output format is different in case of + # a resumed receive + if (defined($receivetoken)) { + $sendsize =~ s/.*\s([0-9]+)$/$1/; + } else { + $sendsize =~ s/^size\s*//; + } chomp $sendsize; # to avoid confusion with a zero size pv, give sendsize @@ -1006,6 +1059,21 @@ sub escapeshellparam { return "'$par'"; } +sub getreceivetoken() { + my ($rhost,$fs,$isroot) = @_; + my $token = getzfsvalue($rhost,$fs,$isroot,"receive_resume_token"); + + if ($token ne '-' && $token ne '') { + return $token; + } + + if ($debug) { + print "DEBUG: no receive token found \n"; + } + + return +} + __END__ =head1 NAME @@ -1043,3 +1111,4 @@ Options: --quiet Suppresses non-error output --dumpsnaps Dumps a list of snapshots during the run --no-command-checks Do not check command existence before attempting transfer. Not recommended + --resume Save the state of unfinished receive streams and resume interrupted ones if available From 1f64c9c35aac5d45c433833af56bba4126a0bcbd Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 20 Feb 2018 18:16:35 +0100 Subject: [PATCH 27/97] let monitor-health check the capacity too --- sanoid | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sanoid b/sanoid index d6e58ce..6b1257e 100755 --- a/sanoid +++ b/sanoid @@ -798,6 +798,17 @@ sub check_zpool() { ## determine health of zpool and subsequent error status if ($health eq "ONLINE" ) { $state = "OK"; + + # check capacity + my $capn = $cap; + $capn =~ s/\D//g; + + if ($capn >= 80) { + $state = "WARNING"; + } + if ($capn >= 95) { + $state = "CRITICAL"; + } } else { if ($health eq "DEGRADED") { $state = "WARNING"; From 06d029db684f49a577a4d912f82c4ea0462e137e Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 22 Feb 2018 16:53:30 +0100 Subject: [PATCH 28/97] remove destroyed snapshots from cache file instead of regenerating the whole thing (which can take very long on systems with many snapshots and/or datasets) --- sanoid | 47 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/sanoid b/sanoid index d6e58ce..238f955 100755 --- a/sanoid +++ b/sanoid @@ -39,6 +39,7 @@ my %config = init($conf_file,$default_conf_file); # if we call getsnaps(%config,1) it will forcibly update the cache, TTL or no TTL my $forcecacheupdate = 0; +my $cache = '/var/cache/sanoidsnapshots.txt'; my $cacheTTL = 900; # 15 minutes my %snaps = getsnaps( \%config, $cacheTTL, $forcecacheupdate ); @@ -232,17 +233,23 @@ sub prune_snapshots { # print "found some snaps to prune!\n" if (checklock('sanoid_pruning')) { writelock('sanoid_pruning'); + my @pruned; foreach my $snap( @prunesnaps ){ if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; } if (iszfsbusy($path)) { print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; } else { - if (! $args{'readonly'}) { system($zfs, "destroy",$snap) == 0 or warn "could not remove $snap : $?"; } + if (! $args{'readonly'}) { + if (system($zfs, "destroy", $snap) == 0) { + push(@pruned, $snap); + } else { + warn "could not remove $snap : $?"; + } + } } } removelock('sanoid_pruning'); - $forcecacheupdate = 1; - %snaps = getsnaps(%config,$cacheTTL,$forcecacheupdate); + removecachedsnapshots(@pruned); } else { print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n"; } @@ -484,7 +491,6 @@ sub getsnaps { my ($config, $cacheTTL, $forcecacheupdate) = @_; - my $cache = '/var/cache/sanoidsnapshots.txt'; my @rawsnaps; my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, $mtime, $ctime, $blksize, $blocks) = stat($cache); @@ -1056,6 +1062,39 @@ sub getchilddatasets { return @children; } +#######################################################################################################################3 +#######################################################################################################################3 +#######################################################################################################################3 + +sub removecachedsnapshots { + my @prunedlist = shift; + my %pruned = map { $_ => 1 } @prunedlist; + + if (checklock('sanoid_cacheupdate')) { + writelock('sanoid_cacheupdate'); + + if ($args{'verbose'}) { + print "INFO: removing destroyed snapshots from cache.\n"; + } + open FH, "< $cache"; + my @rawsnaps = ; + close FH; + + open FH, "> $cache" or die 'Could not write to $cache!\n'; + foreach my $snapline ( @rawsnaps ) { + my @columns = split("\t", $snapline); + my $snap = $columns[0]; + print FH $snapline unless ( exists($pruned{$snap}) ); + } + close FH; + + removelock('sanoid_cacheupdate'); + %snaps = getsnaps(\%config,$cacheTTL,$forcecacheupdate); + } else { + if ($args{'verbose'}) { print "WARN: skipping cache update (snapshot removal) - valid cache update lock held by another sanoid process.\n"; } + } +} + __END__ =head1 NAME From d0f1445784c4e54470b72588d9d8954d0e2bf258 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 22 Feb 2018 17:29:58 +0100 Subject: [PATCH 29/97] defer cache updates after snapshot pruning and do them after all pruning is done (waiting for the cache update lock if necessary) --- sanoid | 74 ++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/sanoid b/sanoid index 238f955..e73b4a4 100755 --- a/sanoid +++ b/sanoid @@ -42,6 +42,7 @@ my $forcecacheupdate = 0; my $cache = '/var/cache/sanoidsnapshots.txt'; my $cacheTTL = 900; # 15 minutes my %snaps = getsnaps( \%config, $cacheTTL, $forcecacheupdate ); +my %pruned; my %snapsbytype = getsnapsbytype( \%config, \%snaps ); @@ -233,7 +234,6 @@ sub prune_snapshots { # print "found some snaps to prune!\n" if (checklock('sanoid_pruning')) { writelock('sanoid_pruning'); - my @pruned; foreach my $snap( @prunesnaps ){ if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; } if (iszfsbusy($path)) { @@ -241,7 +241,7 @@ sub prune_snapshots { } else { if (! $args{'readonly'}) { if (system($zfs, "destroy", $snap) == 0) { - push(@pruned, $snap); + $pruned{$snap} = 1; } else { warn "could not remove $snap : $?"; } @@ -249,7 +249,7 @@ sub prune_snapshots { } } removelock('sanoid_pruning'); - removecachedsnapshots(@pruned); + removecachedsnapshots(0); } else { print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n"; } @@ -258,7 +258,9 @@ sub prune_snapshots { } } - + # if there were any deferred cache updates, + # do them now and wait if necessary + removecachedsnapshots(1); } # end prune_snapshots @@ -1067,32 +1069,48 @@ sub getchilddatasets { #######################################################################################################################3 sub removecachedsnapshots { - my @prunedlist = shift; - my %pruned = map { $_ => 1 } @prunedlist; + my $wait = shift; - if (checklock('sanoid_cacheupdate')) { - writelock('sanoid_cacheupdate'); - - if ($args{'verbose'}) { - print "INFO: removing destroyed snapshots from cache.\n"; - } - open FH, "< $cache"; - my @rawsnaps = ; - close FH; - - open FH, "> $cache" or die 'Could not write to $cache!\n'; - foreach my $snapline ( @rawsnaps ) { - my @columns = split("\t", $snapline); - my $snap = $columns[0]; - print FH $snapline unless ( exists($pruned{$snap}) ); - } - close FH; - - removelock('sanoid_cacheupdate'); - %snaps = getsnaps(\%config,$cacheTTL,$forcecacheupdate); - } else { - if ($args{'verbose'}) { print "WARN: skipping cache update (snapshot removal) - valid cache update lock held by another sanoid process.\n"; } + if (not %pruned) { + return; } + + my $unlocked = checklock('sanoid_cacheupdate'); + + if ($wait != 1 && not $unlocked) { + if ($args{'verbose'}) { print "INFO: deferring cache update (snapshot removal) - valid cache update lock held by another sanoid process.\n"; } + return; + } + + # wait until we can get a lock to do our cache changes + while (not $unlocked) { + if ($args{'verbose'}) { print "INFO: waiting for cache update lock held by another sanoid process.\n"; } + sleep(10); + $unlocked = checklock('sanoid_cacheupdate'); + } + + writelock('sanoid_cacheupdate'); + + if ($args{'verbose'}) { + print "INFO: removing destroyed snapshots from cache.\n"; + } + open FH, "< $cache"; + my @rawsnaps = ; + close FH; + + open FH, "> $cache" or die 'Could not write to $cache!\n'; + foreach my $snapline ( @rawsnaps ) { + my @columns = split("\t", $snapline); + my $snap = $columns[0]; + print FH $snapline unless ( exists($pruned{$snap}) ); + } + close FH; + + removelock('sanoid_cacheupdate'); + %snaps = getsnaps(\%config,$cacheTTL,$forcecacheupdate); + + # clear hash + undef %pruned; } __END__ From 6c695f1a86274d8c7763833076b3b17e3a415b1b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 26 Feb 2018 18:16:06 +0100 Subject: [PATCH 30/97] allow monitor-health to optionally check zpool capacity too by providing limits along the flag --- README.md | 2 +- sanoid | 89 ++++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 66 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 9c4e6ba..6239b79 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da + --monitor-health - This option is designed to be run by a Nagios monitoring system. It reports on the health of the zpool your filesystems are on. It only monitors filesystems that are configured in the sanoid.conf file. + This option is designed to be run by a Nagios monitoring system. It reports on the health of the zpool your filesystems are on. It only monitors filesystems that are configured in the sanoid.conf file. Optionally it can check capacity limits too by appending them like '=80,95" (>= 80% warning, >=95% critical). + --force-update diff --git a/sanoid b/sanoid index 6b1257e..9dd6198 100755 --- a/sanoid +++ b/sanoid @@ -17,7 +17,7 @@ use Time::Local; # to parse dates in reverse my %args = ("configdir" => "/etc/sanoid"); GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet", - "monitor-health", "force-update", "configdir=s", + "monitor-health:s", "force-update", "configdir=s", "monitor-snapshots", "take-snapshots", "prune-snapshots" ) or pod2usage(2); @@ -51,7 +51,7 @@ my @params = ( \%config, \%snaps, \%snapsbytype, \%snapsbypath ); if ($args{'debug'}) { $args{'verbose'}=1; blabber (@params); } if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); } -if ($args{'monitor-health'}) { monitor_health(@params); } +if (defined($args{'monitor-health'})) { monitor_health(@params); } if ($args{'force-update'}) { my $snaps = getsnaps( \%config, $cacheTTL, 1 ); } if ($args{'cron'}) { @@ -76,13 +76,32 @@ sub monitor_health { my @messages; my $errlevel=0; + my %capacitylimits; + + # if provided, parse capacity limits + if ($args{'monitor-health'} ne "") { + my @values = split(',', $args{'monitor-health'}); + + if (!check_capacity_limit($values[0])) { + die "ERROR: invalid zpool capacity warning limit!\n"; + } + $capacitylimits{"warn"} = $values[0]; + + if (scalar @values > 1) { + if (!check_capacity_limit($values[1])) { + die "ERROR: invalid zpool capacity critical limit!\n"; + } + $capacitylimits{"crit"} = $values[1]; + } + } + foreach my $path (keys %{ $snapsbypath}) { my @pool = split ('/',$path); $pools{$pool[0]}=1; } foreach my $pool (keys %pools) { - my ($exitcode, $msg) = check_zpool($pool,2); + my ($exitcode, $msg) = check_zpool($pool,2,\%capacitylimits); if ($exitcode > $errlevel) { $errlevel = $exitcode; } chomp $msg; push (@messages, $msg); @@ -748,6 +767,8 @@ sub check_zpool() { my $pool=shift; my $verbose=shift; + my $capacitylimitsref=shift; + my %capacitylimits=%$capacitylimitsref; my $size=""; my $used=""; @@ -799,15 +820,21 @@ sub check_zpool() { if ($health eq "ONLINE" ) { $state = "OK"; - # check capacity - my $capn = $cap; - $capn =~ s/\D//g; + if (%capacitylimits) { + # check capacity + my $capn = $cap; + $capn =~ s/\D//g; - if ($capn >= 80) { - $state = "WARNING"; - } - if ($capn >= 95) { - $state = "CRITICAL"; + if ($capacitylimits{"warn"}) { + if ($capn >= $capacitylimits{"warn"}) { + $state = "WARNING"; + } + } + if ($capacitylimits{"crit"}) { + if ($capn >= $capacitylimits{"crit"}) { + $state = "CRITICAL"; + } + } } } else { if ($health eq "DEGRADED") { @@ -911,6 +938,20 @@ sub check_zpool() { return ($ERRORS{$state},$msg); } # end check_zpool() +sub check_capacity_limit() { + my $value = shift; + + if ($value !~ /^\d+\z/) { + return undef; + } + + if ($value < 1 || $value > 100) { + return undef; + } + + return 1 +} + ###################################################################################################### ###################################################################################################### ###################################################################################################### @@ -1081,19 +1122,19 @@ Assumes --cron --verbose if no other arguments (other than configdir) are specif Options: - --configdir=DIR Specify a directory to find config file sanoid.conf + --configdir=DIR Specify a directory to find config file sanoid.conf - --cron Creates snapshots and purges expired snapshots - --verbose Prints out additional information during a sanoid run - --readonly Simulates creation/deletion of snapshots - --quiet Suppresses non-error output - --force-update Clears out sanoid's zfs snapshot cache + --cron Creates snapshots and purges expired snapshots + --verbose Prints out additional information during a sanoid run + --readonly Simulates creation/deletion of snapshots + --quiet Suppresses non-error output + --force-update Clears out sanoid's zfs snapshot cache - --monitor-health Reports on zpool "health", in a Nagios compatible format - --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format - --take-snapshots Creates snapshots as specified in sanoid.conf - --prune-snapshots Purges expired snapshots as specified in sanoid.conf + --monitor-health[=wlimit[,climit]] Reports on zpool "health", in a Nagios compatible format + --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format + --take-snapshots Creates snapshots as specified in sanoid.conf + --prune-snapshots Purges expired snapshots as specified in sanoid.conf - --help Prints this helptext - --version Prints the version number - --debug Prints out a lot of additional information during a sanoid run + --help Prints this helptext + --version Prints the version number + --debug Prints out a lot of additional information during a sanoid run From 6f29bed441aa0c4db015271ceb3b24931286cc3b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 27 Feb 2018 17:53:04 +0100 Subject: [PATCH 31/97] Revert "allow monitor-health to optionally check zpool capacity too by providing limits along the flag" This reverts commit 6c695f1a86274d8c7763833076b3b17e3a415b1b. --- README.md | 2 +- sanoid | 89 +++++++++++++++---------------------------------------- 2 files changed, 25 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 6239b79..9c4e6ba 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da + --monitor-health - This option is designed to be run by a Nagios monitoring system. It reports on the health of the zpool your filesystems are on. It only monitors filesystems that are configured in the sanoid.conf file. Optionally it can check capacity limits too by appending them like '=80,95" (>= 80% warning, >=95% critical). + This option is designed to be run by a Nagios monitoring system. It reports on the health of the zpool your filesystems are on. It only monitors filesystems that are configured in the sanoid.conf file. + --force-update diff --git a/sanoid b/sanoid index 9dd6198..6b1257e 100755 --- a/sanoid +++ b/sanoid @@ -17,7 +17,7 @@ use Time::Local; # to parse dates in reverse my %args = ("configdir" => "/etc/sanoid"); GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet", - "monitor-health:s", "force-update", "configdir=s", + "monitor-health", "force-update", "configdir=s", "monitor-snapshots", "take-snapshots", "prune-snapshots" ) or pod2usage(2); @@ -51,7 +51,7 @@ my @params = ( \%config, \%snaps, \%snapsbytype, \%snapsbypath ); if ($args{'debug'}) { $args{'verbose'}=1; blabber (@params); } if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); } -if (defined($args{'monitor-health'})) { monitor_health(@params); } +if ($args{'monitor-health'}) { monitor_health(@params); } if ($args{'force-update'}) { my $snaps = getsnaps( \%config, $cacheTTL, 1 ); } if ($args{'cron'}) { @@ -76,32 +76,13 @@ sub monitor_health { my @messages; my $errlevel=0; - my %capacitylimits; - - # if provided, parse capacity limits - if ($args{'monitor-health'} ne "") { - my @values = split(',', $args{'monitor-health'}); - - if (!check_capacity_limit($values[0])) { - die "ERROR: invalid zpool capacity warning limit!\n"; - } - $capacitylimits{"warn"} = $values[0]; - - if (scalar @values > 1) { - if (!check_capacity_limit($values[1])) { - die "ERROR: invalid zpool capacity critical limit!\n"; - } - $capacitylimits{"crit"} = $values[1]; - } - } - foreach my $path (keys %{ $snapsbypath}) { my @pool = split ('/',$path); $pools{$pool[0]}=1; } foreach my $pool (keys %pools) { - my ($exitcode, $msg) = check_zpool($pool,2,\%capacitylimits); + my ($exitcode, $msg) = check_zpool($pool,2); if ($exitcode > $errlevel) { $errlevel = $exitcode; } chomp $msg; push (@messages, $msg); @@ -767,8 +748,6 @@ sub check_zpool() { my $pool=shift; my $verbose=shift; - my $capacitylimitsref=shift; - my %capacitylimits=%$capacitylimitsref; my $size=""; my $used=""; @@ -820,21 +799,15 @@ sub check_zpool() { if ($health eq "ONLINE" ) { $state = "OK"; - if (%capacitylimits) { - # check capacity - my $capn = $cap; - $capn =~ s/\D//g; + # check capacity + my $capn = $cap; + $capn =~ s/\D//g; - if ($capacitylimits{"warn"}) { - if ($capn >= $capacitylimits{"warn"}) { - $state = "WARNING"; - } - } - if ($capacitylimits{"crit"}) { - if ($capn >= $capacitylimits{"crit"}) { - $state = "CRITICAL"; - } - } + if ($capn >= 80) { + $state = "WARNING"; + } + if ($capn >= 95) { + $state = "CRITICAL"; } } else { if ($health eq "DEGRADED") { @@ -938,20 +911,6 @@ sub check_zpool() { return ($ERRORS{$state},$msg); } # end check_zpool() -sub check_capacity_limit() { - my $value = shift; - - if ($value !~ /^\d+\z/) { - return undef; - } - - if ($value < 1 || $value > 100) { - return undef; - } - - return 1 -} - ###################################################################################################### ###################################################################################################### ###################################################################################################### @@ -1122,19 +1081,19 @@ Assumes --cron --verbose if no other arguments (other than configdir) are specif Options: - --configdir=DIR Specify a directory to find config file sanoid.conf + --configdir=DIR Specify a directory to find config file sanoid.conf - --cron Creates snapshots and purges expired snapshots - --verbose Prints out additional information during a sanoid run - --readonly Simulates creation/deletion of snapshots - --quiet Suppresses non-error output - --force-update Clears out sanoid's zfs snapshot cache + --cron Creates snapshots and purges expired snapshots + --verbose Prints out additional information during a sanoid run + --readonly Simulates creation/deletion of snapshots + --quiet Suppresses non-error output + --force-update Clears out sanoid's zfs snapshot cache - --monitor-health[=wlimit[,climit]] Reports on zpool "health", in a Nagios compatible format - --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format - --take-snapshots Creates snapshots as specified in sanoid.conf - --prune-snapshots Purges expired snapshots as specified in sanoid.conf + --monitor-health Reports on zpool "health", in a Nagios compatible format + --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format + --take-snapshots Creates snapshots as specified in sanoid.conf + --prune-snapshots Purges expired snapshots as specified in sanoid.conf - --help Prints this helptext - --version Prints the version number - --debug Prints out a lot of additional information during a sanoid run + --help Prints this helptext + --version Prints the version number + --debug Prints out a lot of additional information during a sanoid run From 01398789f60f31f67002666daacce715c6e626a5 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 27 Feb 2018 17:53:32 +0100 Subject: [PATCH 32/97] Revert "let monitor-health check the capacity too" This reverts commit 1f64c9c35aac5d45c433833af56bba4126a0bcbd. --- sanoid | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/sanoid b/sanoid index 6b1257e..d6e58ce 100755 --- a/sanoid +++ b/sanoid @@ -798,17 +798,6 @@ sub check_zpool() { ## determine health of zpool and subsequent error status if ($health eq "ONLINE" ) { $state = "OK"; - - # check capacity - my $capn = $cap; - $capn =~ s/\D//g; - - if ($capn >= 80) { - $state = "WARNING"; - } - if ($capn >= 95) { - $state = "CRITICAL"; - } } else { if ($health eq "DEGRADED") { $state = "WARNING"; From f961a9f447344b0b5558b3c8382aec3c2b74632e Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 27 Feb 2018 17:58:51 +0100 Subject: [PATCH 33/97] implemented monitor-capacity flag for checking zpool capacity limits in the nagios monitoring format --- README.md | 5 ++ sanoid | 146 ++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 137 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 9c4e6ba..1b3b37f 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,11 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da This option is designed to be run by a Nagios monitoring system. It reports on the health of the zpool your filesystems are on. It only monitors filesystems that are configured in the sanoid.conf file. ++ --monitor-capacity + + This option is designed to be run by a Nagios monitoring system. It reports on the capacity of the zpool your filesystems are on. It only monitors pools that are configured in the sanoid.conf file. The default limits are 80% for the warning and 95% for the critical state. Those can be overridden by providing them + along like '=80,95". + + --force-update This clears out sanoid's zfs snapshot listing cache. This is normally not needed. diff --git a/sanoid b/sanoid index d6e58ce..660dfb2 100755 --- a/sanoid +++ b/sanoid @@ -18,7 +18,8 @@ use Time::Local; # to parse dates in reverse my %args = ("configdir" => "/etc/sanoid"); GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet", "monitor-health", "force-update", "configdir=s", - "monitor-snapshots", "take-snapshots", "prune-snapshots" + "monitor-snapshots", "take-snapshots", "prune-snapshots", + "monitor-capacity:s" ) or pod2usage(2); # If only config directory (or nothing) has been specified, default to --cron --verbose @@ -52,6 +53,7 @@ my @params = ( \%config, \%snaps, \%snapsbytype, \%snapsbypath ); if ($args{'debug'}) { $args{'verbose'}=1; blabber (@params); } if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); } if ($args{'monitor-health'}) { monitor_health(@params); } +if (defined($args{'monitor-capacity'})) { monitor_capacity(@params); } if ($args{'force-update'}) { my $snaps = getsnaps( \%config, $cacheTTL, 1 ); } if ($args{'cron'}) { @@ -174,6 +176,57 @@ sub monitor_snapshots { exit $errorlevel; } + +#################################################################################### +#################################################################################### +#################################################################################### + +sub monitor_capacity { + my ($config, $snaps, $snapsbytype, $snapsbypath) = @_; + my %pools; + my @messages; + my $errlevel=0; + + my %capacitylimits = ( + "warn" => 80, + "crit" => 95 + ); + + # if provided, parse capacity limits + if ($args{'monitor-capacity'} ne "") { + my @values = split(',', $args{'monitor-capacity'}); + + if (!check_capacity_limit($values[0])) { + die "ERROR: invalid zpool capacity warning limit!\n"; + } + $capacitylimits{"warn"} = $values[0]; + + if (scalar @values > 1) { + if (!check_capacity_limit($values[1])) { + die "ERROR: invalid zpool capacity critical limit!\n"; + } + $capacitylimits{"crit"} = $values[1]; + } + } + + foreach my $path (keys %{ $snapsbypath}) { + my @pool = split ('/',$path); + $pools{$pool[0]}=1; + } + + foreach my $pool (keys %pools) { + my ($exitcode, $msg) = check_zpool_capacity($pool,\%capacitylimits); + if ($exitcode > $errlevel) { $errlevel = $exitcode; } + chomp $msg; + push (@messages, $msg); + } + + my @warninglevels = ('','*** WARNING *** ','*** CRITICAL *** '); + my $message = $warninglevels[$errlevel] . join (', ',@messages); + print "$message\n"; + exit $errlevel; +} + #################################################################################### #################################################################################### #################################################################################### @@ -900,6 +953,70 @@ sub check_zpool() { return ($ERRORS{$state},$msg); } # end check_zpool() +sub check_capacity_limit() { + my $value = shift; + + if ($value !~ /^\d+\z/) { + return undef; + } + + if ($value < 1 || $value > 100) { + return undef; + } + + return 1 +} + +sub check_zpool_capacity() { + my %ERRORS=('DEPENDENT'=>4,'UNKNOWN'=>3,'OK'=>0,'WARNING'=>1,'CRITICAL'=>2); + my $state="UNKNOWN"; + my $msg="FAILURE"; + + my $pool=shift; + my $capacitylimitsref=shift; + my %capacitylimits=%$capacitylimitsref; + + my $statcommand="/sbin/zpool list -H -o cap $pool"; + + if (! open STAT, "$statcommand|") { + print ("$state '$statcommand' command returns no result!\n"); + exit $ERRORS{$state}; + } + + my $line = ; + close(STAT); + + chomp $line; + my @row = split(/ +/, $line); + my $cap=$row[0]; + + ## check for valid capacity value + if ($cap !~ m/^[0-9]{1,3}%$/ ) { + $state = "CRITICAL"; + $msg = sprintf "ZPOOL {%s} does not exist and/or is not responding!\n", $pool; + print $state, " ", $msg; + exit ($ERRORS{$state}); + } + + $state="OK"; + + # check capacity + my $capn = $cap; + $capn =~ s/\D//g; + + if ($capn >= $capacitylimits{"warn"}) { + $state = "WARNING"; + } + + if ($capn >= $capacitylimits{"crit"}) { + $state = "CRITICAL"; + } + + $msg = sprintf "ZPOOL %s : %s\n", $pool, $cap; + $msg = "$state $msg"; + return ($ERRORS{$state},$msg); +} # end check_zpool_capacity() + ###################################################################################################### ###################################################################################################### ###################################################################################################### @@ -1070,19 +1187,20 @@ Assumes --cron --verbose if no other arguments (other than configdir) are specif Options: - --configdir=DIR Specify a directory to find config file sanoid.conf + --configdir=DIR Specify a directory to find config file sanoid.conf - --cron Creates snapshots and purges expired snapshots - --verbose Prints out additional information during a sanoid run - --readonly Simulates creation/deletion of snapshots - --quiet Suppresses non-error output - --force-update Clears out sanoid's zfs snapshot cache + --cron Creates snapshots and purges expired snapshots + --verbose Prints out additional information during a sanoid run + --readonly Simulates creation/deletion of snapshots + --quiet Suppresses non-error output + --force-update Clears out sanoid's zfs snapshot cache - --monitor-health Reports on zpool "health", in a Nagios compatible format - --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format - --take-snapshots Creates snapshots as specified in sanoid.conf - --prune-snapshots Purges expired snapshots as specified in sanoid.conf + --monitor-health Reports on zpool "health", in a Nagios compatible format + --monitor-capacity[=wlimit[,climit]] Reports on zpool capacity, in a Nagios compatible format + --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format + --take-snapshots Creates snapshots as specified in sanoid.conf + --prune-snapshots Purges expired snapshots as specified in sanoid.conf - --help Prints this helptext - --version Prints the version number - --debug Prints out a lot of additional information during a sanoid run + --help Prints this helptext + --version Prints the version number + --debug Prints out a lot of additional information during a sanoid run From b405b589801dd05ea12be500766ef333638b3e80 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 1 Mar 2018 09:22:22 +0100 Subject: [PATCH 34/97] let monitor-capacity parse the limits from the configuration file --- README.md | 3 +- sanoid | 92 ++++++++++++++++++++++++-------------------- sanoid.defaults.conf | 4 ++ 3 files changed, 55 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 1b3b37f..324fd30 100644 --- a/README.md +++ b/README.md @@ -64,8 +64,7 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da + --monitor-capacity - This option is designed to be run by a Nagios monitoring system. It reports on the capacity of the zpool your filesystems are on. It only monitors pools that are configured in the sanoid.conf file. The default limits are 80% for the warning and 95% for the critical state. Those can be overridden by providing them - along like '=80,95". + This option is designed to be run by a Nagios monitoring system. It reports on the capacity of the zpool your filesystems are on. It only monitors pools that are configured in the sanoid.conf file. + --force-update diff --git a/sanoid b/sanoid index 660dfb2..a4a256a 100755 --- a/sanoid +++ b/sanoid @@ -19,7 +19,7 @@ my %args = ("configdir" => "/etc/sanoid"); GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet", "monitor-health", "force-update", "configdir=s", "monitor-snapshots", "take-snapshots", "prune-snapshots", - "monitor-capacity:s" + "monitor-capacity" ) or pod2usage(2); # If only config directory (or nothing) has been specified, default to --cron --verbose @@ -53,7 +53,7 @@ my @params = ( \%config, \%snaps, \%snapsbytype, \%snapsbypath ); if ($args{'debug'}) { $args{'verbose'}=1; blabber (@params); } if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); } if ($args{'monitor-health'}) { monitor_health(@params); } -if (defined($args{'monitor-capacity'})) { monitor_capacity(@params); } +if ($args{'monitor-capacity'}) { monitor_capacity(@params); } if ($args{'force-update'}) { my $snaps = getsnaps( \%config, $cacheTTL, 1 ); } if ($args{'cron'}) { @@ -187,35 +187,39 @@ sub monitor_capacity { my @messages; my $errlevel=0; - my %capacitylimits = ( - "warn" => 80, - "crit" => 95 - ); + # build pool list with corresponding capacity limits + foreach my $section (keys %config) { + my @pool = split ('/',$section); - # if provided, parse capacity limits - if ($args{'monitor-capacity'} ne "") { - my @values = split(',', $args{'monitor-capacity'}); + if (scalar @pool == 1 || !defined($pools{$pool[0]}) ) { + my %capacitylimits; - if (!check_capacity_limit($values[0])) { - die "ERROR: invalid zpool capacity warning limit!\n"; - } - $capacitylimits{"warn"} = $values[0]; + if (!check_capacity_limit($config{$section}{'capacity_warn'})) { + die "ERROR: invalid zpool capacity warning limit!\n"; + } - if (scalar @values > 1) { - if (!check_capacity_limit($values[1])) { + if ($config{$section}{'capacity_warn'} != 0) { + $capacitylimits{'warn'} = $config{$section}{'capacity_warn'}; + } + + if (!check_capacity_limit($config{$section}{'capacity_crit'})) { die "ERROR: invalid zpool capacity critical limit!\n"; } - $capacitylimits{"crit"} = $values[1]; - } - } - foreach my $path (keys %{ $snapsbypath}) { - my @pool = split ('/',$path); - $pools{$pool[0]}=1; + if ($config{$section}{'capacity_crit'} != 0) { + $capacitylimits{'crit'} = $config{$section}{'capacity_crit'}; + } + + if (%capacitylimits) { + $pools{$pool[0]} = \%capacitylimits; + } + } } foreach my $pool (keys %pools) { - my ($exitcode, $msg) = check_zpool_capacity($pool,\%capacitylimits); + my $capacitylimitsref = $pools{$pool}; + + my ($exitcode, $msg) = check_zpool_capacity($pool,\%$capacitylimitsref); if ($exitcode > $errlevel) { $errlevel = $exitcode; } chomp $msg; push (@messages, $msg); @@ -956,11 +960,11 @@ sub check_zpool() { sub check_capacity_limit() { my $value = shift; - if ($value !~ /^\d+\z/) { + if (!defined($value) || $value !~ /^\d+\z/) { return undef; } - if ($value < 1 || $value > 100) { + if ($value < 0 || $value > 100) { return undef; } @@ -1004,12 +1008,16 @@ sub check_zpool_capacity() { my $capn = $cap; $capn =~ s/\D//g; - if ($capn >= $capacitylimits{"warn"}) { - $state = "WARNING"; + if (defined($capacitylimits{"warn"})) { + if ($capn >= $capacitylimits{"warn"}) { + $state = "WARNING"; + } } - if ($capn >= $capacitylimits{"crit"}) { - $state = "CRITICAL"; + if (defined($capacitylimits{"crit"})) { + if ($capn >= $capacitylimits{"crit"}) { + $state = "CRITICAL"; + } } $msg = sprintf "ZPOOL %s : %s\n", $pool, $cap; @@ -1187,20 +1195,20 @@ Assumes --cron --verbose if no other arguments (other than configdir) are specif Options: - --configdir=DIR Specify a directory to find config file sanoid.conf + --configdir=DIR Specify a directory to find config file sanoid.conf - --cron Creates snapshots and purges expired snapshots - --verbose Prints out additional information during a sanoid run - --readonly Simulates creation/deletion of snapshots - --quiet Suppresses non-error output - --force-update Clears out sanoid's zfs snapshot cache + --cron Creates snapshots and purges expired snapshots + --verbose Prints out additional information during a sanoid run + --readonly Simulates creation/deletion of snapshots + --quiet Suppresses non-error output + --force-update Clears out sanoid's zfs snapshot cache - --monitor-health Reports on zpool "health", in a Nagios compatible format - --monitor-capacity[=wlimit[,climit]] Reports on zpool capacity, in a Nagios compatible format - --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format - --take-snapshots Creates snapshots as specified in sanoid.conf - --prune-snapshots Purges expired snapshots as specified in sanoid.conf + --monitor-health Reports on zpool "health", in a Nagios compatible format + --monitor-capacity Reports on zpool capacity, in a Nagios compatible format + --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format + --take-snapshots Creates snapshots as specified in sanoid.conf + --prune-snapshots Purges expired snapshots as specified in sanoid.conf - --help Prints this helptext - --version Prints the version number - --debug Prints out a lot of additional information during a sanoid run + --help Prints this helptext + --version Prints the version number + --debug Prints out a lot of additional information during a sanoid run diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 35c804d..d86cc47 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -70,3 +70,7 @@ monthly_warn = 32 monthly_crit = 35 yearly_warn = 0 yearly_crit = 0 + +# default limits for capacity checks (if set to 0, limit will not be checked) +capacity_warn = 80 +capacity_crit = 95 From 8dfdd1a7169b97a8b6d2a6a6c84b6ac3826caa49 Mon Sep 17 00:00:00 2001 From: Janne Savikko Date: Thu, 15 Mar 2018 13:50:12 +0200 Subject: [PATCH 35/97] syncoid: fix --version in options list of helptext syncoid does not have verbose option. Replace verbose with version in options list of helptext. --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 7337f5b..29f0176 100755 --- a/syncoid +++ b/syncoid @@ -956,7 +956,7 @@ Options: --sshoption|o=OPTION Passes OPTION to ssh for remote usage. Can be specified multiple times --help Prints this helptext - --verbose Prints the version number + --version Prints the version number --debug Prints out a lot of additional information during a syncoid run --monitor-version Currently does nothing --quiet Suppresses non-error output From ecf2a852b5e0fa54131c878597c9b24cffb8a663 Mon Sep 17 00:00:00 2001 From: danielewood Date: Thu, 12 Apr 2018 17:43:08 -0700 Subject: [PATCH 36/97] Added support for ZStandard compression. Available in all major distros with a simple yum/apt-get/pkg. References: ZSTD Compression by Allan Jude - https://www.youtube.com/watch?v=hWnWEitDPlM Zstandard - https://facebook.github.io/zstd/ --- syncoid | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 7337f5b..baf5053 100755 --- a/syncoid +++ b/syncoid @@ -368,6 +368,18 @@ sub compressargset { decomrawcmd => '/usr/bin/pigz', decomargs => '-dc', }, + 'zstd-fast' => { + rawcmd => '/usr/bin/zstd', + args => '-3', + decomrawcmd => '/usr/bin/zstd', + decomargs => '-dc', + }, + 'zstd-slow' => { + rawcmd => '/usr/bin/zstd', + args => '-19', + decomrawcmd => '/usr/bin/zstd', + decomargs => '-dc', + }, 'lzo' => { rawcmd => '/usr/bin/lzop', args => '', @@ -378,7 +390,7 @@ sub compressargset { if ($value eq 'default') { $value = $DEFAULT_COMPRESSION; - } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'lzo', 'default', 'none'))) { + } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lzo', 'default', 'none'))) { warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION"; $value = $DEFAULT_COMPRESSION; } From 1b5ab20b0d9fd83be65ab7266e294dc914a78cf0 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 25 Apr 2018 14:58:01 +0200 Subject: [PATCH 37/97] use resumeable zfs send/receive as default if supported by source and target --- README.md | 5 +++++ syncoid | 40 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9c4e6ba..b66deac 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,7 @@ syncoid root@remotehost:data/images/vm backup/images/vm Which would pull-replicate the filesystem from the remote host to the local system over an SSH tunnel. Syncoid supports recursive replication (replication of a dataset and all its child datasets) and uses mbuffer buffering, lzop compression, and pv progress bars if the utilities are available on the systems used. +If ZFS supports resumeable send/receive streams on both the source and target those will be enabled as default. ##### Syncoid Command Line Options @@ -147,6 +148,10 @@ Syncoid supports recursive replication (replication of a dataset and all its chi This argument tells syncoid to restrict itself to existing snapshots, instead of creating a semi-ephemeral syncoid snapshot at execution time. Especially useful in multi-target (A->B, A->C) replication schemes, where you might otherwise accumulate a large number of foreign syncoid snapshots. ++ --no-resume + + This argument tells syncoid to not use resumeable zfs send/receive streams. + + --dumpsnaps This prints a list of snapshots during the run. diff --git a/syncoid b/syncoid index 8a31f37..170799b 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "resume") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -46,6 +46,7 @@ my $rawsourcefs = $args{'source'}; my $rawtargetfs = $args{'target'}; my $debug = $args{'debug'}; my $quiet = $args{'quiet'}; +my $resume = !$args{'no-resume'}; my $zfscmd = '/sbin/zfs'; my $sshcmd = '/usr/bin/ssh'; @@ -163,7 +164,7 @@ sub syncdataset { my $receiveextraargs = ""; my $receivetoken; - if (defined $args{'resume'}) { + if ($resume) { # save state of interrupted receive stream $receiveextraargs = "-s"; @@ -457,6 +458,8 @@ sub checkcommands { $avail{'localmbuffer'} = 1; $avail{'sourcembuffer'} = 1; $avail{'targetmbuffer'} = 1; + $avail{'sourceresume'} = 1; + $avail{'targetresume'} = 1; return %avail; } @@ -564,6 +567,37 @@ sub checkcommands { $avail{'localpv'} = 1; } + # check for ZFS resume feature support + if ($resume) { + my $resumechkcmd = "$zfscmd get receive_resume_token -d 0"; + + if ($debug) { print "DEBUG: checking availability of zfs resume feature on source...\n"; } + $avail{'sourceresume'} = system("$sourcessh $resumechkcmd >/dev/null 2>&1"); + $avail{'sourceresume'} = $avail{'sourceresume'} == 0 ? 1 : 0; + + if ($debug) { print "DEBUG: checking availability of zfs resume feature on target...\n"; } + $avail{'targetresume'} = system("$targetssh $resumechkcmd >/dev/null 2>&1"); + $avail{'targetresume'} = $avail{'targetresume'} == 0 ? 1 : 0; + + if ($avail{'sourceresume'} == 0 || $avail{'targetresume'} == 0) { + # disable resume + $resume = ''; + + my @hosts = (); + if ($avail{'sourceresume'} == 0) { + push @hosts, 'source'; + } + if ($avail{'targetresume'} == 0) { + push @hosts, 'target'; + } + my $affected = join(" and ", @hosts); + print "WARN: ZFS resume feature not available on $affected machine - sync will continue without resume support.\n"; + } + } else { + $avail{'sourceresume'} = 0; + $avail{'targetresume'} = 0; + } + return %avail; } @@ -1111,4 +1145,4 @@ Options: --quiet Suppresses non-error output --dumpsnaps Dumps a list of snapshots during the run --no-command-checks Do not check command existence before attempting transfer. Not recommended - --resume Save the state of unfinished receive streams and resume interrupted ones if available + --no-resume Don't use the ZFS resume feature if available From e6eec0aca041dc5464f30cdededaa597dc3e062f Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 25 Apr 2018 17:15:55 +0200 Subject: [PATCH 38/97] fix snapshot list generation and incremental sync for snapshots containing whitespaces --- syncoid | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/syncoid b/syncoid index 8825dcb..0654ac3 100755 --- a/syncoid +++ b/syncoid @@ -189,6 +189,7 @@ sub syncdataset { return 0; } } + my $newsyncsnapescaped = escapeshellparam($newsyncsnap); # there is currently (2014-09-01) a bug in ZFS on Linux # that causes readonly to always show on if it's EVER @@ -257,7 +258,7 @@ sub syncdataset { # $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); - $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnapescaped $sourcefsescaped\@$newsyncsnap"; + $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap","$sourcefs\@$newsyncsnap",$sourceisroot); $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -325,7 +326,7 @@ sub syncdataset { system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped"); } - my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnap"; + my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); @@ -909,7 +910,7 @@ sub getsnaps() { my $guid = $line; $guid =~ s/^.*\sguid\s*(\d*).*/$1/; my $snap = $line; - $snap =~ s/^.*\@(\S*)\s*guid.*$/$1/; + $snap =~ s/^.*\@(.*)\tguid.*$/$1/; $snaps{$type}{$snap}{'guid'}=$guid; } } @@ -921,7 +922,7 @@ sub getsnaps() { my $creation = $line; $creation =~ s/^.*\screation\s*(\d*).*/$1/; my $snap = $line; - $snap =~ s/^.*\@(\S*)\s*creation.*$/$1/; + $snap =~ s/^.*\@(.*)\tcreation.*$/$1/; $snaps{$type}{$snap}{'creation'}=$creation; } } From 4ebc0abef5a4999c344e9e7d37d2686e101609f3 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 25 Apr 2018 14:58:35 -0400 Subject: [PATCH 39/97] fix use of uninitialized string in escapeshellparam() --- syncoid | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index 784bd90..10c78c1 100755 --- a/syncoid +++ b/syncoid @@ -1087,9 +1087,12 @@ sub getdate { } sub escapeshellparam { - my ($par) = @_; - # "escape" all single quotes - $par =~ s/'/'"'"'/g; + my $par; + if (scalar @_) { + ($par) = @_; + # "escape" all single quotes + $par =~ s/'/'"'"'/g; + } # single-quote entire string return "'$par'"; } From 2732392088cf713898882e54561a20a84b1b6538 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 25 Apr 2018 15:18:00 -0400 Subject: [PATCH 40/97] fix use of uninitialized value in escapeshellparam() --- syncoid | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index 10c78c1..df08122 100755 --- a/syncoid +++ b/syncoid @@ -1087,11 +1087,14 @@ sub getdate { } sub escapeshellparam { - my $par; - if (scalar @_) { - ($par) = @_; + my ($par) = @_; + # avoid use of uninitialized string in regex + if (length($par)) { # "escape" all single quotes $par =~ s/'/'"'"'/g; + } else { + # avoid use of uninitialized string in concatenation below + $par = ''; } # single-quote entire string return "'$par'"; From 72245338a805b565183f7a0579e98757c3e6cb14 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 25 Apr 2018 15:21:56 -0400 Subject: [PATCH 41/97] special character handling and resumable zfs receive support in syncoid --- CHANGELIST | 3 +++ VERSION | 2 +- sanoid | 2 +- syncoid | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 9af90f6..515d05d 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,6 @@ +1.4.18 implemented special character handling and support of ZFS resume/receive tokens by default in syncoid, + thank you @phreaker0! + 1.4.17 changed die to warn when unexpectedly unable to remove a snapshot - this allows sanoid to continue taking/removing other snapshots not affected by whatever lock prevented the first from being taken or removed diff --git a/VERSION b/VERSION index 04e0d3f..f689e8c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.17 +1.4.18 diff --git a/sanoid b/sanoid index d6e58ce..b6dc9fe 100755 --- a/sanoid +++ b/sanoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '1.4.17'; +$::VERSION = '1.4.18'; use strict; use warnings; diff --git a/syncoid b/syncoid index df08122..e63b68c 100755 --- a/syncoid +++ b/syncoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '1.4.16'; +$::VERSION = '1.4.18'; use strict; use warnings; From 979cb8bc3d154e30e7f0fbc016a1fdccc509941d Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Wed, 25 Apr 2018 15:24:24 -0400 Subject: [PATCH 42/97] resume/receive support in syncoid 1.4.18 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b66deac..cc75bdf 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,8 @@ Which would pull-replicate the filesystem from the remote host to the local syst Syncoid supports recursive replication (replication of a dataset and all its child datasets) and uses mbuffer buffering, lzop compression, and pv progress bars if the utilities are available on the systems used. If ZFS supports resumeable send/receive streams on both the source and target those will be enabled as default. +As of 1.4.18, syncoid also automatically supports and enables resume of interrupted replication when both source and target support this feature. + ##### Syncoid Command Line Options + [source] From b3b69c598c6350cedd7db938692f591935417997 Mon Sep 17 00:00:00 2001 From: dcrdev Date: Sat, 28 Apr 2018 17:38:19 +0100 Subject: [PATCH 43/97] Restructure dist packages & bump rpm spec to 1.4.18 --- {debian => packages/debian}/changelog | 0 {debian => packages/debian}/compat | 0 {debian => packages/debian}/control | 0 {debian => packages/debian}/copyright | 0 {debian => packages/debian}/rules | 0 .../debian}/sanoid.README.Debian | 0 {debian => packages/debian}/sanoid.docs | 0 {debian => packages/debian}/sanoid.service | 0 {debian => packages/debian}/sanoid.timer | 0 {debian => packages/debian}/source/format | 0 packages/rhel/sanoid-1.4.18.tar.gz | Bin 0 -> 45685 bytes sanoid.spec => packages/rhel/sanoid.spec | 9 +++++---- packages/rhel/sources | 1 + 13 files changed, 6 insertions(+), 4 deletions(-) rename {debian => packages/debian}/changelog (100%) rename {debian => packages/debian}/compat (100%) rename {debian => packages/debian}/control (100%) rename {debian => packages/debian}/copyright (100%) rename {debian => packages/debian}/rules (100%) rename {debian => packages/debian}/sanoid.README.Debian (100%) rename {debian => packages/debian}/sanoid.docs (100%) rename {debian => packages/debian}/sanoid.service (100%) rename {debian => packages/debian}/sanoid.timer (100%) rename {debian => packages/debian}/source/format (100%) create mode 100644 packages/rhel/sanoid-1.4.18.tar.gz rename sanoid.spec => packages/rhel/sanoid.spec (92%) create mode 100644 packages/rhel/sources diff --git a/debian/changelog b/packages/debian/changelog similarity index 100% rename from debian/changelog rename to packages/debian/changelog diff --git a/debian/compat b/packages/debian/compat similarity index 100% rename from debian/compat rename to packages/debian/compat diff --git a/debian/control b/packages/debian/control similarity index 100% rename from debian/control rename to packages/debian/control diff --git a/debian/copyright b/packages/debian/copyright similarity index 100% rename from debian/copyright rename to packages/debian/copyright diff --git a/debian/rules b/packages/debian/rules similarity index 100% rename from debian/rules rename to packages/debian/rules diff --git a/debian/sanoid.README.Debian b/packages/debian/sanoid.README.Debian similarity index 100% rename from debian/sanoid.README.Debian rename to packages/debian/sanoid.README.Debian diff --git a/debian/sanoid.docs b/packages/debian/sanoid.docs similarity index 100% rename from debian/sanoid.docs rename to packages/debian/sanoid.docs diff --git a/debian/sanoid.service b/packages/debian/sanoid.service similarity index 100% rename from debian/sanoid.service rename to packages/debian/sanoid.service diff --git a/debian/sanoid.timer b/packages/debian/sanoid.timer similarity index 100% rename from debian/sanoid.timer rename to packages/debian/sanoid.timer diff --git a/debian/source/format b/packages/debian/source/format similarity index 100% rename from debian/source/format rename to packages/debian/source/format diff --git a/packages/rhel/sanoid-1.4.18.tar.gz b/packages/rhel/sanoid-1.4.18.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..df137cd5d635538c965b675d9e48d0f1abab742a GIT binary patch literal 45685 zcmV(wKq2Q&StY$ zR}O~+XOQ8LT!NH5uC4p+_dE@Nt>;5B9uh#K(f#z(-B205f49hrd6d0d z>L}6G7oYw!#pmI}hw^`XZvP(-55Jfl>>p2ONBhS|{C#$KG&}ra@f*l;=hN6as<32L z6!pj6`?cq{H}`+!bF{DGV!hUR{rJ)Gqj>(!JU&QfM~C`gs*lrYI*+HbhtVue;y6B< zK6-RGO9ubjGJpPjx{rg}go!{PtoFP#7W{=vcQ4*nnFx8FsH|K#WI4qlmcnQ2A=ohV!C z*hHC%mr)hPwXW1M%9G6G3x)qH+mvNd)v8F!j;+-je1fpD;=$LHGMD`h%CokH(x-G#Be%w6IHpoS?XLh zxqc7XAY-<{KcaaiFI0M6VC6{JJSy$7sMQE77#miLvaGnV%2_%G#6@14yusjTg&6@VhxC1 zR9?))Z}XvA>x#R4vaBLoPiH$;>G!qH6a1E%_mneedQ_;QtWA;69$!b<(0$+McU4B# z*2}7B7E6w}x_UWoC;nr!D&}u?T0InLI+|}rMXuWg>Y{`qbDW*EQMI82Rn${Je+vyn zt5mG3I>M1chl!yawp);%lr)KcG@W*y9bclAs)R#z6P zIkB5HENq3@&U>c#=?7hMH3TGfEPy zr_3TRDm>Sig)iPLO}tb|p`EQ!h9L|JhCZaptc%>#MK$&|b99f*w6?)jXbsBRrFwdC z?tiw*o5xr^ZK?`_!*_}t|MAvk={VRhd}&lX_H#wmqTwdhC~Oi(PV>%nRH?M7n+g)< zS|?$1wpeS-L+wC}%8K0-FsM@09A=&c>s5uDkdoF#Sf#V{?(4patnAD!Mick?IHVmD zu%K3x;)oz{r58XX%T08^13<`&8&x)0W>pNpkxianK2wVZPw`zxI0vf#TkM2N>Abyt zpBq_6b-dJqZyY-AuLnE9P{@OYznabV`O=dl8m*zK7V!ei9jGfW+l|RG5wC-#YU9-x zOiz1gikBu!f=T1#Qlc{*I#_ucnauVrw!`Y;Xn!ut~aWR#Q12#>Zjliz324v+yWAM z;93MzX`?V~!JUo*sc2w{8|O`9cRB`1XkxG%{O(PGN0z5JUVKX$fz{fq^#?I{gSd=C zhCy^|^%fcS*NrkuaJ<%5Fy|)P1Uz$4CLyV(2x!y0I+fmSic)!}`-zIe`q5hdba%6u z&WXXz1+L>yfgw*lS-TU>U@-RJNCcofCecxVh$(?pGe#jpFNrdbOSAYeN`d`8cx^wh z;5lRU3L>h%XvlPSFq-Wj%&re$1?@nnTK}yvAP;o5FhLI~u>an~iw~w0dlmN$KMUEB zuu?X{1dor#`|3~Ty9o^t-W?_Ub@t78=6)TkRzgY;VJtRq^mSNtu4`(wCyq0>Z8<~+ zSr$fe&n~va8mr5Ku^42nw|y9G(!gk1^MFqkWyy2ipZ@FD_X*-76&$Y?oK?G#!r3+N z)f_p2N-7}fo)b6w07AuZKBTK66gsqVG>YqYBqdNb8^Hnv;VX%*jLhMu2o@G)qnn}v zOf1YIngb~=PA|FqN?;9+P~5=!U^geSON0o0(*}Jtn@+!04TR`OAo;M0u3>*dpzQ|7 zwdMdE8srAdk&4WVx`u9byb|i`gMSTCR6al#44{3vxQ;g(`k5+32#-~;MUNGJtKqwv5JEvHB5 zb$J)99ypi?R{1tsBci@WWbp!h(?S91YK@~>cN~pIjQ#VQBkci z!{~flHb;f=Q-%y;&}MG<1^(zmLkGpJ;q~x0jIL&h;|7HZX6K`*p*b5-K$bHWnEw4Ql{_$KJyWVI6t<0!OQZqKVgX%{MtFe~1F@LyYh6L^rJf`AvOMdr9!wfrP39(_lzcWKaYhOmGP|_M zR)7;f#Buj_gwC&(jGBX}iTt)_#)CG9`kZ|?Yz^rIP7xs34xwCo0Kt3*${;u~?w!AV zdG>~91WkB=Lj+RiJ8LojSO4L#hoNC8r2@vJMd3N*yfwdEy}EKRVLmvu8*3XMQ;=>b zZG;)kHBZV%AekB(7&)9UF_=fZiWZn%k~&fWjs?ds*Cuh8;)g2}P76?>nP5{SSZS+y z$gttA1ohp{)T=jAZz){D3%neWCaEk}BkOc7d+5%pe-!*7I1CxNUtcN&BqXw7M^6m% zunPFxWXJ*?pOq|%e(DkwXD0FloDIg4fg?(W1sosIF2ZrN4oq%FCR#4MV9{6$%eRu*LfXGI9*CKdkE~rv`6%qk51)JR zPFe(Waf(z_4%)UvxJ`kV2$IqMbW|4EW_B<=dV*8UF+XyQ^)Ra9lqNY+;hV8^$1d)5 zSxJF*{k*Sne#YuQsB;=` zzWVn)uYxF~6Z3{gC3Q@g=!K9v=L)H4Q>rDjDV&(k_RieWD?%u}-fKYKjEKW%|bp>|mT>!32GfJ^dL zfjU4M@<2Tfa7Rk-O3my&8-XkuQe|3}_~jX^B0)k6PaJSNlvRm3O7<&O?nA^YZD;?HaSr^cR?ojej+LvG1ngi*Jd5An@3nfTlO0D z8XOlo|Db-*b6thBzJ*_8YVqpbhq5TL-MyZ9N+GvmErySj;IgZ=WIlhufsMq9SxFnI zzZMtjdOChMK6Fcj!U4+|Zo-26OsX0jB{R}PTEt{lhy;`6jg&Y7VYahGVoHyEhscSf zGc3{?ASa+42iPjWqbM!b4#hp}gC)FoqCi;TI~Uh!_6v^#QpS}2Nk%0KQc@rvt*o1S z+QO?ZlHZz=O1y8au|MdQ)@9ohlI7OSLJ3)}HbDa5_X%XywfI0$=exi+)9L#FgX7 z3Ycs1?ILFn2a|=27*S)TLFQzqlxGHcZVG%bnzVYmBXX8KAt5U-}uHMEEqCc2I%ZlM3)IJ_v@b`s1U-J=rXq08QUeZX|Y&(|Qc zTjhI1K~PKh=7e<@|79|BUx$j{CsFrcUyaA(J*TkIs9PDLkE$LSe&D72rUjZ_XrHs& z>Vz#^<&HHP*3@biK`5wDSH9up+}#^HULj{D1qL77w|CE=4p#8K4#k0!Juk`eeUeB8 zzDWv~!gei47=fuh=?ejwz4x9WN(D8SKy&%(_372s56{F8CYHgNm=8(ZSz-ZT+;%T= zOAHpuZh`?8d24`yG+V!ihuz>7rMZPapI*Ja_>oPGN}>=g6%$f@tL*K!rOe9VJ9u ziOzV>^S8{bv=AzY-|Jgison8HLdo`e9+TDz(P4zW4rv8yl$e96YaPRVy6{bnp%_VsG7vcMzs$L9TBSEOYD?d0C|#|?#TD0 z@v;;}Kr~nnGIkw~+|A?O?i{nUUPoJ8Q1Ze}TUNp>9+M7$fmJH~+_n{l_x~l)TxFN( zWB*#uM%R0{*WTZH!&qSFys;Z^G6Ij1ROY$hy^j*NCV@e_%-kaCFBzEEMdAWrH=_k$ zxF&9gKB2mYucTw{dAnNY5ABVNHaWCDv`pLVvP&dfo9*)@)qm`>egE(Io3k_6*qC8= ztv}ToAHV-MogM5S-M;^KbTE7P`TpPU@Y%kI<*sD-Yd!v+6(L}|`tOr(ULNz=x_%ET zSQ7z#4dZrqV9<38Zq=dzCek*N1O+tW%JT#19=7cB4A zLMn?|m<)NNZByuh2#B!1DcNaMaV1>p+Tj1f+Ik%X42XO=mbIjzFRCF-&`VrLfztCn zq9h*jeBf)`LsE}(scd+f&@1IjG+%FAt1wR_`<93=TZeq!7nSqH{w0h7r*WZ$R~cNw zI%augaGuH>z8?~~?x3q@Z5?Ph%c1Wk$Yj1;WF(9vP#3R;xqEqS-@$F|WH6vVJek|% zByAq^k~zT(Hs&MyEv5L!1>ry z{*3+q_-7mc{T0nmyvDEK|Ni08!JYfxv;EKazkiR<)sL4?U!6Z2jNE4sZb7+Z$=$88 z%b_Z-hl(lXP_AAe z-ZD-h=%!r*Ve(bzo}H+utV4E(V{+GR{L%2Wr-*5+XH_!fXP1E046%PH3e%3W{o4OhU6bnNsdwR zrpi#_t?QME2vU;rbKRb%E?i;uTD$z7_qCAKuuB|nO1gG&Khi-HgR9fa&UXj?-e}Kr zTk3{{tbtWMl`84U$$4&`v-B*-<*(t^p6%#CxNKo+xYeB@Jnp&fl@o+cJ$-$8N!I0V zwJoLh{g*G*@zMA{n;srMy!-cG)8pgM z@&9-Dytw>cy*Rr(dvkiBUVr}`3_y7!xQ$Zb^Jib7I~c0{N9vCa!r1$*IdOj4VUi)u0Xw}CnX3U1`ir(9QC zo4WStC}tPHYB58zWI5-B8a_0<)GLt^BQp~tvg@`F=Tb>YfTXA>WjhJE>}T{YvflrB z&0CRD-*cC4MQb{j%Y3h!IaWhn;#TDCX5Jn_2%yB7PU}`0kcEK_%HUR9U|bo&>(nKx z{vcw7*O?{?-cqoHRW{_K%P>U&Zlgb9dQ7>=;XbmbR^{Omp`Yhl%7QqbNpzO0YsElVtV_q4kybj> zk-Lk|f(7H{MkI_Hc8DqCkNyrIXW}!?BHHr%w?mi6!V-R9CF4f3s~D{)4mHrqGvp)g zPinxGzrpBrKBR>YKuBUuf{f|!-|e>5Y!|=fehfq#giA4f6Y6=QbHF?Dsa=2E)xw!S z=`PtXs}wiJF87A*676+6e}R>qT2d@3iZv8d`+)-~Y?!(?l<)M*8)13h9fxeOlg_o0 zkSPpA&Q9NQ89PkJ?p!->Qux34k!q6nPq|*ezw>ccF0g0iG(!0-TzL@7@MWQTA?pI0SMk+gl@F$s-Msko_cS%SHlrIL|cS?ON z$+c9I(gI!So+f#6cRpztvb$Yx83J8I_9&Wp&KpqgBa-srMxi_ouBZEzVcH*7s6~o6u7r?RQ_SF5g`-&l`5JL zoHoTdb|mu@qQr!D?`0KwiP!Qu2JR-qd?l(cH8AK5_Mh$=zI%o5cbkitU0-fs{BOeU z$3&Rq@JA=*m&a2pu>59fa-{h&l?Q*j>j_yLMIc?DfDTZ(f%}^p}e19CuJ+B`7%L=pL?K_fe84twl|6Fq{!+uEq? z1q4z}KOUj;ahyluUDV`VO~tm6^f z4$y@afR=VDs4!9sQOw5_`G^lyKtWx|WDu2?Q8?V#Anme0V-p=2os5PNEypNgj^Nk{ zrOPu7mqcLWoyh?jcuft|3$d4|13fH0MG6q<{)&4B0N=YODk!r=FR?6(F+UOX4CiLx zNW&3Dv2@VK#Hx;f4`X0r6xWzpBj-qu0E(cMdh;O*$v(hnKxMol&X2D4$$*_IP~j4i z*7Jm~JGU7K5H(E;fDxtG%j`78xGa`>i5pXg`3#PiP8GFevNDTA@hBxPm?MnvKr<#m zS+!6ZaXd5|iU2X}3`0@X$Zm;8K6)bPVWlCpCAl_ne4`>qkfRGF+pG>FJqN-Gct%=Ek9Z!7fE6MZ0%Cw(V&JRgzx4&VVc;}% z#!5=aB!jExm?p>jWk{NdooNjxqaiLF@e_{WNG(SL0SG(!W^_Fx(?Lh`jNN||oA06< zQyek}49bL<)ngAF<8urgO&rlJ0-K*umIw-|Hmn3nK&-rI3#Y)G0PdF~M?knIN{u2N z4^E*Fm5Ic=dt87HjiD4c7b__UlFPkA9G6YcK&XgBoE(vSeGAb_rXLzGm*H+qmxT_u z_Fo-P#K*?&&gS;v_U_IBzP#AR06=vKr1rH#?~bW4drJGOr|6zNM^m|Q?#N1&HF32} z;z922yB2x<^x|`&>&#NaK%s0Ekz~^p(t-|!{S<)&M-wkIV7~0U6AFujCrWwBpT~wjT4T9l3x)0bULdRbWR6XSYPP{OYq&x1y zF$Yagtt;0FF|ef8kAMa2Fjpd!9(~MvV>QPjFeGd(mWg|pSWV^>A%BpnTt(a<{Z%_6 z`3heo>$CS30b||NI_P_j(OB@TCZG_FI;mI(c1+o9uY2k}QvE6aY=MlYJ3fKqT849% z&_ry`ct6su5KE|QHi2XAL63v~iGVPV%wA%oR89~zIUK5&61Ji2ln#z@H^^N_5RT|m zhu&~|1cw30^apYnWXrthJn+m%N+gYC;y7*$Xx3TG96IjVo~)BS!YC>;w{#>5=m>@9 zVRx!I_K6Azc!C(ty1~qQG^Eqs6t7ds$AB?Yr2b{>!u#}0l;Oa2CB2^A34qaAQbcz= zr+DUtOR!5I+K>#`MS;4GD3*=I8rlW2F0lm4QVK@%63>8vSuv+{;4BvG`zS%yQEV?B z4UulzbTeeLLB5;R8_Z0L>WSwMkYSu1xN*)B9)KPkqiF@;4LtG%5RG+bWH2v@U*E}) z<;ZK1$V9{)Fy^tiE1MVdh=WvbROqOm4%pU$JOnDZ)FG2vqkLgUW_5)f+o9Wl#w>U?8~!4sK`Jb=`d+H)KnX0{CK#&J2leX+vtBDaQxe(WI!cps z*%+%lbOe0+n3P{`AJD&+83T5U6&A@k8)oW8;oUWH1SfXVWMP*f4jY4d5D7%bp882b zLM#$SbSE64SrZXssSQ#BiJluMdg`5D^W@Q*h{`@WVzvO4LhLxv59mDUxA<84hm|D^ zwENU{kI;T2-Mh1eyocnv&`#rk@E&Ne^0oraHTM9?=aC6=O22`4nj#+-*>GnHGvKiK znCwR*s9-3!!SG!>NSMTED#9VqwQ@fk>72&*tueFQ@fxW7q!g36EqPcI^O zqi#aZJ$O;|GkDKQxYi#%74o88xIH>xr<9lJ5eB#71y8r5=PxC&>}7u<8-5I}U&Vh1AZ!4+$_7>~ZiOYJN{%If0qFI} zd!5esQ3GzM(ODw)c}o96Ymu}AcEUEwIx3BuQ6!7?K82~35r;O6CkGsbeODu~OBaga z3x)3zr$R(HpvESu6=qQrw4vNUFz5%Em!K*f4ty1PwSoGVa-*pJ9;DO+W2Q0k>D;=8 zk$ZB6LA_Q4*8uRMA6ZoAWI_$Rk$Hik7o8}QCrCvIX>s_pCs`&?e33jR7S{Y8oFaE@ z`LqEM7`t}lq9HmP!q5%$PDec-g)lRr3CKwr_DM)3KTmK?EN2#@m*P(SP&@-pq z&gW6_xDrr{LMBGj<=ziyowtZc<_T3QEAEa&n)8@W^mYRz2Vk-zTsUPsm62;49!s8W zyP#>U+Q-OV{T?G>|nTv zhnqc@@TeB)9_wd-0HbEyt!%k%p2ay(radc=uo$_H(a5Z9J|Qb!w}OEpq|~34i9_83 z-Zmvg^qwmCNgs?{AgluOnBaMFaWsfms2%D?p6&u)?_(A%t?J-P^GLfT-m zf5aCg@iYlAA_=fi8%)cypm5@#h?#3E&-!YnoD=!s3}`RZ=1p@vN9~lDa8btqNJBPB zosBTG!{#5Vs3Vbda6Dx0QAr5`{--N-O+QSZ1Z=Ucq7Mr@;rM2Cn|A;(Bw8>>b=g?C>< zr$xYL9U0sA*q&{dk#ya{0dv$Ofi|(E$NP1v8Q!2J;p&Ekn z?H;e$?!iL&J|EcU2%dGgty&v2g3?#lZ~-yl&TC`^Yh=iWxSe zb2`X+*$G}IAX~;{#&j_9ltOUqF~b+!=9E7&g9V?+>5%tP7y>42X-Ti3AGh~*J;H$r z!N!=N2*g^>hJHUSL%BWS+#-FFZ1Ohw`c$%60cg0E5_ibPc)lc;Ev@ zR3W65Yzsb*5fHdou~}y492*bFf0V)OPdx|Lz8yze%m{@Kd67!1}oJ38%TDy;KHQ~ zllyr-4_F$$Tq0G|p$iCVtO0nKA#T#ICewp_2m(JFWywD5)L3Y)K`M}++AP50XEsyW zwHi^tBJ3Dyxnnj3lMNS1jAaBvrirp|TtuirQTF7_+HATazsH?UHd0nxNYEo^)L>NV z&X|DpqYs?yRp++qVvSII^V}z@bztyTrrPYHHr#F`d-vcl;f#Oq4ebT2^ELe?Mpd|tqGHU--04_D zRlP6%JCXx9Gq z(X{DF&_|J^(WY$KNj&QI(C-9!ZFXYfPy|*fl0#J>xFBIcK8o6hGMEobl~z5+$@oHS z_D(~?$T^tE)73VVY)W{RekVEuboxN1eWlXn$t`~M-Oba^Q)PlhxNUxZC zX5$C-GS%?1im(y=QgC?{N0J3u&9t~B5Z@^yO&^8tZ0dqmVRaBB_z*A7?mMi9XN95V z6+#}0@kgp%2;{^Hr%5u#YA_GgP@D-Exb^{?iHLCu{Eh-fhv<9Q15E6QwEAixRkaSY z8yK0}!~4W)=bI=nqOgmm>9edi>GQ3I)hkAnY9b6Hv8;e+Vje`b_~Zl;8tt~J&eVa3 z7C+8CE491wW#*BpP?EL^|Yi56<@Kcm?f72*qkYMX{acv zP#D`Akcm;`ZonKUGXnp?)QiwF=t-{T7CJ%^&8yUi6$UG^G>lI+_%5e>@ZOcWVaF>d z(}^(vu?KD&;?>BYcY2W%4=iq8)QOi9)O(`>M;lGz_xi(lkg8sZsYJD^kFtu;Ip|JC ztUuG6KLK+8-@s(dW*}|Eu4lA zDY0adLoRQEWu|B8ZTW`xH^*Xt%t-jCsmjM@H9qh{2;kSpho)LciF21*Tr%6mqBVk} zmuCZUIAH_0j)}6AZ_^Nd#%aGSTTEDmF3HZVRm6i?*w1MjGzmRfAPGALnTQ~cgA;uA zsB0))`dcR+9o~kO@liY^&qQlgIYFO?qkN%oMUs%3t5S&-#aX8l=bB#<@fZX`3XL3f zQueUa+{CI^_FdSkr_Xn&Jv>&9jm+2u7P8dMF98X599b|0ai^}pzA{YcA z1J}X3f&}b=iCuch@Juri2}8RKNoiif-R4wO;)BPYe|)^9+R0eazAnMKL=TAGA=sTZ z=YmtM67jHwo`Q-Qp$nJ*LloBcQ+>lTik8&8^p*5O{cSDo!AeJp0=`~wZLC1PJMPrN= zrB%bWkRWXf9(fKk=xO8MtNJqBrEwo_+$M$*oo4Vm$lYBvUghovRLik7dDG?G>q=+? zP;tzR&${RVxuJK_3liN1QAnU_Q+VUbxuZqKmU~FW?JrntGwNoOqw(>iCuw1=)d999 z>zy&2ZB|s`JmM3Lw(&ivG`hiEPvooRxmcmt(=@dJP=Q}{qHZ%f!&b z$g+`@U@y#EH_PjTE0(x$kjU``c&T?M$W1e-P>XB_;OR~V)HU$}l(Bx4h`S;ujH04{ zUa7o3#aABtUR$~r@`T@Jza18kw~12G-|8OZA2H8Eun)`1x_L9A{L5W53^XMUOsiK~f(ar8isDjxjtR}Db96~CJ=6+C`2<}R zlj1ca#zv~)*R{1OgzWjx|Uv?8UF7?7H=>!K?rnyF5T*Tt?mWZIOxr=dMaOYp(P zEl#463QUFcL5WoQs#qoZBJtsHY}WI=ax~X8i-A6Loo|rG?Q`T7$Y$1v$=6}5LwP4r zJVuUv?Wha-As!&u%`~^3)!rRvSv&Es8YbfjCLU70G-3&wTvh6o>dkP5xvPPTj;G{s zoJ5sZ7_U3jZ!1tqm3Wl+a&IIhwWggh48fm>0m;{BTTKt?bRsO)B^Mj+^&n#du*U~t zu?h+XW1|pd*!9m=0VgHsnmO+P;^eqz-OYSuYD9SF62buN_jg}4tlN@5e@~8@w)C=x zUK|y=aFsG$cPi2&??~?GYCR%^<`k>!8zHtIIylM(6MJ4c9Y)-H0 z6fX)Z0L78Y1j6oj6H<#%D(0dQN=ss>BZZnXyGY9tc7~$cipae)9ZTIU83^WbmS#N) z#G}PcBoh$DaAqC!ZakLD*m>2(ol%zOen;Y>G2<*SRhYWZ$_R~lUj3ATx;xr5k9pa$$S!a;5>+!)wq4yN&))qO=) z;EQ2n!P(%%y1r7(6MD7LO&3?rJz~wbEePTH&|15#gnN$0dIGuNhqWg~O3+cf@%mfraHv1vZ~582A2^_-T)3oiL^rc;9|-!YR`_x z5~74;jubwF3k6$5W{LU~ED_+T*39|sioGa1mubY5^3|3R*Rm7?XP0)K-tC4nHjs)X znlgQ)6G9ZVIF9H=NMZ_YQlZ}+nGZG91L=D!{XX1i;JQ|N(v^#K8 z8Fhj~KNc+0Ri2>TMT!y-#@Ss)Zce9E`-+_HIHiH~-Iz%zLN_J)$r(R3#UY8LNp-(K z`xe@~C`=|*Cc^5Z)SX!sHAN?f+$=pHV!HIOIEPi_G1UmMZ^KPVY#E)|>&u1kc^luS{?_LWFvZGV z!5s`cd6EvSaO-eZrtczdr{t*6O2UYQ`lbaX%DR$h9?r+^h<6j-3Dw< z-&G!*NH!$CDskgghk=v9nn?5qoA)W#g})P4M%M|ufszKg5%w6EXlxl#{r14CZUn*% zE|2{fMf@N$f|O#SH+U9J_c{xg!U+<5ayRa=k7;P|?hgXDI=2!pfr)OJXl51o7om>_ zQs+EauBmI3ZDAY5^QNIya52{wX&)#rKwqwHnvTb{7nspwh4tXVe@|{sb{VSd*9IF? zG4@1j>hUF42>=@%z*Mf_#8&h+p*tJZ6TsggU8{*|KC-T$#NOcAPgz0K4S7Btnng?kT(4v(ys{fv9pz@-ApI&F3v@xPIz!&1 zl#R~LFR~pPuS4Q2*bxAwKj)*&p)G+=w9oo^_kY#J;*c_2xo4F3<7vIzd7)(wR&nIy3&ctD7+WdVT1x=UP+|s0H$Og3Dh|@xG>Tp3O>%is*Gd0O&1VF$^-dU+o zLfkNi1Rn@#PNJir6IvEKgi%q+*by+wmd9T&Ib?a~Co7()rZQCGvMz>U7jo-4JSorB zvliizHj8sl2w06+2Y@K*+cjjlqDSCDqsIKXO#(_fV10z|_d)S!He$r(f-3Bs1a#9J zs4fvO(-xD;Zh(|j3t3kz20QlRMp($VqeV9J6rRX4D)MFX*oIM7V_85FWYN;l0Pn>6 z-U~!Z$Q|4!U@V40C9GjV#a0Zz4Nn8IwZy8A7oJlXG4JUA$SDs={H|mX$HEDZBAYXk z?XI4xy2t<;(h-q^eXwtAgXH6jgC3tj{{;D*yCb>5o=>k zBhYldRnmDRonzK(I^KAIj>Trl?>cZ$&LFIH;CL9)D`O5r4j6v8Xu8DJP$2;t3F9#= zioOEbHY2D9K}gpEV&nnQTT^LC+JaV+Cb_qk=<2n&g40Atsh$;;Yqt6P?Byo32D+Y# zF&a(C{Sh?ru1O8e%W1%9)s`cEd zM2Oi#hRR>b$bsAx$z>K4>b_wIpoQXU%BQ)l_lkX)`XY^-9lI;87h`{`Jq&1iVQ=Oi zz(0{(n%N8?SpGIh(5hYuk-cRD?`7m>5N6-CN&7dX&?S^Kgc%j{;R}vvD}+m^@1$=W zk%Dl=STndN*g)q!SBC%O= z8xkV^rkO>qbyzCtc62ZPeN=X68j?u(rAy54;9 ziZIig**Rb|rY{CWQ^yx-M9HZv<(U#|6e{y%6(vwXvQ2r-xtxhX~#8^ z9K`6ytX^9KQ(m38xwDaI%t{dVw(I(n}025f6#WBbKC zWn}D96eX)+$jFKH8&2ac3kaB3k2G`YYrot*l|g~Y@X@@KF5}eD@3zK1~#bJ=V z|KyQM-^u8MA*B;km7w|TM1}asW7OqkN3grj3ex0T3CIe2%`}~7Bc)@Hdy@aSPu@+U z_41R#;W*CmNuo-YmfB`d(^_iExdCe(vr$bQZwf3Uo++!*rI>&UF}`p~dx`0SY%zN- z11E*C@x>F#d7I=eH=zrJP!|Q6CY(lp);9f~jUfr%bjfghu#1(7YG(M4Be-~yu9uC& zv!{9;H=Wax6#I=k!sNj2;6=);zCLa0O@Ybt$SmmXI5FJ7%ngv;wIShA*=72;h#aIY z^z2uK^j3_`Cpz*Kue2bS%%v$Cne}Qe z;hS@;Q1z1ORQVKM17Z2AGrdN}HJ zF{_rPIxWtb6Nua%=?{@`3JxN35phZj^2@@#l!MOR?sUreh`Y2eoRwk*OTwee!lxky zE3!2D6Ft3|h)}Xvy{<_6R0|i$L}GBE>X8UuO+-7z=J#~W`ZtL$L+5}@#Fr+EQkn2Q z2}!qPIIY{Xa|7+U#0yK7Vr|3Yoh5^q139T!!$kD{{Dz-YqZZbo5(%fL2_l$`7At3GA5rFAK4_`&SdZM;*KC{2~4)TTP8`V2#pRLcaExP3_M}MBf(suETfP^ih$@S82~}Dy43A={S zJJGu!C(G#2OOvz2Ibq1DYoZx2pGbvKW}9J|gCt0kikE9IbgF}UOg5Tno0kp17m!JI zLl4r4%3AP-NW#sM=uF}3WIwE;L&O&N-oZ&O!fRCm9m}BE)I#d*Yv-mHMq~{dxK_>f z=SxZ@X?jqBB~*BWr=R8p6*i-?sC+Ovmy2ErhhWB!6yap4wy6z-VO=i*28OC({*h5i zS4uXzpvc&&6nSi?<79uZI?sEEJ;R$e<4ax?UzoSfT~aTXZ-F@jC+HO1lPve*T#!O{ z)ag=4I~2N#mji=JG&#h)Ix!e8CWoHZ2LiiQ@k=IQ9WJh zbg1IBNnuGo`V_3-MYTvF^FLA1WeB9v61C0Fg$y8HUXn9{=bj+c4tAhK>*2v}4`!b%-jH+lA)013=<+i{o0|(Db%# z$9{6z1|>A%6`yC4-atdk^0Ytcjbp77!=8Gj%v^y=KvSe>q!?-{Xz8w*TrVZPys!R| z31SU1N`6vQ6)80TQ>x*eB^bt%?S zg3M->r}!dAK+8}TQ=|;u3q`^d872FK-bO*?3Gq*@EUPhd`bNUZm`96TUJjS*)la@q zDBvKmG1V@}WQzBl^n(l(~w;dXoZMXkqjn z#&p@!Di+c?Lvs#J9R8&QhXCWCA6y@LXluGY%#bupN2!hG6|Y6B^&;U0XUR^4*r1CM zB|WO4!{tn92wT*Md13^&XJ=CM1lVw9RAL?wntD>_ctY>?1VC!#a)y(P1e^UjTYzkknvXpjXhJVv3wE@9NNI6!9= z9xlWgFcr7r1T9M(a2c_IdkK(|iQR*?bq!}!`ca(2nZ&)=+TSwU2WDs2{J6fqzrJ($ zlXJo_nH-`jh+y|Mo6<(7H5 z{v%dS`tOacy+iZki>)2Ai{1aYeXwN?4%e~4_Kx{+fBSHI=X>gKV|VYT{q65x97Zp8 zUv6&gQ-!Jp7@ZoLz4iUW?X3fx#}C_^TmGC(ZT$eIT{Ay!AHLXqeP|~i?LLQ2elmaE z-q~!Lt!?UP>%aH*w+;?q0?_yNE12;X{J6cd@$&U1%%y3bK@U5-hvwxrtPEZ|+-*iU zn!KyM<7A-!S6llVFW~F?v+bAL052@7^?dtq2ZkWnt@F$`UcX%5kM>^g@9iFJwGD$1 zIst(1Zy)^CzzPJg|Mhy^b_CFcK3=WwY;56J-kKxWCS0!hY4-pBk;r0((O?(rEIe7hQOW=KQ2%SdjFJGFStqqv(`uG1H`~+bNz7L(2U{VXIuF8 z{?-lviq6>j#>VS?IB)m{wt(3jyoM9Fy~DeK>!l;Uy}udhAtStfzP|nP^?niYaO7P8 zA$Ca!=_1M0W1?HpvHj+PaE<%=q*&Om@J*E1X$a z{F{Xek11S=U4$cK36pbHfKD)l3(Nn^ph{2ixybC8j;YhjI1&^CTRu>Yggio>)={4I zFn*pACo}tjJbHST_Pn`Q=uA8w%sG+Q8y3Y%)>$FZFVt1}XbuIr1ue=>#R%HUYXF-y>J|qjAyQ;+M&LAbH#X=(MIH5Sl z9B%GR>@SnN5L?<5%Sb-v97gE4cS;&{>jo-zv2+}TrL&nGqH>$48s&NtVIf|vtZMB` zDjRE&H1ZK^qEmG&%0S1yYUq^Z{M9FLoAl6me2h!O8Qa!$T)T zMa+o!FVSXtSq;Oa#MBdYA>~1d_txxhB>vPaPg6r-S=1tCn2}J&`UTCNb3C!RU8$5U z@}`1Z9}%3?UUR&8>z~gloJhJp8l}lGT2Nw}oJbVp?EphSyuVAyN~By?eWPJMVs^Sq z7=gOTG<@%p$32vKt?ITBZ1;+-Hy&(qI#zj5Ob#43RBch5$tA~mAYVwp&vYKXsbWk? zwcz|HIQ*y{hMP9Zl1;lB0B)VOcGW4G7c@wRq;cl#VLLf1Ey|!x=sdF8xZ zWDb@r4g#{H=R;y1pN_}Fm4$`#^Yiw}VA9S;CkyKCvG4>Yw2rQO7=Cp+~M_G02130eIJ9U+;9ye^e2#56wACT3`oneR{fPkB zQKxizF+@cp848W9KdAGgIoQ^Md0MC5k8uuD!A{Y18nb&$EQmE}J8G#zM|}c5=CDwS zc2(~6?_Y1byvI_&fhIH|ErY26s{|NvlzpgKk4c#_d6}YyQ?_^zD?+kxm9enkLT8!9v&I4>FgDhWZn!P zh&*kNHP+mQ;~8CfJXvQrgT`285Bs?JX~KX!`1`{-!-lRRwf>NMQ4$_Sj?p!2l%WMP zk%C?q5@yE;P&s1*3@H7P`Ivbu|ES#TSv;fSl z-pR1nJ{|XauseVH*ZSn~AWOTgrS|>y(w`Uhx7IgbZMFN|uR6ved>%f$PyfQF@bA*Y zhf9lpTDrIVXmRPm@}md%eChtZ`;Y##_*G{y-S$WdUaRp_gHLN_oQ``*`0UB!h2fJZDh=-s4d?LeUTbOj(T4|1-1T|d9iOhj z69aNH(DbzjOS&Ky6rLRL7)0EL8IX&MR(C|Acs_`SaLu57V-TzPJIlB-*Z+Hdz=6M% zmJ-P&rC|xv*E2Ufn0zpQ{oxhf$e3~<&qhtl%u~L!u@&-&-b$C|acoXU$#LaM$z=+* zqzkY@H$nj5yQ5w_cwc*>ndQg{ChY?u9>=}M3-J?}Bfzh`q5d#Aj02MxPufSx!v8zT z&R_04%O{JA-SOIc@OMMzqIq=k56zFkB>&2aCWT* zD}-nHWcZ}MpJ3oLnz!)H6>5kV4K{8YVX7$LkkKtzv5KuiL&T?3m^_x5<0w>d zic4(m?b8V$pi63hxcllQUbV;R3CqK9CoUx8jsRT#Y2zE1kVj@H;W%U}MO3eKlj9g; zfNIGqn4%2fKjYNIK@nLOgRINd>5h};x0poO)_5?u>M&-Zy9TL3Xc7n`ETlm&(iLk&o;dTi1>N+_Zm`&f*H>b8IYUGgeXB<> zGPg*$B^8V;U2ZS$3?gM|3T5=ShCGpm6Lf!1J`6dUXFvhXGyu`IZaRvhTL;!$O$5p} ztNJ7mCHSrhDBq@#EW{R%M(_}bJ}o;T80S|kM#>UVW3;4HtjpAQG7VyN^dBcNUMaQ%a1=!p8ysrI~8dhKmgiH{P_qAz~Aa;K{y-Huk6 z1;6%NPXWLELaYW&#^YNF{m@}#TWE#!oCh!29FKhFCL^biE$)%5kAibroK{s=M_gNu z`aNj{wCk7S-0|5xcKHF~2&`h;YR_zPa3sK-iC;z$^|1~x1b~Q(dFb52G(Y)}TKEGW zl#G$SOgI9HoXtlgbd4bOqZ{A|iwTDVc_)5!ZpRNnvCB z{S+<&47!Gn*!_;2@mx=^pt6+Ym_r#t6@)&+R%FxkRsXg;>l1T78M+5$7DA_!IFCWf zv=+|G)Z_QD65EDz?WOARbZo?|#RKvqIb>{Q<$lQGF>Au8BzTHhz{Uv{uUNTy0mDgt z+PdZ{?7VqSI0CDcbv%<%Nt}T>ICx>kSgp8Mo^o-HpAGXN%39%+t0u`q*jW}~Sut0@ zYe8gJv&8Y+KIYX6XlOS#sAb4plX86xO(+i3^=LxQjQkG;0QY{$h7<}gN~Bl^-Kb}* z97PIpgtSd^0ZqnQFPv_7>l`mQqhu95(%R8B9}{m$n6+POHV(QA8fw8oX++({)!Ek5 zAa7if&&2^+En^9#8=xQn5#i9hL;L}&^z2Dznh*-Fc1lD&sVkixh4>)TFq%-^XCL5X zprI+2r>AMSNmMybs7jz?J!LA>c+a1?zqd+9#XVR?k&UFQOkDoNLdkfA$$<#xh$R0E zly5AKyTuN@pFXqK)(B|5#mRuPA^2*7kOJ;m#+UKSJoYe+3_VYvs||{k7)wAWSKDUT zteBHu(qYpK)01CX$1!QK@y{HD2SQy_m)&oe|G=vbE?2S3-N-x6S*P!eR#H6y{)-r>*$;-BtDixiJwkB^*_CTQrLqd7=V;_QcqF+7$?JE9*#@E ztjbzxMUJ81oHd7#>Ol{;O}Nf{V_`fwy2yr&>;kwySU~Hmtbn_lxdKJ)ueE|f){^VC z1uRzQJu<(WoU?$wixWngTL)NcIXsd&vz&j=`Zo(g)A=AzpCspG4`N3ODWEJ?jCTW) zKSLye88V?$XabNpDJE(Fa!al_=+d-OpZ&-=?7g)u<49V69~rD(7DbfYiDT|!(i(Iw z54VsuSu&A5teqD~AIJ|MCFAoX0cN_see6p9kVuzOQUc|gYz`kfCEZLklmQi``4lIq zY%J<^c?PJE9K~as?uxA9X;6`@hv!r)aA2Y}4n%Q%AQa^a<^few!_9yxldN?0erTo| ztwTVO*22z_UG%Ue2l4Eknl3tr&SbqJXYr-L|RSt!wNFYS~!-}v$o*0l~z>F z$W7RJ zk6x*FDIYCx|4&axaXwyL!i0R9pXfOKKu)%z|7&HUyjog2i+fG}dl~?^aI2mLk&l`bxau;I`6TtK}6G>w%ViLM8w_4^u)4n-~dl>%l_ym?RIePl{QQpeY zY3zwHKcYVCJF`AHG0TffkId5Y%JQFA?tf=mi}x27Z#a?P_ft*(F>Y@rwfn5k)%^c* zk^Vn;uy{lNe~-_1HwW(D`ZMMHQ!3%FHO5uv|Ivez{Quz5^7763{~bP@_;p42fJma@ zSAcoKET@BL4|CL|<__B<*!un+ZJY<|_t} z>on=1qPYHK&3xC04n*OcwzUDlb805+w+0K?=oA}|%| z^Y=q1Z+)MRuls%*ex^3hM%j6e>$v7P(O%qnPZN^|>!Z$TItHLkFoGA$-`eHQee9=s zXGOkWHsJxRz2`mn#$;-rk%K4WFxvzs`SDgRl2g?sj^563Phd1Z{clbOfV~RTf805iNK*4a`tg|pA zoCiUp#ftOX;YW*YF$+uW#f9kgkoE7aow)B&6d^E=uXn_rMA37^`4w|FvJt&2EGf(s zEHwGt3IrAwNcH-!^+ascM*K?AghFEhA1CPPZ55IKz6gz(`PzVbmpX zlo0gbiW6I^qRW-UBT}o4Y~PfF1p9A$gJTUyGfX@0IRZ%PTDbDo&b_(Mc0DP$!=+le zvgU0?W}5($s{Ej7lmhn{-1A-q2ual{>EtN+ix9wnrXbtVin_CBmo7?;F z+FX6(^*;RBP-GQ81Kk2fX*YS-J$;8>uJ6Qq4!V5We-8x6w1&`FI%+U^gJrLpH_@$I zouT@WsN(@v^QG^-BpF`x;bQwx*?da(E+DBeKp1K{rOB*06G=wCth)PRuT^b=5K>A0 zJZ(%6M;2ro{+oO%>A&o#WP{+V?^ld*mHubx-u)u|fAnzq z#{c8@_#9^JQQBlKmfeGiAO!K3z!6qbl0xuhKv-@L`#-^F*7=8R{wiGHRp)a5lg;j{xq;((_)P!r2M=%jzkip{to>)> z?yn!?YWeR$iT}fAH}wCv|J-XYnmUH~pCx}>5ZcY>5Br4YUyL4Kd5qcge`yK0y;A(o z{d@Or&j0W5@$7%(WrkTNDT9N0kWj`+E;UJJKn!Jft}4^KP3CiSkuhgTUTiJ&7-J{K zBdmVXg>fhveO4~?lVJ~?Y$)?K8WNWQFIr1WEiB|?)>&cgr-={h)Jo-CH0*PU0Gv|F zIn;8DR=B{F1J`p(S~%)uM+^NJxZBaf%k7P=or5h*!iBJ4UxHQo8-Rd8_4#TP^)JjE zh8M4yc@|>f%ls-nz5~oJK0m?i=qGvo>H7ZnKQyEXx`e1x1~kQE`3Ra}8vFVjzW#Au zexAQ(Dhx-{>Ti`>lv78%{5C8YU~ z^8c=JVjldv?sX!6xvpStwqc(Bf(}W-Dxi!*P7bH0=9DhX_&T|H{U;lGjZ$EB)xpvXuo109^=*2F_s@Z__E%) z!qogOl)4=f5J^EjhFZ_>=@Md9QL3*`E9KZWS zgu>hEJWV?OGz6L&9!5R zj5&?rX)0AIXnQFkBGEIrI5C%q=ETqfy!AXKD{MFEPKLmnGZf%&NxQd(6zA5hO6prd zLT~Z8oKE&k@()8`F#qu%W)@wg7wK>5_*O8hC7cx?EdM_a`i(bjY2x*-P;?Vn<9V!3 ze^F<$CKFlZ(Y#aQA(tEyi|zJ^IHX6E(Iav-#(yTtTZCtfh8HnN0v&DYw4-oLD-gPd z*Vp+e+TQuKbhK7S48exbsKJn=O~prp1SUF9duPA4=ZQr?jO4MK^doWw2rjA>DBZ0( z(CIWCaZ&EU-T@)Ja5?7r3tR?rJmja@dVyu&CWA56W;35^LVIiq?ltrH`HLs3G7D&~ zKja;MyIS}+d27`@q^=c}Y$t$AYYRfhZc|9Omlun2 zNntaoe6{g%+oc90m&P@2=Osy_$5eC!I*KvnVV(@*5hW&bb-H>NZ6b&|FPyPV_tuhf4Qt!(f_nsZcW{6f=Na3&Vt{=0q{lcUr4*#X!sAwB;7PI zwXOaA-Tf8l3hy?O7b(4%1BQ&=CUND`UQqsqN{4*M5Mx4*+$H09C<3Y6uL5(*0)R== zsD6dpB8CafMVg%_GE2RI$u%7dz*9CoO81JJ+@Iu}pHbSm9%%z(-n=GA&XgfnqaX(8 zice!oJDZ0PqZuwxD5hESwERqe^ozElZWMTg3Aa=xYmYHg z|GTube1G{~N&mZi6aVqMd~Wo=U#|b1TUn8WiK?qz`ZLOh)78TGZG+sZS5~$MDaEI* zn%h*25(@ww#jSjNA->n-c}-0=K@M11*_`xYi&jbLblVWaj8VC!C0bx9%14o-sBoaB zIK;}z9!jX9uewd!IO>ma>ojyHrD|vID5Vdz`#u?G!|}?>OAJ~3$9a84yum57#%4>! zFBm1%;HB+l-IbNsd3=(%!G>9Pd4iAV4fGztPCIdLRo}qsq46lE1n->YbA)k4Se3(5 zm)C2yY*5XtJux-UvkZl0n6PA$ufMs&QbkRwd(@g(4GqABf1nQ;{)-?-jSc=yxoT_8 zs><`Vu#gMB?Uz2G9(7)8`4xTGgihEjEK!6{3Uza8USCwx)`GIWj5?>=7&l5vC)pc~ zvIG$w-X3I>n6`nq07nDU1Snh5)M5b$dv>eyYM~(Wz#1e1{bRFCoR(bi`0~6mTFxYa zE%mV&9fo8vIH78MHX zoC0k;JbYICQ_<`Tq8yTw5P84IV9X^=W?zZVANSw7iIA*yXkFh{voTqrr5MlSK zR$=})$$1TXhm_7G2c{xw*{I9yNPCb^j&N>IQKaq@B`HtD{=rN1zS2?}p#_e06_<@$ zw?3NU30PZNMOpp`^N81FOyaM-_9b$_wmf___N=f$^p zj#+dKM!^nb6nfwGU7tgg=%9(o%=PmVToD!Act>T~>eRHk+_2h2ece29%cA3=Ur>nF zmsma*6*u&by^tccZXqPYu3Dw_Q{g(PGa`HQ^Ue9YckfaT%I%%+4gM{UHukpQ>b z@-wxZOW!z?ptGOtB;T60`FjSmw?5y5zU*xH5VHwgo{_Qq79AYh9Cq>Y<_P&m$dMd0 zw%j%YE(bX5O@P%m$C9jv$xm~sx!k-*-2Coeo3yP>yOWfclQeB@>WGn1yMITweanoo za0pcy%cWL_6yE@$7;udQg#_zJ4`36q8_ss&%3WZdD_1d~$%ta=uGTV;YZx-=Ht-@D z;km2$E&6p7gX9L9Fx_CZxO&6UjLe>>MaAs=L@7*uqQnOKEgPeydbFH+o}p)Z)89!? zs`*k;T}o;>KDeR|<+dNS$)&x>`OXR7uXww%0BzV-Y!_;ntmuOjbe4Ybnwx6E6gr9E zW{SbZet}3xC1CiA$^^r|T#uJ%MF2438SC6fgRq$Alw&oHWCRjynswEB%{o@B&Oy0o z6RsN}?}&xW!!yF5jQl zSO9uuTZI}nq$9N^kZqg!clcm_)qjOfmj4PqaIa_&C48{`Uz)px3E9TlcXea@bw*=3 z;n2RdWK9DgM5S2z)IZ)I2?eE7Kvb?%H1CxBuBYoXvnp%r0_NJ|w_bd~4rEG#R{eJc zUcrm_EPP7<6TF4b8g`*>{Z9mrpFVjoL7?dRs?WR!;wNTN-@THcyz2svG7elSQ{jl< zva0)q6RDbx(g!4uO;f{LWku);suMs=WnA-8RHK!o_YHa86b&%bdwT zuy0wfYwDuy19|YV+k=K3GqV3R&Wh!20qx<{0Oteo`2eSLd5Jz4bsyrSKq+s-lHG;? zICzbJCE(C<1vr-#RxvlOX|I{u?q9!XTxRTU5#M>sIZZr87to~pj=Be?n;k-X9gw>k zrUQ31{(01Zjq>kcX++o36|LmDzJ-@Lyvl4Ss?nOk*vbcJmsRi4Zw>3bh#fkc8Fx?6 zLH&;Gc;bpXnm6nblfP-Of$(^g0AQUSj|sZvhx3yHFPteie8UVgH(V6IKSqJbZgmNs z2lV#N^W7C{7*wNcx2+ZV3pvbdu$a}Fy5EtHL_Ivg><{21^+k`btKjjKrsny+kOV?o%E80DL&|ZGv;GyXcSt*(w zQ)NnPbHb|fSh7KCE0Wq2s5tquqS|_Z<V(iod^XNep?hI!1pT#pi#2=#h=sM=`bj*1U&uY@jtQ1+=dI1e4Td`DW@NLl zMTc<)f3af@CjH`=c_;40sMY4Z(w!pF8Hta&x8Ax{a_6pFjA#f)bz2EHd3!V4@swS^ zk-@~BVRW&Gg5Lm9I_&$~G_rrllJK8x%{Feq>^_btf2cxEK*U(pptoF*Z$i zaw;8^rr&B6xMIuo)@38T88rfK*X|8xgUz$Puz+N$#YW!%s2!w9w}b?xKF^l6cXAKU zHypUmfm2VaUs51cV2zui#hhE8#|kh`0UuzTJDGNp=D!^0N0aP06BL3jq^79kjuDQ>+W}8lytIN#bZKrI+_VA*dSmDz zzPw;l`}pO0+h4&|lk<3TF1O7!1p}D*G`Zk+<ly?%~zP-sq7AKw)|szFgNUGah=-(DMwx`LM#k*cIiab246zMth| zv7#9JhNETlE~+FDcunLunsX?hV8Gg-wbYzLX`msdACKYkhFZ1o?Zx@n{~bdSd>D&x}_xq+~nFX_kp*c`EosY8(kGz zOATDw-?5YhEH6c20pv8Zc)byEH|&TZya99Yb;5{x*hRzROAC#ji1b0yUieeX38m;r zXP-05Pyf?T7C((=%804%k;MY)Trb1jaOk1&anc4@NvjMjd1fphMjcA2JbD(XdtDeP)Bq< zjtQ&P4oou4fT<;TQMFyHTslg$$W9`^gS3tV`5kL_Wqlw6rZttB^4M6?5pg?Dd*Z6= z(tx5CFvBd*Qw&Y8`^pWbi$UC-2r{T>^y&|t##nl&V4p*+1W5xrD^-NYC)sMSd{mpc zW$0^Juj`4(thZ8G3*)<4O{ap)J0C+)zlVVr^Ox^vQniL}cyy*gR?V>r%5~oGG*IPS z5KiDzFkd0`6~sO6Ck!Y2rb^R7GY+*a+6P(;shb*WTo^=F?P5LXB)XBj+{5wBG4L|z ze2Whoo^OADVE8K^QhtM**VSVix9N7+ z_F!jy@8HGmVc7b1p*bJKDXpLVbokTW*46EK!P9%$TR(hponAycZlORgUt3voZ6`SH z{JHgDbUrRCLI>p)l9&BmROQ0FD&FTWeOn8iRlkEeQ`;4zxoL#_R^$w-97S{&W&ti| z`cK2_T>C-}hp2c%;5AF_Q`VHI;>TZeu<%N)id#QD()**-34Xdsp2BUqGSz@JvvmN% znJ}zhM-Sw|vf&R}f@afT;+gBHrSQFZUBso^pf{W>3~xYHF$)E`Q7zjPCVte>6uKJt z{d(`Ol9O`^yeP+2jk3wfDaZfxv+krvEDlkNmP7@~*A&wLa8d(ugvL$%%4CLLyn~6o z)I#d_Pf4u`r?PM!?crT38bu``68Ng(b6*?R?T{%KqKTcx10p=NN)5x~e?t_DM3Kl$ktj~5iR#b)f0ZUWyqd;KRh;Pp zXD4R(jt`k`m?=|Lsnw$io=?gY`fPu)XIXiQ3L;iLpDA=q1}M`p`C%Fzlhs^F z$8-WA>8WOCrx3C7;v3%T=e+S|)1Jz~J+d_y!m)x*$w?~HDLNI=4W$YQRR!NRBi=mB zxkx`yW`MJJv_Rh^a^IQl#~(E3=u>wbHJ@X024i5ZPs7PRjb!^YlWo_LC-Noy9Bs6h3`2-dr}2&tE#rC1y1zs@W)!i>7{b`oqwvr=ri# z=*qGqg=CZHSJ`?obn0A3+dpwzizhDSFI@QK`$XeDg+P zuy|q5U-%dq3~{T{yQ;KjKDe&s)8ZVX7@UHB>J$|`3_PRa!BpQUQvmW~1;}+^$*i7R z2Kn$7lP0)TY;HY!{k`G>`zSL$)N7_)ac0`4<}0dM6H0kAj@@Z$tWb)zK>JG>NeQ#) zEw|7q>#SO)x5{EvIP0jNhN25mq~pA4Z!-Q%sUGVm(5I%qP;cLDym<=$?Oin9Tk|*4htR$@!whaA65Ko zI>1|cYai#8clMfH^XkmkS$tiy^YfM1r7QU+Oh)cpOimLdTi_fu6=`t-34t$W#3}Xf zY9&>vbZoxH=j(^-FY%lX@R0VhxLfsl%M5!-oTGn!j`2|-LUelQ0gq|AnrtCVb2{ge zcQNUvWHF+=JKyk;^rv*qsZ>mXju6g(DIu9ggL1}42_9fGnhZExhzQ{~5~wwXk;pqy z(gii~76@~8%{SrZZ*j(i$o|`w|#nn^3?o_`fbV(jPPv(0)qFkK6N1at* zZKFPP_)B7j8Rq#(Da->)$x*WO;~1|M%14=FGB}QHI6)Z|MHS`=`2okSTX)N*7a!HE zh}djagw5J#@Yg8mOh$QnhOMieGR0-}X^KYeWUao`tl=NEX6;b^*?+wS|6C+F{P!pR z@74kQ7ju!-cIiJmHP;O-cCEhHtkGZa^K<=o{pA66HOSyUJM^Dj?C1Cx`{4gRe_m@` zVJrb05y#Bi93+J%#}=UhU8|=VYzJQ7lfnBzc0Mqa|0aR0!rTuDMP8-639L)V5sTC# zmZFX{J8HWip27sIOb^_8>eK_5o^kgZqc>Syj|(RHkpbSxqj|VTpsP#kx$p2;4YAwh zM6+Z)O~%A<(sVAe3Gi9Es5d|BjNZJ(@9qq=U9799OQ(AUzw;esF<&3eLh#3f6en%# zU3fJFa2~p5M0M#uXN?3}|NWe_l-dpNf|O1VveQH^;P_0wEHNp&Me)Lk}nuVoNp|a}jyT%GIagnB&SjBMdKVN9AoTYlpm5%-cP^ z4i9M?;tbwM&u?tkZ!Q_DHS3*F5t?9`7<-KHZGSI__qLXXQC-cm^+PABFCv+R7?k_X4ZiS#Jq0+bl4IxQ(ygL_wQ3c#TH!nONDU` zPZ_18zw#^w5*shtVjt~-4@SDm6eVm@Lg~4Q1PDk>G!qy|LVg__+{&+mYqA(iJm7#@ z3gFgvjpeodfTS59pR<Xj>c3H39CZP&^#=QVyyWZ$QMtCJS+ln37cwC!b z*Y%Vr{X6XrlVb2Jn?Ucn6IxKSf1$rOin2;@3w)gS`?6XIy58-4T^28Mw`uOqH?QcU zWat5D%}%%OI#xuuO$l6wzt zZNDk+-t5~^-*p)MSt$J*Beh-RTqRmxh1YI%K023!vnsF}c@H9^a$^dX2V~Tw<2>md zU!Map0SofG6AgO{3o9a*fd|p6;2Az%EkMw@o3U{NnQMU?a&^_rHvYm7sHTPGySt8= z2+XiGDI^-PjS3U^BEL=D)V+NdgSr&p(w;WvEmSFyJam?$h!RM zk)dh=RVM5eeRv2FK(OM?KvXbc*)jM8YD3j{sSQt+XE$m@G2L-G$d84fT!3g00s|X5 ze;GzvVJJpL^j<|cRCLBa>F7FXrQZF@@xt%dHPp1l|>duC9qsMECa9DHk&fza&Q(vZmCghKgh&l zM7`9?JK8D-d6j4NuDr;V63A-BZL$Fo#gV;9t(5i*yHUdB9mN})2U9CA+oEwbSZ`Bm z0u$S4Y5{Q52q;rxrzy~gifypwJR9|G7o5advO%g}g7@Pbr4hoC0Anwyp!Fbz|G=SN zwJq!$tu<+i$6Noie1A3Qn>~%Ec0R3NZe}|U`;>wJmAy{uP}`JyQ^PCu=BLdJmVxk9 z@VcM4NF~64_}<_pN<7E_LZk0|cOH3+pdl|36KBrsf?lIOPRo7hy0DjP_9soK$9~Xb zq#3M}rf&pweI4*FZ#;|W!24O$iO1Dk*69ARQ%sMo=^2rqJ(vv0 zeUaKy_Uesncri*(PT@Fo8fJNM@ew|Sx1n|1Ge3^f@tBpiJ29oux+8 zdo_?mWqZIr6vt>CInPG#vGISVeRz%~giIZ47JL-1_^V;l9jzBas zFS+cIynBdRIIQXp9izB+jMaLyDAOuhEJ))#O1daUr$>ORR6K*#%l)jI9$&Egr#BkR zs4rQ-c~~AaeaS( zedq9}Rco_I&ZLMRR$8WNeOPyIFus84V-K&k_BUQYll5oYFSieWLJQaP?Zcg|g9Gz? zci*g=z4iUW?Tyzj*Z0ld>;1jmgRQnP2T8)k`EJhwG2`k8>pPF*w3q9`e}awm!#pK* z6vlc|y{juXk8rhzRhuMI&;fFl6aI~x&X=jQt=Qb{Y}0HHI_;);@SQnKs2ufPk181+ zOt6D{_ZFMxS(cCS%~$KjEG{o$eXe_pkBoVJuucZwA9!b1%wqey_A)-%L}_CMEz ze@;LG)sx7(&W9N_7s&l!me~Q#@!7$qdDLEPZM@#!-`Y7OMb$ypi$~CCX%XB1$ki5) zQi_B<27-*b+a@ANM|0yQABND7Roxhfd@?|jGdo1>9*_bNyE>kXQB77)+9z!U+w<24 zTL9w068wJa z|LyJWzQji;#l!yZ?!4UI*)kswVDT&W+TT6;9tKUiD@%XA_gAxyh1}%nMm$_uyn`1a z38-Y)Jy@&HZ*J{v?Q8-n%&$GU-<*HF^Vgl-A9vu(z2^MxU*X@y=DZZUfG)ueH4)fNa?t36-ee))QTtELXBxRf=jn<;Mz9Q2uV2+LWkX-7biy6zj5 zr*mIB@z8x?=)3RT{)yKQm4T0p4RgfrrLZmc6-%do25Bnze zY|cc?aQ(;-xmEzny3W5PX5N2i-d3w)mWk4kkCJ@S8-HVVb`Q5!0s?|bVskjsCZ7z4=mU== z7x;lDojA#ds_J*F?uaU~GdN)<3Hd@cu7C{kA(PH~zKeiu1Ns5fCb64|57x|MT$wt|^mwMHF zyWkSnTfhAJ9BYl$5gK<+k+3P4N^%f%=0))UkoG52TQjQK3{ z0l=?#pjV_{{Emm3M#V2LXxJ4UUxHW3=dzBACwG?H!ub)5*!RdoLqLRKMmHerdW{l8b{XjwC#mxMT6SXxYJ}m`%p`Rg20z z8elu-8`s)BgjMwdT50W`+gRHXIWLt< zT4~+8f2B}Vr<*S}+}!$pe|>Xnvu4Baw5gEOT1mNI4b)4!By`mF_#M2{T|NxDjp%xt z5C;NBr5B&z4ULfKjIB+>!a=UHba0mS&d41WxJub{CDnM`xKh+~3MTz1nYt#Gva5AB z%IZHE@w$yHgN95|t_oogCgb6R>OvB=VkjNwXa*cu)?)@?vXtpfF0^TZtYUUU*0jci zd$ddhiCm==tvIy8Yh+5T7Ok;}2HaVHwFP7lXN3RmZ>?{_KR?3$;Ma}69=v|FQ0ikT z=)(X#UASr>`y8`?ENgbd+fwu!Jt1R+))3gQI3@o_ge6w}*7};a9EfsER+C$IYO3Td zW;}bL!!Y;63_>gc{TH3#e?9>DW9#7UN=pLbEeLO2`h$FYa7p95x%NQpEIy8r{x3+| zZ|iRkzHKbbn90iAM`&|-o%xKCA;=K`oX?sQrr_>W%n4Ycr{q@fJ55V*)-M^*Wu_OEJ9Rt`7l!vp%!(gF2uuP>lKfn$HAv;gfQJ1TS z>G0m(x_UG2>2leD>^2fh(fso59||B!K8)W#Qkqq1&d!(GgY2u7g33KEPw8=4q2=J$ zSe^B5-GWvxW8~HK83Q|bw7mm@X06e%o1LQ}9P1%pkf=wP8l0wkmy8~E&q54+j4DaQ zE&Q@VUvR8820Mu}s}fx5?$PY->QlRG_}wLytt;DUvw&Sdw;o(ARn*NuF8(Hprm$#t za*!qAKP*SWAE+z9f04sP_U%sY7&D*6)d)c3NG$`5GhMYq=~a$jJ;O|qb!$rPMKv%7 zI?LZija#8Ey&2~J?l4Z3O_6w{#%7b16jC&E_ydY$stKf;@DvI=nhX})&(r~VVwqt| z$}x7}Cs7iawAl0~RV%A z%FkGQRTZ(-rSlZHfB;+rQ~Gji#dEcSf5K-Gr=mj*fk8s};L2F&V=pwmL1i7%a7ikf z6xzzK%N5-=NlknX6F+BFF#ax+0?IqW3l%-;$?FZ>*3_N|5?_ZrQO^@{ki%#l3S+?z zR$&j=k*xT-OvQdke~k`KM_KoR38(tplLAy`WF%~1#E~^rYaD}g2!1T_C|r?1dMYY> ztd-HHY=Wnr!LTcB(U2HUfhQG5zoisVtK;6bzp-$fw;68{iEz$>28d4SxUFcGV!dya z*xVRZwY^%v(IO=&B(_qS^-7=|EvxJQ@}Yp9T@0I#toBJtfH65KliQSRu#L zUjvS7Sf|$;9$0bc@TW-qziivp;1RU2`a9dhnwgtZSQikHq7iTdwBQjRUPQ*@$q0ee zc3ZM6TS80Tdr9dU^G+YI*Nu+KI`j-%)rt$_@}sTY8%4Xt{+it`?$>i=+dOICjS5 zqosv0oG?yDJEDa5>EOLXl+_}_Uos(g;oR4vETxPDCJ}FfIw; z+R0p?Ay1NTm6q8=h?r4!0(4m^1Y2iE8uyN&jY%&d_Yufjfno*uYB3h+uO2e#Bj1}`;azuK z=(T#S%o_>-?MFLw%Q1xyYe407bWBI!+^r)rd=ePpzh{m1U4SJi5sF>7JyI@DVeJrW zp@>n)^CQDJh9xCqA7fj=9X@@q>j0z(2QG~W`y#`_lD?S&82vbQq4WS)u&I8$97QK9bIGmW9s-rSHi7((UMyFaau)+Z0UpN9smb>3{YRLC9Wc;%+y^ zCB!|rF1d^YLbW>?G3O_61P+pV(~SFmZwG0A!jZk2{%C>tb}{;f)A`3F1%eoU4&eNZ zm~alVL5mIJY|NxKex@@E3Dnu}5_pPm7X@kiDJ2v`Qv6C=VkM`~eu77q(4Xz3HU72M zBnC>oQ>iA!I8~4HcpP--wjt=y&f#g_b?4T~r!xWkpJ;CJ9&$BJm=Kw9~7V*N`8lRjq7#uh}fs=FuD1aW~V)@R`8pZ=Xb zyzbUg`+j@r&kNq}eZ?^r;q&m}efk$Z#eeTFJ$mq`rF+Yd7MC6@KYD=A;g6+9e_H&C zbC~*>VCE7S5}hb!y?52`#l`)Te6GjdMB9#jJ1{3GuvgGXmLUu@?Mf?X>!An6kH9`3!Ss7Ax14jj2Vf$P0P&_ zX`}=iBn*=Wdy3?gK-n~2c8`7rw-8mmlbisW^}rRWJ11ttadn)i%fz zP$~DNWy6&ti8Wa=QnJlmitHtcaRCyX9Nm%zYtR_jNofGz5Q}t$pR2LHSNlPy9&Zu7mBt2uO5dxz7e$u2I5S97H z+Nk`gc%_}wb|%2KMq{I6+qRu_Y}@MCwr$(CojbDsOi<0xwy=5 z3y>VbbH9_AQg|>33;7vFDhYh02oavcSvCL39Won)#e|`x$eDAqR}nvK3vrYN?tsJd zjfDdbVjspksPSR@u8##o0{eKRI#+4_=t*4Ya6AuP5)^y4Y|>=Bau%O>_ss+(YSqPs z-2_(UrpQ@%gBe(f&(8wy2gn*51%mH=65ve$o)9*A!mhu(qP`2dFt!gi+(KbYEy(%# zNZIl7h^uiTDn2*wd3NffY}alA4eyS;4hu;lI=|yL@Q}tGYUu=&KLxKbEQ}S=7cnFA z$9`ugW%lcEM~iIj1q(aeqK!+S(0ay?>ja;gAEZ{wlbK-(7r|>|+qbGHMGKeCp_%<= z(m8B4w5pyMjz{JN4AwA{~9;f?xn~i$LiV5NA zDi>T?z5dZ87~=^i+8HUK0BvW1#esxn=-W>X^zZ$epPHR6%lug-wbPgU)gJic*WTW( zarCo)^TF}?ud%!XpXOUL@645I84ugaZGL*&+Qyb3a}Z+_^5pG#lCG(v2hig1dSBL> z@z&2z!@oITRq|#Lf5a@awpQKEQ1%q@jq%+g!Pi3<&fJ@SCGR7Z2e@DH4OjA{ zC|;0!fuV%(`G)wwPuUNRhCh94(V@_Hs0Jxh&{(kI8j-1Z z)s&-Z-a+u!;de+#u2Rr7k43&j3^b!a_?C?;7#;T~g`(F&@u2lVy$_g2EyY=FmyMwA z!EVu5guy#TcC6DT9S;jkPPiep9daNT3oA3eXTH3bwg4awm_}dy3BFTBar~QfroS`< zVXBWeNJC>S9?sfT?j71h<5#L4;%2Ms3v?ks>-P#dPKqBf%F+NfD1 z(}fAJCK@CSG#DN$X(hE{&XLH-w?*XkvR&984EMKu>|JvP(fwE4os;+O;Mn2&`}rOo z_Z}$bjv~|nHU#8gwqNqG(M$Q4@kn_+8VIG4h6J{i5uWjgNTrShmYCywr~ICfvv&Fy z_jTbWN6Qmq=c1(OdTa)pGhAepvWF zAzaaB$eMn3f^LVcb<}&y0FRU@*jdZ)z0MiNn$nqOEV}J`hH}P`YSgmYRXes|E|QSO zXvtY?C74!yY=InB9(16Aq5XjWW}pY%R_C!74Pq~g3$+igOd}E3GaOSXZR%>dC2NP~ zo9(miR`vAi$0pA%3n4HEPPDIzI4;+z*L@D8@2#%$rX{r&KQ$!`9vKDI2gAfw2N-9m zysnOprq0kPW3;B_Z{Gw@tzP}R+&i7!`l?hB`<{XdjW&+^zE6s5AdYz^|_=`V#9`8K*aEH=~~59=_=$BvGrx zS)7dO7LbU7Xn2C9GHkSFWR>=hRaXbvfP>T6km-0weKrl}O=PBSTa6^C$2c((y2Ipt ziGM=RrYKkcoD9^k#rh@8riPb|1Qgr5oytHus2)`V|4~ugSSex;RH&m#N30Eu)FR^R zc8d2IP`90_RNCtQCUPk!hW*ta+y0+!Feq4_KJoPNhk=XRvMeZd12;@@nYI z60!MD~bm8cv8~-=BlaQ|1EIDCtO&oXGqj&u9t+_Mf`OD!vIG+~B zV0D@+z1B8T!?0?spmC(_>71uSLBZSc{7aEF^U-<@VvQM_K4<lpOogEEZ;9WNfi!$FkvzDjHRO`@d{-997T z=W|AUJyZIt;#hD6BH1?Y6B+pIn+dprtjq)CX2~k^d;Y)5%Xv?L$zUl~uBe-TmIhcm z1dj9h#yyzJVQ$}YQ!_5p=Qhv1lQRXDvruL1wSPBQ8cYYfXd-D$^{Be76swE!Mlv2O z8rK+&eHl{YP4S8aIvLO2;W}~QYU2YjaU2B$B+B+%>xoTZ1;-p6S7Ej{T~^PH{mZg6 z&3gkVv}Uhi2jM1M5l^3!0bY(@Ul2*Ke6q6~i!vyZoJh&J1H7+1q@cTJ&)LkV+s)C9 z^x_6TuG&*awa}%9{Z}idS5tb9p$b-8Ea3r$2aOS0wB9yn^r;yl=4~ zA}-F?{C}X;GkxqzT)@mLVqqR)`)S)eQGK@+p|>H!{7P7{lYfivN;jRpLY%ZCXS6-} zYBv8i(jo;wpaBcLO@W=GPJ%dS9nWa6ZT{MTUb;DJzs?HefD!&RQNDLMKMuc zoq*WCU{|ucF(`zUf|v~wTP4joaJ|iJZ>@tV2`+HtI{*G{kDvPs-u{R3jv7#>scYM` zd*$BP+3@kKS(6A|54t{)W*qzvY^o`TtGPu&Y1+&9)q2Y(eq^9@9`3qNJX-Srj4NsM zMkOFmG+~NLbG51B$?D@5B|8!9hP_?>88Z_$7&Sw65`Ae(F|X_7&dqe0gyLj3PRl^g z$DD<=m0~e^We(Jhdp$Th)yMnSwSI>Ao=Kk3g4-db=D6H+rAv2G`c^2|)=#E$>+eCD za~Ie)%nj6wl^-SE_i1rh?(e1jeJXp14v%GahQs|iUi=rY+ndEC{_r^=yO)Wc7Di|@ zy9HU8*RTnywWLpVxEI~!KR7C{NUz~d?#aZv{TuVXaAoYHyE!)uOnc&ey6LCE@{b4n zGAE*Nq@VP2Gfbb3CaKr^-S~Q5dT@(2u1icGUUgIduvA32jTqlO{VV18AH3s_NaK9o z&$Ia;i~FC0FY$js>Ce|NB**=KZw+75AZCL+kq^qP%Z+I_yAPttP<`Ql5Rh?cMmRAWqYISMnEzP=gDVPWFtlRk&jcBUF2ExSw#sp5-k~Q=j_S2= z2D$}IPLjBQxn`!klCsORZcpc+wZa6KiiXo~ZXXx~;bBknxQgp|N2VSC&E7ZtgCD~W zem~zuSr0#6-g}?&_ek?VONg4-`4uPd9`(pse;s$bqVE@#24+vo(OJx2HljAVBTikD zfzSeif=4rw>C0cVG&MK{1nlzQ$p2b9Hxk~Q!cq+C!E*n}1IY?x$CxVF9r!$m+bG{D z@;*bh7R=oik0>zFE3QIXb#lxb`_6S{Jf3=?diyl{7ijOSA@ia&(HT;t=h_UEjl!$@ z%Mg^Ze`_Lgg3`(?H}2Pn32`{1u8(=QlwVaqG{6RAUR!e=OpgKUF6k!8fgQK5y8HLR zbKJvw?{YoulT0!ZGIAhe3IgA_(!3*03bx1Hn%Ny#>+zBNjvGF`4-n7 zL?KyX;vFYbW^oBZQ+lDc^|iFJUnHg*W$b|D@q6T~pUtxX)=Q*Q`11g!HpX(gvi?Sk zljxc_n^(ZNeGPy_ZdKuJ=7rjLc4*VyuTo@_{U55HS#KZae0K1Uz%@eW?jHGDVj zCd08QZ>e(YlkSp65zgl_6)cBSE*Kxv{V?wrAI&DAkvF4nbTHsiBfix^Mwfi5JO&A0CeIRvwWqJoqc!)iOLV+e1RjGgd-f7bV5%jFktPQNhzi_ zGwA5Ujr;c>+)(u_e!cVIa84Jt7t|e!D%FUmpU#g*^C+ZfYzfY+6Q54+qV23MX?N?OSYQ@cX zd&pti`w6$3ukNp${Ks38#wU8&kJ5@i+wx7+PQ#s40KtZ))1v`iK!BzdD_0VQX$#da zZIruw5@Am)#N3b|()L#ElAtsNwD;$Y&yphzzn6JZ9=&sXeZXMhQmATE?;a!e=H{3l z1&bMsUQ;Qs)ln!pBrm1OoQX`^WUx0tC;vbeb2Ywj%_g^&`7;`RgPnO9V|tkYa% zuIaFdruCKQml`469|uFCE|_`Ys%AAxEPFo?X zLj<*3q+10Ki@_|SlO@t@9!IFk7FqjPFe4;+55rf$$oWGeY+TgiJUVUbXmQLXrbnGq z1NH*bP8Y7xEv9pH%y=@@Iev2%EM?+sxGU)pP<|<^DKUgI(5t{@xDp>;u5Dt80rSDW z8}KS^c_q3t-s0aIS4UD7n&UH@lZ^#LJ?6I7p1)%1LtV!jOd4};{c zk`t=!<f=&I3QF!0rl&l@k6t4+NBe4EAP03DeKv!seUVI*Rp;F|mI6sY zd8gPcaW0)ITz~XiyEwFAlf`;;-qTsc2ZBFow!XJ(eOCdKe|Pu0S`Wq^ZJRPPAU37i#38b^vN9DoAdpa#f3o7id^R#EJ?K&s8ge-+Hz1CW zPk6lIbL!mF($iFx$OEg@Ri6bU<*-RPPjsr%!&C3gk`qB3Z7QQoJ~0R$6A4_VU-j!+ zD7RfRwC)XgBSl$Eq87&O4zKtQauh%lwnoOEa`k>>CEU&}tLKG= zq}K3&n0kt>%=auFTaCIl4$x11x^x-Et2>u5ZX#YsT(Rj4%_Wf~Ihrl6wtX9JGyGVi zt*8%Iy7PYpOxNqysQQ1Y1QM>gDx~tw(*1qddSD{t9T;&Nz*UKk8_tP5xKoEA?+Iqz*4k%W`NwsRu?BX= zbyGkMe8bb=3jZm~sQU`^NUQO0O0n94RZi93V`Ajd(N~p;P+=vJx)`zM0Jj4c-l;|l7}*fiI^LL>oh<}cAZ_6Sz{y;HW$OBY z68zi1Bh4vVRfuAw2YpsAXk{25)jMcz&!#~0$p>5MvNcO9$Aa>Qg%dd)SQ?L%-qQ5a z5HN~rh*{Zqyc=3bY_vDEJTv3eoN10e9&a@qRT#2hu}j|(=$HdT4svXabIS^&Pdd1J zGUWTEcuVUqOF-1~*tiMw`RkhzBWUp?cdXK!^G+HNJ(Vj>JbPN{PN_f#fYPOcr4 zRJ%=dK3RKwEqS!{I(@s2*AsD6g%o%<)XR;yI` zNkZeUrJC@KSpwEJcNIcHy zG*VW9a9D*_$#~tga_I|$KVKtpulu4D0h*WF(SL00Q(jnfTr7l^c_||eq^FU9yfd8~ zBqNvHVxEymnE1_pi1N2jgb-F0Kgo-8zQ!ayfwuOxhC7!G^lwc?Hc7VU@ndyA znvsV#L%xPps%i>-VE@zP{C#foO0GJa(~?dagzpeO)>9#8)RyTMySYaFNf-L%%Cre1 z%)Gz)4gwGY?q92wU`tlJnX4V_R*p7WCs#E3ncb`UQeA*-_aE}5F3N0eguZ()@^&+Q z<86=*MH**-v)Q7M`ETGQ!W7vKV?`ANR^(x6w_&E$Jh!U*%ce6zP^SwvQ4G~`)C?cx zZR();QzbJYr+Dzb13t0=0QT+G_edvfF>SgBaRh3$n9BvGQ~0K5dFsKs3hRYcP(m9o zV+ic0q4>Zyd=G>1UEgNriBnbNC7C=*Gc zi+E0)v&CB;V)dx~Ct{<7zjRUW!7n>9X4(RGjS7AYmwT7GoE1)15DmT?JCgE_-n!l> z%U3_}13cQbo9j+>gnzfTwpj2Bb%2*MziDP;9u&?MiAhB~R>c@dQc0H}j0&b|qC+GS zC_Z27Z9Bf?352kX)w}WZg0_qMyY;z$cQGXn?`9NG48$8#eor=0_0rlW(#ZhX{_luY-jt%b$P>C*;wA$QZ|Q(5>yv->y7F7c6J;R{6qhH`=ZQ&$7^s*u{q zl&Vy#hZOS}BOb*%>F7}GR0eB_KNz-%9FSK-Xomi}h80 zMERB<>7}nw;cs@u@AlL&FytSw$u~cxl|j1-PJ%P=*>e0m6D(kf=Ki;}I;SOKPc8XE zBWsi2M#@{hyvfvr_a8^Z9BzAK3SJm3?vg@f$KO?Hc#i~q`D=jJCdHq5G6@X{9G~-w ze&2pWmAWSuiakiV<1FpjReb=jcU!}UrJdiU;Ng}Rz<`yg*FoIRqwU`V3D~m-cZ01TSTiAC;)Y*cxS*xGVX{A>r1W0MRc(2p<$43 zLC>H22#Zd`eR*z@m_CQkC!p)$E0cI7Ej)~C3Y9!@8_Tn9#&1<+^45b1(T2%eyb#>b zk6G~1Hxjpg#o}~Jl1?@XJ4h~|6%)&<{+loQ&mnE-sYIU{oCvr~pWvU8bcH&uzmxz> z)|Ysqvq2*uH`2y!9NirF2q+Nr4)!h-vnpZQbwew6wa9wXxDMfXvfhb+REZ(BTB~GW zP$XkgI+1Ko%_k9RK}V#hRYJzzssy7&cySLU8h}f5U?@T7AZ`3M2V<&2v`edel9ml73l zREUiH`~zZlzIC4yC{KtWr?B3i{ZBfZX6u?dap*ff&S1X>1>psHnahny#JjfTU4xy0 zAoZy%KdPxsqAdd68^V!Z%lAiN!{>)%`>Lb8a#U3fl&It;`w{8#l}*G>aAf_`?RhH$>ffw~Vg7?Or`|jP z-H-g6Mwtm6D>PCD)!&RRwX8o54VgcoO-NGtHAEB4he;D(jmIZ8s_D< z0WC8>zjc)Jhnq;Pf2gkwSQefY-xtW^f`Td&)*_V-TGYOaTKnb1(&W>zjUbdpUDtW= zA9liHwk$*O0xlojsH**Ac4g8xEqewg$wh`>YV>nYdxqyHQF{P+Nb+jwKu>oqm3>|6 zQ#IxQu5ys&tM&TFK%fv9BWd6nB`zSKiD+$+l=2_&TjaGv{6{36YAZvn^{PyLUV{ zoRh*;R_HyC2zgx*S35{Dx7bE?Wn&JhJZ?>eTLFF|cI?XO} zp}pSE*o$pJ=Js0I`)|8BNl-mA_{W7AjAEU7_K+e1LJh;Si=C=h!!>K!N;HdPe$&Vq zMwqqR)`@Zy^Keb^bLET-yMOd@qR=F_(Ebq@{j$vglv1=cLX1UDDVcfdQC0PIUuXn` z$5R|p#71QNa@{IB6@mbXx#1*78cS4(d}wW5VOcspwi?0XlRRvGX*^d_K->K$jd041 zo_wR5l2{8>CLk}m6J=>`^s}vP6`-Bpn5xz} zm@~wXbd`jFJ{5RitWB@FwkpY|;p `eY~bgi^vmTxo4Des{JG?Scyn;d2qyT3%f z9mvgRglC6jMh{7@PD%_oXDcodpN*dqodHnIa-qyBt9tj28p0}nAdyg^6)!_LMwuNw zsdxazI9~9Y6ah+FLY>8D>fHtRMD_(wAslak$c)V^+K8aOH>byKGmo|Oz=0tFNd2^3_g`Jl0+})BJH-`5ym6N;Jbz0LwURbz_q?tJ;1}5+ zPCm>ZdRfFfUS==SKfI6MWlEc`A#6I<^*b_x&>1%I))`qML1dhwsQSF=19Y*4Ua?|i z*N?9bb-dpk^cH)k_4)O@lzfjV-N6~XGC{LCR3Ltz$eY@WBWv5$$xX~2DV1_&y=9yH z=^(|S*YZgyB7?uC0%y=s=^Sd4u>+xnH}e_Ux*za*Jfz=Y(aMHcL2emPnb zB;r)x)a!O@^kN?Z3H4y=%N1z4{*g@+m_zY^Ny=SnGM#;c;QE#|D^xpigd{}nFsY~y z2@7qZ;ls7}J7Fs!E+;2Fz6pk^E-od6Rx~dnh4jj(Z#&Gl(ddhg@!N(OKk)JGFzQKU zL$fE9&HVHk_|jaK`Ak8q73qV4WHVUUnqxV?c=Li>!Wl>nF>OmR8exo8jki7)9krMl zi=vi74og6WxtheuH#5U_LHeJUo5C&;QRQrrU@%lXfhTNo;@Uj@l+b7f=G47s7i#7z z3=O2thFxRA?R}p5VvpxY+x0f;^Ila3VDlT@j*<%rgBdeEKn~s! z)z*S8HQ76kIC95I#+BWOkJQWz3DL=Mfkz4w?5SHa;@8JrNXvJ}cRi2VSr>3eG*ICV zBOdt)?J#CltH-xz8Qxn0Y$`C7ry45SN3QJ)dzSPADirsn{U$Q1nRg!AW|D26uY z6oY?pHG91*PCrwI*aT=-J^E~hr_4kN=gC)-({Vxu1Ay()T}9z5vStMfmBag)UPb;T z^Gx^p@S6j+F*j+`YGd)*;x5H@5*Svqz`@X`;>+^1Q1^+@r+}AWueF-G z>{%wO88#iXwoK2p-Szvu?u~P`T04qnJ8FQoDFJk$o#-#$i42|I$P$xQ$*hl&Ix1Ih z>Xz)wR`$sjS~RQGCgpNN?0jL^qhG9%Jf~+C5XdE#!&@q~l!Oz>l+#6IY&-;O9;qqO zE+IKjA&1r6KUV19;+~!_CjgV--rr#i#|S%%`ZTnDRi?m>IJt=jn|L^YBx>bf_1M@oKiVW-sPHJL6Qj5bb_ z4p>z^+luU$`&C&+3y9R);TWeVL>fggfD{tIfzYgvgIN}t1^F6ov2} z#ls}XE}MFlB*8LVEaV{)(g z8BK#2s?mV$Mm*Bx zr(%WOTDC9k0+Axw>|cBaL#NrC^rJnTFe-dvJOD7 zp#I8bNhDAQ+uQ@sI7^9+jM|qPc+lr!5M}}s*fR%QN1^B5odseBZ61M>kMLgmrknPF z3%L#*tE$gL8i)d-+!(+0wqe2}>fMCi`$tQ^8SQnTw)*hw|H}kXv;VVW_>09&4}R3; zDw+u|GH)kK3qAoW-F<*@vF3vr`D8T$+kKzZ@EQC)W!U|cg-7~4VhDA5RS3vi-+mWk zJeFiUf?a7gZvFQ9*t54fowKebs3ow*+n{7_mo;;f=0eD?iutN}8my6hxT46+UBSH< zck%ct&FcAVmRI-6)l16z5y8vzg%v2NXoM)KIU_z$2xR`0CO|8pY}#H~X&xgL$x4~IOV@*FIqwU?g3x}OTjv-egX?zTuo2|DEctFMq4 zTXYq4b;?xtG`$!CQaH#^72W51Ch%!4I`DHW`e`FdNOteP*f@|At`wzvS91%AOp@Dr z^X?e@!jtJ^zfv7#D51!i9QN{jPVDrZEwSok4g5^Te-SVQ`mgtdlIsf9JiS!vMgwT4 zKVnI}8PZ)0edKFDFn&6fqYj;+EM!Af1>BxhxJbFXw1+QuctdIT z!s|g9?pg2)Pb4Ki2EGCx*-K^Yat(=iR>ogjaAF!-7t<+@&U;%~7Tbfaud?qviG&Dv zNP01}pqIP}4WqfLa&=6c`XQ2KETFP*?^$QPumYY>`*qqWD{B+1*4oq*Wk*_!h6WNy z8CiofIQFz(V5P!>gtIaEtj@45@ot#W;@b>-2LNL6Q`FN{()ZDV3DK(>s@oLYf=v72 zKcKyc&il>QvJI+UU0>ZLW3MIY_Lk9heNM@*Zax+1c-^LHU;K{N*Ov`9nZb6zy>9{vePEi&b>|U{M>aQ1|5vF<70v=0>o$E?O z9HE4@{Fi&lQm*o#InE?BrG~R)YZ<~tGbiwBO@_H%x|1w@ie!A`q!Rc#+w6-dPKKF- zDD)TpheGb0>O$`a`DmTpeb*Dmc2~^vc1TAF_@-K7hnI~i(zH$9%BCBR-N+i~4&H`} z3YfEf)O!FM_73%IFPuqd006{#W0-bMlNVZWFIVf=MUg(F*xzGI1#a27%$!(AC3qTC zQ1H?Y#RY23Bv?H&EX>)}$Bm8~~ ztb_E2c4~1QQXZ$Ldp;$f%S^`Gimtp+Y8kOp34nEz@*bU6Lfk;-NF(sdahkg-HH4gM zWDm1^IwFR9OLN?#>_x+#l!T2;PO}SL@?9l!T{~iyfvPA2Oa{p)>@TjTvWsb@3FFWv zP^8pRYMF5YK&xd-tkQBoCH@~%^!^+2m}#__;%Ydux}iZw)Kf|I6alS|@JUh+%aw_H zQCm6Hl!(tsZMjkxV*7??)jPYhFM+kB9>6tPjs3&K{M`GWB%1>aH}Yz4jdXoYemJn9 zi8i)k3=19#QGgQrQS=h-bR}MGI{p}JHu{>`5tzX5nrV*D*s$HQ~! z0aa778*_BWX@+r1<93AEz50AR0Q>Rsg!B*ih*aN96?|}&EahTyRK!oC~Dup;;Av#0s;Uza>_cq(GNz@vsiL z4!n+kBb`_NHMPR#gq$OsP}Rs@pqAoQcEX$3j$?wNjl6F}0JZx^HQ{y8QgqLyI5f-X z`A=OW@$>k+PjUBX8=iDrzWa-0X+#=_P}#32yet3&6sqO!IhMJZ1#+D<5;!2 z??|SqB-Y1j*6l^V#_0#=p4Kp`bfr&O3|_4Mcz_cVnoH@9Zb&9BT@PSk?dbD{qI}6A zdy^LZ3NsbH%lKo+sLvuh6ngH1y}n#yT%3=Pg%O#jST#rG`j-jvsrzT?$NApkb12>1 za7~%WTe_>XR=924%B{qu;j`Ad;|>PLr%;QsuAS>&jUzs;0U1|;m0B8l3zNKH)+Uaf zQjtGPbAVg90qH{(F{`@VzFwJx?}q#K{+|WB<#dw)&k{d;1!P%c>=KbeoYBzvT2L-p z_eq7WHYXq3#Kz8T_uv@fCzpmonyJ0qBu@OWndKb#Z;7+XLT?fRu(*KB3VZ8|p7mdx zCJdB~arI`Lei@V3d<@%&{)`O_V&9qsG<#Oa~qZOhx1 apCd18cKY=HGgSBf765;sEdZJW0s0Tr??sRR literal 0 HcmV?d00001 diff --git a/sanoid.spec b/packages/rhel/sanoid.spec similarity index 92% rename from sanoid.spec rename to packages/rhel/sanoid.spec index c0f33ed..ab299a5 100644 --- a/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -1,4 +1,4 @@ -%global version 1.4.14 +%global version 1.4.18 %global git_tag v%{version} # Enable with systemctl "enable sanoid.timer" @@ -6,13 +6,13 @@ Name: sanoid Version: %{version} -Release: 2%{?dist} +Release: 1%{?dist} BuildArch: noarch Summary: A policy-driven snapshot management tool for ZFS file systems Group: Applications/System License: GPLv3 URL: https://github.com/jimsalterjrs/sanoid -Source0: https://github.com/jimsalterjrs/%{name}/archive/%{git_tag}/%{name}-%{version}.tar.gz +Source0: https://github.com/jimsalterjrs/%{name}/archive/%{git_tag}/%{name}-%{version}.tar.gz Requires: perl, mbuffer, lzop, pv %if 0%{?_with_systemd} @@ -110,6 +110,8 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name} %endif %changelog +* Sat Apr 28 2018 Dominic Robinson - 1.4.18-1 +- Bump to 1.4.18 * Thu Aug 31 2017 Dominic Robinson - 1.4.14-2 - Add systemd timers * Wed Aug 30 2017 Dominic Robinson - 1.4.14-1 @@ -121,6 +123,5 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name} - Version bump - Clean up variables and macros - Compatible with both Fedora and Red Hat - * Sat Feb 13 2016 Thomas M. Lapp - 1.4.4-1 - Initial RPM Package diff --git a/packages/rhel/sources b/packages/rhel/sources new file mode 100644 index 0000000..d6068d4 --- /dev/null +++ b/packages/rhel/sources @@ -0,0 +1 @@ +cf0ec23c310d2f9416ebabe48f5edb73 sanoid-1.4.18.tar.gz From faf825003fbb7448be5a566fe1c238711a031298 Mon Sep 17 00:00:00 2001 From: Eric Coutu Date: Sun, 29 Apr 2018 15:06:31 -0400 Subject: [PATCH 44/97] Document compatibility with (t)csh shells Also, recommend installing mbuffer on FreeBSD systems when remote user is using bourne compatible shell. --- FREEBSD.readme | 11 +++++++++++ INSTALL | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/FREEBSD.readme b/FREEBSD.readme index 9960201..732940d 100644 --- a/FREEBSD.readme +++ b/FREEBSD.readme @@ -11,3 +11,14 @@ If you don't want to have to change the shebangs, your other option is to drop a root@bsd:~# ln -s /usr/local/bin/perl /usr/bin/perl After putting this symlink in place, ANY perl script shebanged for Linux will work on your system too. + +Syncoid assumes a bourne style shell on remote hosts. Using (t)csh (the default for root under FreeBSD) +has some known issues: + +* If mbuffer is present, syncoid will fail with an "Ambiguous output redirect." error. So if you: + root@bsd:~# ln -s /usr/local/bin/mbuffer /usr/bin/mbuffer + make sure the remote user is using an sh compatible shell. + +To change to a compatible shell, use the chsh command: + +root@bsd:~# chsh -s /bin/sh diff --git a/INSTALL b/INSTALL index 1492546..33b510d 100644 --- a/INSTALL +++ b/INSTALL @@ -9,7 +9,7 @@ is not available on either end of the transport. On Ubuntu: apt install pv lzop mbuffer On CentOS: yum install lzo pv mbuffer lzop -On FreeBSD: pkg install pv lzop +On FreeBSD: pkg install pv mbuffer lzop FreeBSD notes: FreeBSD may place pv and lzop in somewhere other than /usr/bin ; syncoid currently does not check path. @@ -19,6 +19,8 @@ FreeBSD notes: FreeBSD may place pv and lzop in somewhere other than or similar, as appropriate, to create links in /usr/bin to wherever the utilities actually are on your system. + See note about mbuffer in FREEBSD.readme + SANOID ------ From 63979973b2ed5da4c7c1884c43f428041cdcde84 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 2 May 2018 00:15:14 +0200 Subject: [PATCH 45/97] if syncoid is instructed to skip snapshot creation it shouldn't attempt to prune existing ones --- syncoid | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index e63b68c..999ee79 100755 --- a/syncoid +++ b/syncoid @@ -387,9 +387,11 @@ sub syncdataset { } } - # prune obsolete sync snaps on source and target. - pruneoldsyncsnaps($sourcehost,$sourcefs,$newsyncsnap,$sourceisroot,keys %{ $snaps{'source'}}); - pruneoldsyncsnaps($targethost,$targetfs,$newsyncsnap,$targetisroot,keys %{ $snaps{'target'}}); + if (!defined $args{'no-sync-snap'}) { + # prune obsolete sync snaps on source and target (only if this run created ones). + pruneoldsyncsnaps($sourcehost,$sourcefs,$newsyncsnap,$sourceisroot,keys %{ $snaps{'source'}}); + pruneoldsyncsnaps($targethost,$targetfs,$newsyncsnap,$targetisroot,keys %{ $snaps{'target'}}); + } } # end syncdataset() From bb654c1cf6d59f06727be8b93c66bbdd2a42b9d1 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 5 Jun 2018 11:41:48 +0200 Subject: [PATCH 46/97] use utc for timestamps as default --- README.md | 4 +++- packages/debian/sanoid.service | 1 + packages/rhel/sanoid.spec | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cc75bdf..be6195d 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,11 @@ More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: ``` -* * * * * /usr/local/bin/sanoid --cron +* * * * * TZ=UTC /usr/local/bin/sanoid --cron ``` +`Note`: Using UTC as timezone is recommend to prevent problems with daylight saving times + And its /etc/sanoid/sanoid.conf might look something like this: ``` diff --git a/packages/debian/sanoid.service b/packages/debian/sanoid.service index b54c586..2d01bbf 100644 --- a/packages/debian/sanoid.service +++ b/packages/debian/sanoid.service @@ -5,5 +5,6 @@ After=zfs.target ConditionFileNotEmpty=/etc/sanoid/sanoid.conf [Service] +Environment=TZ=UTC Type=oneshot ExecStart=/usr/sbin/sanoid --cron diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index ab299a5..3a9412f 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -58,6 +58,7 @@ Requires=zfs.target After=zfs.target [Service] +Environment=TZ=UTC Type=oneshot ExecStart=%{_sbindir}/sanoid --cron EOF From 52661afb4125ba350ce0ec54e3928a4ab2140f33 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 5 Jun 2018 12:45:13 +0200 Subject: [PATCH 47/97] updated debian changelog --- packages/debian/changelog | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/debian/changelog b/packages/debian/changelog index ab530b0..2bcf423 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -1,3 +1,18 @@ +sanoid (1.4.18) unstable; urgency=medium + + implemented special character handling and support of ZFS resume/receive tokens by default in syncoid, + thank you @phreaker0! + + -- Jim Salter Wed, 25 Apr 2018 16:24:00 -0400 + +sanoid (1.4.17) unstable; urgency=medium + + changed die to warn when unexpectedly unable to remove a snapshot - this + allows sanoid to continue taking/removing other snapshots not affected by + whatever lock prevented the first from being taken or removed + + -- Jim Salter Wed, 8 Nov 2017 15:25:00 -0400 + sanoid (1.4.16) unstable; urgency=medium * merged @hrast01's extended fix to support -o option1=val,option2=val passthrough to SSH. merged @JakobR's From e65879d1f8c4a1304ee0d16d4393645b5e07de1e Mon Sep 17 00:00:00 2001 From: Lucas Salibian Date: Fri, 8 Jun 2018 11:52:52 -0400 Subject: [PATCH 48/97] Fix --help typo --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 999ee79..a15f872 100755 --- a/syncoid +++ b/syncoid @@ -1148,7 +1148,7 @@ Options: --sshoption|o=OPTION Passes OPTION to ssh for remote usage. Can be specified multiple times --help Prints this helptext - --verbose Prints the version number + --version Prints the version number --debug Prints out a lot of additional information during a syncoid run --monitor-version Currently does nothing --quiet Suppresses non-error output From c0c30500765395cd22f975abb198ce3d4fb1d1f3 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 19 Jun 2018 18:21:06 +0200 Subject: [PATCH 49/97] added option for skipping the parent dataset in recursive replication --- README.md | 4 ++++ syncoid | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cc75bdf..7392fa7 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This will also transfer child datasets. ++ --skip-parent + + This will skip the syncing of the parent dataset. Does nothing without '--recursive' option. + + --compress Currently accepted options: gzip, pigz-fast, pigz-slow, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. diff --git a/syncoid b/syncoid index 999ee79..5cbc1c4 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-resume") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "skip-parent") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -141,6 +141,11 @@ sub getchilddatasets { my @children = ; close FH; + if (defined $args{'skip-parent'}) { + # parent dataset is the first element + shift @children; + } + return @children; } @@ -1137,6 +1142,7 @@ Options: --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none --recursive|r Also transfers child datasets + --skip-parent Skipp the syncing of the parent dataset. Doesg nothing without '--recursive' option. --source-bwlimit= Bandwidth limit on the source transfer --target-bwlimit= Bandwidth limit on the target transfer --no-stream Replicates using newest snapshot instead of intermediates From 70b259ac3cb9ef9167776eac0424d0a78ea0c0c4 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 19 Jun 2018 18:24:34 +0200 Subject: [PATCH 50/97] typos --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 5cbc1c4..8ab3a41 100755 --- a/syncoid +++ b/syncoid @@ -1142,7 +1142,7 @@ Options: --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none --recursive|r Also transfers child datasets - --skip-parent Skipp the syncing of the parent dataset. Doesg nothing without '--recursive' option. + --skip-parent Skips syncing of the parent dataset. Does nothing without '--recursive' option. --source-bwlimit= Bandwidth limit on the source transfer --target-bwlimit= Bandwidth limit on the target transfer --no-stream Replicates using newest snapshot instead of intermediates From 0f8fee7637ec1265bd0a72011a08b0666e6123c4 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 19 Jun 2018 19:43:36 +0200 Subject: [PATCH 51/97] added option for using extra identification in the snapshot name for replicating to multiple targets --- README.md | 4 ++++ syncoid | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cc75bdf..632a0af 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This is the destination dataset. It can be either local or remote. ++ --identifier= + + Adds the given identifier to the snapshot name after "syncoid_" prefix and before the hostname. This enables the use case of reliable replication to multiple targets from the same host. The following chars are allowed: a-z, A-Z, 0-9 and -. + + -r --recursive This will also transfer child datasets. diff --git a/syncoid b/syncoid index 999ee79..dcb9e2f 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-resume") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "identifier=s") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -71,6 +71,17 @@ if (length $args{'sshkey'}) { } my $sshoptions = join " ", map { "-o " . $_ } @{$args{'sshoption'}}; # deref required +my $identifier = ""; +if (length $args{'identifier'}) { + if ($args{'identifier'} !~ /^[a-zA-Z0-9-]+$/) { + # invalid extra identifier + print("CRITICAL: extra identifier contains invalid chars!\n"); + pod2usage(2); + exit 127; + } + $identifier = "$args{'identifier'}_"; +} + # figure out if source and/or target are remote. $sshcmd = "$sshcmd $args{'sshcipher'} $sshoptions $args{'sshport'} $args{'sshkey'}"; if ($debug) { print "DEBUG: SSHCMD: $sshcmd\n"; } @@ -812,7 +823,7 @@ sub pruneoldsyncsnaps { # only prune snaps beginning with syncoid and our own hostname foreach my $snap(@snaps) { - if ($snap =~ /^syncoid_\Q$hostid\E/) { + if ($snap =~ /^syncoid_\Q$identifier$hostid\E/) { # no matter what, we categorically refuse to # prune the new sync snap we created for this run if ($snap ne $newsyncsnap) { @@ -898,7 +909,7 @@ sub newsyncsnap { if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } my $hostid = hostname(); my %date = getdate(); - my $snapname = "syncoid\_$hostid\_$date{'stamp'}"; + my $snapname = "syncoid\_$identifier$hostid\_$date{'stamp'}"; my $snapcmd = "$rhost $mysudocmd $zfscmd snapshot $fsescaped\@$snapname\n"; system($snapcmd) == 0 or die "CRITICAL ERROR: $snapcmd failed: $?"; @@ -1136,6 +1147,7 @@ syncoid - ZFS snapshot replication tool Options: --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none + --identifier=EXTRA Extra identifier which is included in the snapshot name. Can be used for replicating to multiple targets. --recursive|r Also transfers child datasets --source-bwlimit= Bandwidth limit on the source transfer --target-bwlimit= Bandwidth limit on the target transfer From 34b942ea45cbc686c1e0fe16c8dd6c4515115826 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 21 Jun 2018 18:18:28 +0200 Subject: [PATCH 52/97] correctly parse zfs column output (space can be included in the values) --- sanoid | 2 +- syncoid | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sanoid b/sanoid index b6dc9fe..c07e090 100755 --- a/sanoid +++ b/sanoid @@ -521,7 +521,7 @@ sub getsnaps { } foreach my $snap (@rawsnaps) { - my ($fs,$snapname,$snapdate) = ($snap =~ m/(.*)\@(.*ly)\s*creation\s*(\d*)/); + my ($fs,$snapname,$snapdate) = ($snap =~ m/(.*)\@(.*ly)\t*creation\t*(\d*)/); # avoid pissing off use warnings if (defined $snapname) { diff --git a/syncoid b/syncoid index 999ee79..ca7328f 100755 --- a/syncoid +++ b/syncoid @@ -664,7 +664,7 @@ sub getzfsvalue { open FH, "$rhost $mysudocmd $zfscmd get -H $property $fsescaped |"; my $value = ; close FH; - my @values = split(/\s/,$value); + my @values = split(/\t/,$value); $value = $values[2]; return $value; } @@ -985,7 +985,7 @@ sub getsnaps() { if ($line =~ /\Q$fs\E\@.*guid/) { chomp $line; my $guid = $line; - $guid =~ s/^.*\sguid\s*(\d*).*/$1/; + $guid =~ s/^.*\tguid\t*(\d*).*/$1/; my $snap = $line; $snap =~ s/^.*\@(.*)\tguid.*$/$1/; $snaps{$type}{$snap}{'guid'}=$guid; @@ -997,7 +997,7 @@ sub getsnaps() { if ($line =~ /\Q$fs\E\@.*creation/) { chomp $line; my $creation = $line; - $creation =~ s/^.*\screation\s*(\d*).*/$1/; + $creation =~ s/^.*\tcreation\t*(\d*).*/$1/; my $snap = $line; $snap =~ s/^.*\@(.*)\tcreation.*$/$1/; $snaps{$type}{$snap}{'creation'}=$creation; @@ -1056,9 +1056,9 @@ sub getsendsize { # the output format is different in case of # a resumed receive if (defined($receivetoken)) { - $sendsize =~ s/.*\s([0-9]+)$/$1/; + $sendsize =~ s/.*\t([0-9]+)$/$1/; } else { - $sendsize =~ s/^size\s*//; + $sendsize =~ s/^size\t*//; } chomp $sendsize; From 1f885801993eec2ce259d3ba4beb7704c6eb496d Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 28 Jun 2018 17:45:18 +0200 Subject: [PATCH 53/97] implemented support for excluding datasets from replication with a regular expression --- README.md | 4 ++++ syncoid | 17 ++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cc75bdf..e98763b 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This argument tells syncoid to restrict itself to existing snapshots, instead of creating a semi-ephemeral syncoid snapshot at execution time. Especially useful in multi-target (A->B, A->C) replication schemes, where you might otherwise accumulate a large number of foreign syncoid snapshots. ++ --exclude=REGEX + + The given regular expression will be matched against all datasets which would be synced by this run and excludes them. This argument can be specified multiple times. + + --no-resume This argument tells syncoid to not use resumeable zfs send/receive streams. diff --git a/syncoid b/syncoid index 999ee79..5cd925c 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-resume") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -141,6 +141,20 @@ sub getchilddatasets { my @children = ; close FH; + if (defined $args{'exclude'}) { + my $excludes = $args{'exclude'}; + foreach (@$excludes) { + for my $i ( 0 .. $#children ) { + if ($children[$i] =~ /$_/) { + if ($debug) { print "DEBUG: excluded $children[$i] because of $_\n"; } + undef $children[$i] + } + } + + @children = grep{ defined }@children; + } + } + return @children; } @@ -1141,6 +1155,7 @@ Options: --target-bwlimit= Bandwidth limit on the target transfer --no-stream Replicates using newest snapshot instead of intermediates --no-sync-snap Does not create new snapshot, only transfers existing + --exclude=REGEX Exclude specific datasets which match the given regular expression. Can be specified multiple times --sshkey=FILE Specifies a ssh public key to use to connect --sshport=PORT Connects to remote on a particular port From ba3836ec520efc30689238e294de1d1c2026fc1b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 6 Jul 2018 15:52:54 +0200 Subject: [PATCH 54/97] fixed monitor-health command for pools containing cache and log devices --- sanoid | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 9cd9d33..485ee08 100755 --- a/sanoid +++ b/sanoid @@ -976,6 +976,11 @@ sub check_zpool() { ## other cases my ($dev, $sta) = /^\s+(\S+)\s+(\S+)/; + if (!defined($sta)) { + # cache and logs are special and don't have a status + next; + } + ## pool online, not degraded thanks to dead/corrupted disk if ($state eq "OK" && $sta eq "UNAVAIL") { $state="WARNING"; @@ -1111,7 +1116,7 @@ sub checklock { # make sure lockfile contains something if ( -z $lockfile) { # zero size lockfile, something is wrong - die "ERROR: something is wrong! $lockfile is empty\n"; + die "ERROR: something is wrong! $lockfile is empty\n"; } # lockfile exists. read pid and mutex from it. see if it's our pid. if not, see if From f9c1cbb74a3c07fc9e0368721cdcedcc0b2b4a0f Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Sat, 7 Jul 2018 12:06:35 -0400 Subject: [PATCH 55/97] Update INSTALL --- INSTALL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL b/INSTALL index 33b510d..f0de17b 100644 --- a/INSTALL +++ b/INSTALL @@ -8,7 +8,7 @@ default for SSH transport since v1.4.6. Syncoid runs will fail if one of them is not available on either end of the transport. On Ubuntu: apt install pv lzop mbuffer -On CentOS: yum install lzo pv mbuffer lzop +On CentOS: yum install lzo pv mbuffer lzop perl-Data-Dumper On FreeBSD: pkg install pv mbuffer lzop FreeBSD notes: FreeBSD may place pv and lzop in somewhere other than From dc0afebb30865faa16f42c496b2c59e5ed72b690 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sat, 7 Jul 2018 20:06:17 +0200 Subject: [PATCH 56/97] allow extra identifier to contain all characters for snapshots names which are allowed by zfs --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index d25d9de..aaf5490 100755 --- a/syncoid +++ b/syncoid @@ -73,7 +73,7 @@ my $sshoptions = join " ", map { "-o " . $_ } @{$args{'sshoption'}}; # deref req my $identifier = ""; if (length $args{'identifier'}) { - if ($args{'identifier'} !~ /^[a-zA-Z0-9-]+$/) { + if ($args{'identifier'} !~ /^[a-zA-Z0-9-_:.]+$/) { # invalid extra identifier print("CRITICAL: extra identifier contains invalid chars!\n"); pod2usage(2); From f409b955694b9dc5f403f16593cc6540cac7fe67 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sat, 7 Jul 2018 20:10:02 +0200 Subject: [PATCH 57/97] updated parameter documention of --identifier regarding allowed characters --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e1f651..e359886 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --identifier= - Adds the given identifier to the snapshot name after "syncoid_" prefix and before the hostname. This enables the use case of reliable replication to multiple targets from the same host. The following chars are allowed: a-z, A-Z, 0-9 and -. + Adds the given identifier to the snapshot name after "syncoid_" prefix and before the hostname. This enables the use case of reliable replication to multiple targets from the same host. The following chars are allowed: a-z, A-Z, 0-9, _, -, : and . . + -r --recursive From 75d8174e6997286aad57d03d1c6ad86d2fd89c79 Mon Sep 17 00:00:00 2001 From: Piotr Paczynski Date: Wed, 18 Jul 2018 01:19:37 +0200 Subject: [PATCH 58/97] Fix 'resume support' detection on FreeBSD --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index aaf5490..31ff757 100755 --- a/syncoid +++ b/syncoid @@ -614,7 +614,7 @@ sub checkcommands { # check for ZFS resume feature support if ($resume) { - my $resumechkcmd = "$zfscmd get receive_resume_token -d 0"; + my $resumechkcmd = "$zfscmd get -d 0 receive_resume_token"; if ($debug) { print "DEBUG: checking availability of zfs resume feature on source...\n"; } $avail{'sourceresume'} = system("$sourcessh $resumechkcmd >/dev/null 2>&1"); From f5508a240387bb80f3db6937c8724fa006e4a5b2 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 26 Jul 2018 21:46:50 +0200 Subject: [PATCH 59/97] fix typo to make local source bwlimit work --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index f53aee4..cf941bf 100755 --- a/syncoid +++ b/syncoid @@ -757,7 +757,7 @@ sub buildsynccmd { $synccmd = "$sendcmd |"; # avoid confusion - accept either source-bwlimit or target-bwlimit as the bandwidth limiting option here my $bwlimit = ''; - if (length $args{'bwlimit'}) { + if (length $args{'source-bwlimit'}) { $bwlimit = $args{'source-bwlimit'}; } elsif (length $args{'target-bwlimit'}) { $bwlimit = $args{'target-bwlimit'}; From e85c375bbea636aa068bb3cd4add7d621568784c Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 26 Jul 2018 21:53:56 +0200 Subject: [PATCH 60/97] fixed typo --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index f53aee4..8be70e5 100755 --- a/syncoid +++ b/syncoid @@ -863,7 +863,7 @@ sub pruneoldsyncsnaps { $prunecmd = escapeshellparam($prunecmd); } system("$rhost $prunecmd") == 0 - or warn "CRITICAL ERROR: $rhost $prunecmd failed: $?"; + or warn "WARNING: $rhost $prunecmd failed: $?"; $prunecmd = ''; $counter = 0; } From 7c68ef5e8f2f491fe7de3b23a53f8b64a026a883 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 29 Jul 2018 13:16:53 +0200 Subject: [PATCH 61/97] return a non zero exit code if there was a problem replicating datasets --- syncoid | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index 6c569b2..6453d0f 100755 --- a/syncoid +++ b/syncoid @@ -97,6 +97,7 @@ my $targetsudocmd = $targetisroot ? '' : $sudocmd; my %avail = checkcommands(); my %snaps; +my $exitcode = 0; ## break here to call replication individually so that we ## ## can loop across children separately, for recursive ## @@ -127,7 +128,7 @@ if ($targethost ne '') { close FH; } -exit 0; +exit $exitcode; ############################################################################## ############################################################################## @@ -186,6 +187,7 @@ sub syncdataset { # make sure target is not currently in receive. if (iszfsbusy($targethost,$targetfs,$targetisroot)) { warn "Cannot sync now: $targetfs is already target of a zfs receive process.\n"; + if ($exitcode < 1) { $exitcode = 1; } return 0; } @@ -236,6 +238,7 @@ sub syncdataset { $newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot); if ($newsyncsnap eq 0) { warn "CRITICAL: no snapshots exist on source $sourcefs, and you asked for --no-sync-snap.\n"; + if ($exitcode < 1) { $exitcode = 1; } return 0; } } @@ -292,6 +295,7 @@ sub syncdataset { # make sure target is (still) not currently in receive. if (iszfsbusy($targethost,$targetfs,$targetisroot)) { warn "Cannot sync now: $targetfs is already target of a zfs receive process.\n"; + if ($exitcode < 1) { $exitcode = 1; } return 0; } system($synccmd) == 0 @@ -318,6 +322,7 @@ sub syncdataset { # make sure target is (still) not currently in receive. if (iszfsbusy($targethost,$targetfs,$targetisroot)) { warn "Cannot sync now: $targetfs is already target of a zfs receive process.\n"; + if ($exitcode < 1) { $exitcode = 1; } return 0; } @@ -325,9 +330,12 @@ sub syncdataset { if ($debug) { print "DEBUG: $synccmd\n"; } if ($oldestsnap ne $newsyncsnap) { - system($synccmd) == 0 - or warn "CRITICAL ERROR: $synccmd failed: $?"; + my $ret = system($synccmd); + if ($ret != 0) { + warn "CRITICAL ERROR: $synccmd failed: $?"; + if ($exitcode < 1) { $exitcode = 1; } return 0; + } } else { if (!$quiet) { print "INFO: no incremental sync needed; $oldestsnap is already the newest available snapshot.\n"; } } @@ -380,6 +388,7 @@ sub syncdataset { # make sure target is (still) not currently in receive. if (iszfsbusy($targethost,$targetfs,$targetisroot)) { warn "Cannot sync now: $targetfs is already target of a zfs receive process.\n"; + if ($exitcode < 1) { $exitcode = 1; } return 0; } @@ -910,6 +919,7 @@ sub getmatchingsnapshot { } # if we got this far, we failed to find a matching snapshot. + if ($exitcode < 2) { $exitcode = 2; } print "\n"; print "CRITICAL ERROR: Target $targetfs exists but has no snapshots matching with $sourcefs!\n"; From 63eec4994c20d7eb6207f6c6927badf138442f22 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 30 Jul 2018 22:21:14 +0200 Subject: [PATCH 62/97] don't die on some critical sync errors, but continue to replicate all the other datasets. after all is done exit with an error code --- syncoid | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/syncoid b/syncoid index 6453d0f..1ef1506 100755 --- a/syncoid +++ b/syncoid @@ -298,8 +298,11 @@ sub syncdataset { if ($exitcode < 1) { $exitcode = 1; } return 0; } - system($synccmd) == 0 - or die "CRITICAL ERROR: $synccmd failed: $?"; + system($synccmd) == 0 or do { + warn "CRITICAL ERROR: $synccmd failed: $?"; + if ($exitcode < 2) { $exitcode = 2; } + return 0; + }; # now do an -I to the new sync snapshot, assuming there were any snapshots # other than the new sync snapshot to begin with, of course - and that we @@ -359,8 +362,11 @@ sub syncdataset { if (!$quiet) { print "Resuming interrupted zfs send/receive from $sourcefs to $targetfs (~ $disp_pvsize remaining):\n"; } if ($debug) { print "DEBUG: $synccmd\n"; } - system("$synccmd") == 0 - or die "CRITICAL ERROR: $synccmd failed: $?"; + system("$synccmd") == 0 or do { + warn "CRITICAL ERROR: $synccmd failed: $?"; + if ($exitcode < 2) { $exitcode = 2; } + return 0; + }; # a resumed transfer will only be done to the next snapshot, # so do an normal sync cycle @@ -416,8 +422,11 @@ sub syncdataset { if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; } if ($debug) { print "DEBUG: $synccmd\n"; } - system("$synccmd") == 0 - or die "CRITICAL ERROR: $synccmd failed: $?"; + system("$synccmd") == 0 or do { + warn "CRITICAL ERROR: $synccmd failed: $?"; + if ($exitcode < 2) { $exitcode = 2; } + return 0; + }; # restore original readonly value to target after sync complete # dyking this functionality out for the time being due to buggy mount/unmount behavior From 9668567a870def5418032fb922d3a27a643059fa Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 30 Jul 2018 22:53:48 +0200 Subject: [PATCH 63/97] continue replication on more critical errors --- syncoid | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index 1ef1506..73205ce 100755 --- a/syncoid +++ b/syncoid @@ -233,6 +233,10 @@ sub syncdataset { if (!defined $args{'no-sync-snap'}) { # create a new syncoid snapshot on the source filesystem. $newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot); + if (!$newsyncsnap) { + # we already whined about the error + return 0; + } } else { # we don't want sync snapshots created, so use the newest snapshot we can find. $newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot); @@ -267,6 +271,11 @@ sub syncdataset { } my $oldestsnap = getoldestsnapshot(\%snaps); if (! $oldestsnap) { + if (defined ($args{'no-sync-snap'}) ) { + # we already whined about the missing snapshots + return 0; + } + # getoldestsnapshot() returned false, so use new sync snapshot if ($debug) { print "DEBUG: getoldestsnapshot() returned false, so using $newsyncsnap.\n"; } $oldestsnap = $newsyncsnap; @@ -752,7 +761,7 @@ sub getoldestsnapshot { # must not have had any snapshots on source - luckily, we already made one, amirite? if (defined ($args{'no-sync-snap'}) ) { # well, actually we set --no-sync-snap, so no we *didn't* already make one. Whoops. - die "CRIT: --no-sync-snap is set, and getoldestsnapshot() could not find any snapshots on source!\n"; + warn "CRIT: --no-sync-snap is set, and getoldestsnapshot() could not find any snapshots on source!\n"; } return 0; } @@ -774,6 +783,7 @@ sub getnewestsnapshot { # we also probably need an argument to mute this WARN, for people who deliberately exclude # datasets from recursive replication this way. warn "WARN: --no-sync-snap is set, and getnewestsnapshot() could not find any snapshots on source for current dataset. Continuing.\n"; + if ($exitcode < 2) { $exitcode = 2; } } return 0; } @@ -961,8 +971,12 @@ sub newsyncsnap { my %date = getdate(); my $snapname = "syncoid\_$identifier$hostid\_$date{'stamp'}"; my $snapcmd = "$rhost $mysudocmd $zfscmd snapshot $fsescaped\@$snapname\n"; - system($snapcmd) == 0 - or die "CRITICAL ERROR: $snapcmd failed: $?"; + system($snapcmd) == 0 or do { + warn "CRITICAL ERROR: $snapcmd failed: $?"; + if ($exitcode < 2) { $exitcode = 2; } + return 0; + }; + return $snapname; } From fb8edad885522040369c6ab95edfbb922fb6901d Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Fri, 3 Aug 2018 09:13:35 +0100 Subject: [PATCH 64/97] compression warnings are no longer hidden by --quiet --- syncoid | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index aec99cd..85d5e16 100755 --- a/syncoid +++ b/syncoid @@ -452,13 +452,13 @@ sub checkcommands { if ($avail{'sourcecompress'} eq '') { if ($compressargs{'rawcmd'} ne '') { - if (!$quiet) { print "WARN: $compressargs{'rawcmd'} not available on source $s- sync will continue without compression.\n"; } + print "WARN: $compressargs{'rawcmd'} not available on source $s- sync will continue without compression.\n"; } $avail{'compress'} = 0; } if ($avail{'targetcompress'} eq '') { if ($compressargs{'rawcmd'} ne '') { - if (!$quiet) { print "WARN: $compressargs{'rawcmd'} not available on target $t - sync will continue without compression.\n"; } + print "WARN: $compressargs{'rawcmd'} not available on target $t - sync will continue without compression.\n"; } $avail{'compress'} = 0; } @@ -471,7 +471,7 @@ sub checkcommands { # corner case - if source AND target are BOTH remote, we have to check for local compress too if ($sourcehost ne '' && $targethost ne '' && $avail{'localcompress'} eq '') { if ($compressargs{'rawcmd'} ne '') { - if (!$quiet) { print "WARN: $compressargs{'rawcmd'} not available on local machine - sync will continue without compression.\n"; } + print "WARN: $compressargs{'rawcmd'} not available on local machine - sync will continue without compression.\n"; } $avail{'compress'} = 0; } From 39d1fd38c1edd52f45b44707beabe32e3a20900d Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Fri, 5 Jan 2018 21:54:06 +0000 Subject: [PATCH 65/97] added ability to skip datasets... simply set syncoid:no-sync=true --- syncoid | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/syncoid b/syncoid index 6c569b2..8adaa87 100755 --- a/syncoid +++ b/syncoid @@ -183,6 +183,11 @@ sub syncdataset { if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; } + if (getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:no-sync') eq 'true') { + print "Skipping dataset (syncoid:nosync=true): $sourcefs...\n"; + return 0; + } + # make sure target is not currently in receive. if (iszfsbusy($targethost,$targetfs,$targetisroot)) { warn "Cannot sync now: $targetfs is already target of a zfs receive process.\n"; From 22c137627a9a923807565dff01794d9d3775e767 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Sat, 6 Jan 2018 10:15:30 +0000 Subject: [PATCH 66/97] updated doc for syncoid:no-sync --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index e359886..ba325e6 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,14 @@ If ZFS supports resumeable send/receive streams on both the source and target th As of 1.4.18, syncoid also automatically supports and enables resume of interrupted replication when both source and target support this feature. +##### Syncoid Dataset Properties + ++ syncoid:no-sync + + Setting this to `true` will prevent the dataset from being handled by syncoid in _any_ way - it will be skipped. This can be useful for preventing certain datasets from being transferred when recursively handling a tree. + + Note that this will also prevent syncoid from handling the dataset if given explicitly on the command line. + ##### Syncoid Command Line Options + [source] From b02b9a582bd3f92c01f16c954add64a78e647896 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Sat, 6 Jan 2018 10:43:30 +0000 Subject: [PATCH 67/97] now obeying --quiet, and added 'INFO: ' prefix to skip message --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 8adaa87..2201f41 100755 --- a/syncoid +++ b/syncoid @@ -184,7 +184,7 @@ sub syncdataset { if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; } if (getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:no-sync') eq 'true') { - print "Skipping dataset (syncoid:nosync=true): $sourcefs...\n"; + if (!$quiet) { print "INFO: Skipping dataset (syncoid:nosync=true): $sourcefs...\n"; } return 0; } From c188e47a6da2d8ee2857a8eb1c0400bc78fc7a53 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Fri, 9 Mar 2018 15:15:06 +0000 Subject: [PATCH 68/97] inverted sync/no-sync logic --- syncoid | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index 2201f41..20c0853 100755 --- a/syncoid +++ b/syncoid @@ -183,8 +183,8 @@ sub syncdataset { if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; } - if (getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:no-sync') eq 'true') { - if (!$quiet) { print "INFO: Skipping dataset (syncoid:nosync=true): $sourcefs...\n"; } + if (getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:sync') eq 'false') { + if (!$quiet) { print "INFO: Skipping dataset (syncoid:sync=false): $sourcefs...\n"; } return 0; } From f0a37310a54b5e421264d05a02ad226a1409bb98 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Fri, 9 Mar 2018 15:18:03 +0000 Subject: [PATCH 69/97] implemented sync true/false/${HOSTS} filtering --- syncoid | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 20c0853..7877723 100755 --- a/syncoid +++ b/syncoid @@ -183,9 +183,20 @@ sub syncdataset { if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; } - if (getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:sync') eq 'false') { + my $sync = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:sync'); + + if ($sync eq 'true' || $sync eq '-') { + # definitely sync this dataset - if a host is called 'true' or '-', then you're special + } elsif ($sync eq 'false') { if (!$quiet) { print "INFO: Skipping dataset (syncoid:sync=false): $sourcefs...\n"; } return 0; + } else { + my $hostid = hostname(); + my @hosts = split(/,/,$sync); + if (!(grep $hostid eq $_, @hosts)) { + if (!$quiet) { print "INFO: Skipping dataset (syncoid:sync doesn't include $hostid): $sourcefs...\n"; } + return 0; + } } # make sure target is not currently in receive. From a4e490f430b73454422c28254012a3328f9bbe98 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Fri, 9 Mar 2018 15:28:39 +0000 Subject: [PATCH 70/97] updated documentation for the syncoid:sync property's new behaviour --- README.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ba325e6..bb6ec89 100644 --- a/README.md +++ b/README.md @@ -120,11 +120,27 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup ##### Syncoid Dataset Properties -+ syncoid:no-sync ++ syncoid:sync - Setting this to `true` will prevent the dataset from being handled by syncoid in _any_ way - it will be skipped. This can be useful for preventing certain datasets from being transferred when recursively handling a tree. + Available values: - Note that this will also prevent syncoid from handling the dataset if given explicitly on the command line. + + `true` (default if unset) + + This dataset will be synchronised to all hosts. + + + `false` + + This dataset will not be synchronised to any hosts - it will be skipped. This can be useful for preventing certain datasets from being transferred when recursively handling a tree. + + + `host1,host2,...` + + A comma seperated list of hosts. This dataset will only be synchronised by hosts listed in the property. + + _Note_: this check is performed by the host running `syncoid`, thus the local hostname must be present for inclusion during a push operation // the remote hostname must be present for a pull. + + _Note_: this will also prevent syncoid from handling the dataset if given explicitly on the command line. + + _Note_: syncing a child of a no-sync dataset will currently result in a critical error ##### Syncoid Command Line Options From 60b2dedc456a8464f86cf75dacfe62c0b6410dbc Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Mon, 6 Aug 2018 14:07:58 +0100 Subject: [PATCH 71/97] fixed typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bb6ec89..8c05fb3 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + `host1,host2,...` - A comma seperated list of hosts. This dataset will only be synchronised by hosts listed in the property. + A comma separated list of hosts. This dataset will only be synchronised by hosts listed in the property. _Note_: this check is performed by the host running `syncoid`, thus the local hostname must be present for inclusion during a push operation // the remote hostname must be present for a pull. From d0ba0bc284bb92a8f0e63cfbd773e8c61e175973 Mon Sep 17 00:00:00 2001 From: Attie Grande Date: Mon, 6 Aug 2018 14:16:25 +0100 Subject: [PATCH 72/97] handled empty syncoid:sync property - behaves like unset --- README.md | 4 +++- syncoid | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8c05fb3..b833dec 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,9 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup _Note_: this will also prevent syncoid from handling the dataset if given explicitly on the command line. - _Note_: syncing a child of a no-sync dataset will currently result in a critical error + _Note_: syncing a child of a no-sync dataset will currently result in a critical error. + + _Note_: empty properties will be handled as if they were unset. ##### Syncoid Command Line Options diff --git a/syncoid b/syncoid index 7877723..fa00751 100755 --- a/syncoid +++ b/syncoid @@ -185,7 +185,8 @@ sub syncdataset { my $sync = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:sync'); - if ($sync eq 'true' || $sync eq '-') { + if ($sync eq 'true' || $sync eq '-' || $sync eq '') { + # empty is handled the same as unset (aka: '-') # definitely sync this dataset - if a host is called 'true' or '-', then you're special } elsif ($sync eq 'false') { if (!$quiet) { print "INFO: Skipping dataset (syncoid:sync=false): $sourcefs...\n"; } From 8de3cdce212ae15a849768e824d929afd6bc1501 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 3 Sep 2018 17:45:03 +0200 Subject: [PATCH 73/97] let monitor-health also check vdev member io/checksum errors --- sanoid | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) mode change 100755 => 100644 sanoid diff --git a/sanoid b/sanoid old mode 100755 new mode 100644 index 485ee08..7ae1b5b --- a/sanoid +++ b/sanoid @@ -974,7 +974,7 @@ sub check_zpool() { } ## other cases - my ($dev, $sta) = /^\s+(\S+)\s+(\S+)/; + my ($dev, $sta, $read, $write, $cksum) = /^\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/; if (!defined($sta)) { # cache and logs are special and don't have a status @@ -994,8 +994,21 @@ sub check_zpool() { ## no display for verbose level 1 next if ($verbose==1); ## don't display working devices for verbose level 2 - next if ($verbose==2 && $state eq "OK"); - next if ($verbose==2 && ($sta eq "ONLINE" || $sta eq "AVAIL" || $sta eq "INUSE")); + if ($verbose==2 && ($state eq "OK" || $sta eq "ONLINE" || $sta eq "AVAIL" || $sta eq "INUSE")) { + # check for io/checksum errors + + my @vdeverr = (); + if ($read != 0) { push @vdeverr, "read" }; + if ($write != 0) { push @vdeverr, "write" }; + if ($cksum != 0) { push @vdeverr, "cksum" }; + + if (scalar @vdeverr) { + $dmge=$dmge . "(" . $dev . ":" . join(", ", @vdeverr) . " errors) "; + if ($state eq "OK") { $state = "WARNING" }; + } + + next; + } ## show everything else if (/^\s{3}(\S+)/) { From 997487d12bf0f7d725199fb9fb9dd5cb4e484504 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 3 Sep 2018 17:46:21 +0200 Subject: [PATCH 74/97] restore filemode --- sanoid | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 sanoid diff --git a/sanoid b/sanoid old mode 100644 new mode 100755 From f39ed1ec49e989aa4fc7ae8c79f6329f5764cf3b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 3 Sep 2018 18:32:17 +0200 Subject: [PATCH 75/97] for remote target/source it's required to specify a user --- syncoid | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index b62eee8..30aef49 100755 --- a/syncoid +++ b/syncoid @@ -1218,9 +1218,9 @@ syncoid - ZFS snapshot replication tool =head1 SYNOPSIS syncoid [options]... SOURCE TARGET - or syncoid [options]... SOURCE [USER@]HOST:TARGET - or syncoid [options]... [USER@]HOST:SOURCE [TARGET] - or syncoid [options]... [USER@]HOST:SOURCE [USER@]HOST:TARGET + or syncoid [options]... SOURCE USER@HOST:TARGET + or syncoid [options]... USER@HOST:SOURCE TARGET + or syncoid [options]... USER@HOST:SOURCE USER@HOST:TARGET SOURCE Source ZFS dataset. Can be either local or remote TARGET Target ZFS dataset. Can be either local or remote From 055f26b9709aa412b2597e53c99e00c09cf5f5df Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 18:47:16 +0200 Subject: [PATCH 76/97] ignore unknown interval type to prevent perl warnings --- sanoid | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sanoid b/sanoid index 7ae1b5b..31376ad 100755 --- a/sanoid +++ b/sanoid @@ -438,6 +438,9 @@ sub take_snapshots { push @preferredtime,$datestamp{'year'}; $lastpreferred = timelocal(@preferredtime); if ($lastpreferred > time()) { $lastpreferred -= 60*60*24*31*365.25; } # preferred time is later this year - so look at last year + } else { + # unknown type + next; } # reconstruct our human-formatted most recent preferred snapshot time into an epoch time, to compare with the epoch of our most recent snapshot From d5f4d1c121e40da89fcca720ffd55332c3847b04 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 19:01:15 +0200 Subject: [PATCH 77/97] add lz4 compression --- README.md | 2 +- syncoid | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b833dec..ce09a37 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --compress - Currently accepted options: gzip, pigz-fast, pigz-slow, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. + Currently accepted options: gzip, pigz-fast, pigz-slow, zstd-fast, zstd-slow, lz4, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. + --source-bwlimit diff --git a/syncoid b/syncoid index 30aef49..c42a008 100755 --- a/syncoid +++ b/syncoid @@ -515,11 +515,17 @@ sub compressargset { decomrawcmd => '/usr/bin/lzop', decomargs => '-dfc', }, + 'lz4' => { + rawcmd => '/usr/bin/lz4', + args => '', + decomrawcmd => '/usr/bin/lz4', + decomargs => '-dc', + }, ); if ($value eq 'default') { $value = $DEFAULT_COMPRESSION; - } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lzo', 'default', 'none'))) { + } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lz4', 'lzo', 'default', 'none'))) { warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION"; $value = $DEFAULT_COMPRESSION; } @@ -1227,7 +1233,7 @@ syncoid - ZFS snapshot replication tool Options: - --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none + --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, zstd-fast, zstd-slow, lz4, lzo (default) & none --identifier=EXTRA Extra identifier which is included in the snapshot name. Can be used for replicating to multiple targets. --recursive|r Also transfers child datasets --skip-parent Skips syncing of the parent dataset. Does nothing without '--recursive' option. From 4af294838256fee6cee00d034bd75936992063d3 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 19:18:21 +0200 Subject: [PATCH 78/97] fix uninitialized value warning in debug mode on initial run (no snapshots yet) --- sanoid | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/sanoid b/sanoid index 7ae1b5b..0af79a4 100755 --- a/sanoid +++ b/sanoid @@ -492,16 +492,20 @@ sub blabber { my $path = $config{$section}{'path'}; print "Filesystem $path has:\n"; print " $snapsbypath{$path}{'numsnaps'} total snapshots "; - print "(newest: "; - my $newest = sprintf("%.1f",$snapsbypath{$path}{'newest'} / 60 / 60); - print "$newest hours old)\n"; + if ($snapsbypath{$path}{'numsnaps'} == 0) { + print "(no current snapshots)" + } else { + print "(newest: "; + my $newest = sprintf("%.1f",$snapsbypath{$path}{'newest'} / 60 / 60); + print "$newest hours old)\n"; - foreach my $type (keys %{ $snapsbytype{$path} }){ - print " $snapsbytype{$path}{$type}{'numsnaps'} $type\n"; - print " desired: $config{$section}{$type}\n"; - print " newest: "; - my $newest = sprintf("%.1f",($snapsbytype{$path}{$type}{'newest'} / 60 / 60)); - print "$newest hours old, named $snapsbytype{$path}{$type}{'newestname'}\n"; + foreach my $type (keys %{ $snapsbytype{$path} }){ + print " $snapsbytype{$path}{$type}{'numsnaps'} $type\n"; + print " desired: $config{$section}{$type}\n"; + print " newest: "; + my $newest = sprintf("%.1f",($snapsbytype{$path}{$type}{'newest'} / 60 / 60)); + print "$newest hours old, named $snapsbytype{$path}{$type}{'newestname'}\n"; + } } print "\n\n"; } From 807fc53afb4d28b3742af506d8e3f9b0ee2a4337 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 22:46:14 +0200 Subject: [PATCH 79/97] added all available syncoid/sanoid parameters to README --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index b833dec..9fcfcf6 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,13 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da This prints out quite alot of additional information during a sanoid run, and is normally not needed. ++ --readonly + + Skip creation/deletion of snapshots (Simulate). + ++ --help + + Show help message. ---------- @@ -206,6 +213,14 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup Allow sync to/from boxes running SSH on non-standard ports. ++ --sshcipher + + Instruct ssh to use a particular cipher set. + ++ --sshoption + + Passes option to ssh. This argument can be specified multiple times. + + --sshkey Use specified identity file as per ssh -i. @@ -218,6 +233,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This prints out quite alot of additional information during a sanoid run, and is normally not needed. ++ --help + + Show help message. + + --version Print the version and exit. From e7bd567acb1aed210f7d36cd9933460cab080da2 Mon Sep 17 00:00:00 2001 From: Andrew DeMaria Date: Mon, 10 Sep 2018 22:14:05 -0400 Subject: [PATCH 80/97] Do not monitor snapshots types that are set to 0. Signed-off-by: Andrew DeMaria --- sanoid | 1 + 1 file changed, 1 insertion(+) diff --git a/sanoid b/sanoid index 7ae1b5b..aa6369d 100755 --- a/sanoid +++ b/sanoid @@ -127,6 +127,7 @@ sub monitor_snapshots { my @types = ('yearly','monthly','daily','hourly'); foreach my $type (@types) { + if ($config{$section}{$type} == 0) { next; } my $smallerperiod = 0; # we need to set the period length in seconds first From e9a330f89a8ed9894df7d1a7bf6a996bcec3918f Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 5 Oct 2018 16:19:55 +0200 Subject: [PATCH 81/97] adapt test as it works correctly know after the DST patch --- tests/1_one_year/run.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/1_one_year/run.sh b/tests/1_one_year/run.sh index 7cec813..1cae7b4 100755 --- a/tests/1_one_year/run.sh +++ b/tests/1_one_year/run.sh @@ -10,7 +10,7 @@ set -x POOL_NAME="sanoid-test-1" POOL_TARGET="" # root RESULT="/tmp/sanoid_test_result" -RESULT_CHECKSUM="aa15e5595b0ed959313289ecb70323dad9903328ac46e881da5c4b0f871dd7cf" +RESULT_CHECKSUM="68c67161a59d0e248094a66061972f53613067c9db52ad981030f36bc081fed7" # UTC timestamp of start and end START="1483225200" @@ -46,10 +46,4 @@ done saveSnapshotList "${POOL_NAME}" "${RESULT}" # hourly daily monthly -verifySnapshotList "${RESULT}" 8759 366 12 "${RESULT_CHECKSUM}" - -# hourly count should be 8760 but one hour get's lost because of DST - -# daily count should be 365 but one additional daily is taken -# because the DST change leads to a day with 25 hours -# which will trigger an additional daily snapshot +verifySnapshotList "${RESULT}" 8760 365 12 "${RESULT_CHECKSUM}" From e83ec060fb68ad47545e24cde71e60c3d63f2752 Mon Sep 17 00:00:00 2001 From: Michael Bushey Date: Sun, 14 Oct 2018 15:10:25 -0700 Subject: [PATCH 82/97] INSTALL: Fix name p5-Config-Inifiles -> p5-Config-IniFiles --- INSTALL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL b/INSTALL index f0de17b..15c4896 100644 --- a/INSTALL +++ b/INSTALL @@ -30,4 +30,4 @@ strongly recommends using your distribution's repositories instead. On Ubuntu: apt install libconfig-inifiles-perl On CentOS: yum install perl-Config-IniFiles -On FreeBSD: pkg install p5-Config-Inifiles +On FreeBSD: pkg install p5-Config-IniFiles From c6ffbf5c4c05e8805a441656b253836631272221 Mon Sep 17 00:00:00 2001 From: Julien Riou Date: Sat, 13 Jan 2018 14:26:26 +0100 Subject: [PATCH 83/97] Add pre and post snapshot scripts --- sanoid | 25 ++++++++++++++++++++++++- sanoid.conf | 11 +++++++++++ sanoid.defaults.conf | 4 ++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 7ae1b5b..e45133b 100755 --- a/sanoid +++ b/sanoid @@ -455,6 +455,18 @@ sub take_snapshots { if ( (scalar(@newsnaps)) > 0) { foreach my $snap ( @newsnaps ) { + my $dataset = (split '@', $snap)[0]; + my $presnapshotfailure = 0; + if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) { + $ENV{'SANOID_TARGET'} = $dataset; + if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; } + if (system($config{$dataset}{'pre_snapshot_script'}) != 0) { + warn "WARN: pre_snapshot_script failed, $?"; + $config{$dataset}{'no_inconsistent_snapshot'} and next; + $presnapshotfailure = 1; + } + delete $ENV{'SANOID_TARGET'}; + } if ($args{'verbose'}) { print "taking snapshot $snap\n"; } if (!$args{'readonly'}) { system($zfs, "snapshot", "$snap") == 0 @@ -462,6 +474,17 @@ sub take_snapshots { # make sure we don't end up with multiple snapshots with the same ctime sleep 1; } + if ($config{$dataset}{'post_snapshot_script'} and !$args{'readonly'}) { + if (!$presnapshotfailure or $config{$dataset}{'force_post_snapshot_script'}) { + $ENV{'SANOID_TARGET'} = $dataset; + if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; } + if (system($config{$dataset}{'post_snapshot_script'}) != 0) { + warn "WARN: post_snapshot_script failed, $?"; + $config{$dataset}{'no_inconsistent_snapshot'} and next; + } + delete $ENV{'SANOID_TARGET'}; + } + } } $forcecacheupdate = 1; %snaps = getsnaps(%config,$cacheTTL,$forcecacheupdate); @@ -661,7 +684,7 @@ sub init { tie my %ini, 'Config::IniFiles', ( -file => $conf_file ) or die "FATAL: cannot load $conf_file - please create a valid local config file before running sanoid!"; # we'll use these later to normalize potentially true and false values on any toggle keys - my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only'); + my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only','no_inconsistent_snapshot','force_post_snapshot_script'); my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON"); my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF"); diff --git a/sanoid.conf b/sanoid.conf index 9b1f19d..218a492 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -67,6 +67,17 @@ daily_warn = 48 daily_crit = 60 +[template_scripts] + ### run script before snapshot + ### dataset name will be supplied as an environment variable $SANOID_TARGET + pre_snapshot_script = /path/to/script.sh + ### run script after snapshot + ### dataset name will be supplied as an environment variable $SANOID_TARGET + post_snapshot_script = /path/to/script.sh + ### don't take an inconsistent snapshot + #no_inconsistent_snapshot = yes + ### run post_snapshot_script when pre_snapshot_script is failing + #force_post_snapshot_script = yes [template_ignore] autoprune = no diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index d86cc47..12a8049 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -15,6 +15,10 @@ path = recursive = use_template = process_children_only = +pre_snapshot_script = +post_snapshot_script = +no_inconsistent_snapshot = +force_post_snapshot_script = # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately # prune any of those type snapshots already present. From 84213216ec2d98182e31c7e43ecce48cd353404a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Belli?= Date: Tue, 2 Oct 2018 00:47:25 +0200 Subject: [PATCH 84/97] Expose snapshot name through ENV variable --- sanoid | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sanoid b/sanoid index e45133b..73efd5e 100755 --- a/sanoid +++ b/sanoid @@ -456,9 +456,11 @@ sub take_snapshots { if ( (scalar(@newsnaps)) > 0) { foreach my $snap ( @newsnaps ) { my $dataset = (split '@', $snap)[0]; + my $snapname = (split '@', $snap)[1]; my $presnapshotfailure = 0; if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) { $ENV{'SANOID_TARGET'} = $dataset; + $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; } if (system($config{$dataset}{'pre_snapshot_script'}) != 0) { warn "WARN: pre_snapshot_script failed, $?"; @@ -466,6 +468,7 @@ sub take_snapshots { $presnapshotfailure = 1; } delete $ENV{'SANOID_TARGET'}; + delete $ENV{'SANOID_SNAPNAME'}; } if ($args{'verbose'}) { print "taking snapshot $snap\n"; } if (!$args{'readonly'}) { @@ -477,12 +480,14 @@ sub take_snapshots { if ($config{$dataset}{'post_snapshot_script'} and !$args{'readonly'}) { if (!$presnapshotfailure or $config{$dataset}{'force_post_snapshot_script'}) { $ENV{'SANOID_TARGET'} = $dataset; + $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; } if (system($config{$dataset}{'post_snapshot_script'}) != 0) { warn "WARN: post_snapshot_script failed, $?"; $config{$dataset}{'no_inconsistent_snapshot'} and next; } delete $ENV{'SANOID_TARGET'}; + delete $ENV{'SANOID_SNAPNAME'}; } } } From fb6608bf47a9508fb3b661a364e597b94b517e5f Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 16 Oct 2018 10:48:04 +0200 Subject: [PATCH 85/97] implemented timeout for pre/post script execution and made sure environment is cleaned up after script failure --- sanoid | 50 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/sanoid b/sanoid index 73efd5e..c12dff8 100755 --- a/sanoid +++ b/sanoid @@ -458,17 +458,21 @@ sub take_snapshots { my $dataset = (split '@', $snap)[0]; my $snapname = (split '@', $snap)[1]; my $presnapshotfailure = 0; + my $timeout = 5; if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; } - if (system($config{$dataset}{'pre_snapshot_script'}) != 0) { - warn "WARN: pre_snapshot_script failed, $?"; + my $ret = runscript('pre_snapshot_script',$dataset,$timeout); + + delete $ENV{'SANOID_TARGET'}; + delete $ENV{'SANOID_SNAPNAME'}; + + if ($ret != 0) { + # warning was already thrown by runscript function $config{$dataset}{'no_inconsistent_snapshot'} and next; $presnapshotfailure = 1; } - delete $ENV{'SANOID_TARGET'}; - delete $ENV{'SANOID_SNAPNAME'}; } if ($args{'verbose'}) { print "taking snapshot $snap\n"; } if (!$args{'readonly'}) { @@ -482,10 +486,8 @@ sub take_snapshots { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; } - if (system($config{$dataset}{'post_snapshot_script'}) != 0) { - warn "WARN: post_snapshot_script failed, $?"; - $config{$dataset}{'no_inconsistent_snapshot'} and next; - } + runscript('post_snapshot_script',$dataset,$timeout); + delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; } @@ -1337,6 +1339,38 @@ sub removecachedsnapshots { undef %pruned; } +#######################################################################################################################3 +#######################################################################################################################3 +#######################################################################################################################3 + +sub runscript { + my $key=shift; + my $dataset=shift; + my $timeout=shift; + + my $ret; + eval { + local $SIG{ALRM} = sub { die "alarm\n" }; + alarm $timeout; + $ret = system($config{$dataset}{$key}); + alarm 0; + }; + if ($@) { + if ($@ eq "alarm\n") { + warn "WARN: $key didn't finish in the allowed time!"; + } else { + warn "CRITICAL ERROR: $@"; + } + return -1; + } else { + if ($ret != 0) { + warn "WARN: $key failed, $?"; + } + } + + return $ret; +} + __END__ =head1 NAME From 0a7fdcb232d5e75eed388f016060cd4736c6185a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Belli?= Date: Sun, 14 Oct 2018 16:28:24 +0200 Subject: [PATCH 86/97] Add pruning hooks --- sanoid | 11 +++++++++++ sanoid.conf | 2 ++ sanoid.defaults.conf | 1 + 3 files changed, 14 insertions(+) diff --git a/sanoid b/sanoid index c12dff8..866eef0 100755 --- a/sanoid +++ b/sanoid @@ -299,6 +299,17 @@ sub prune_snapshots { if (! $args{'readonly'}) { if (system($zfs, "destroy", $snap) == 0) { $pruned{$snap} = 1; + my $dataset = (split '@', $snap)[0]; + my $snapname = (split '@', $snap)[1]; + if ($config{$dataset}{'pruning_script'}) { + $ENV{'SANOID_TARGET'} = $dataset; + $ENV{'SANOID_SNAPNAME'} = $snapname; + if ($args{'verbose'}) { print "executing pruning_script '".$config{$dataset}{'pruning_script'}."' on dataset '$dataset'\n"; } + system($config{$dataset}{'pruning_script'}) == 0 + or warn "WARN: pruning_script failed, $?"; + delete $ENV{'SANOID_TARGET'}; + delete $ENV{'SANOID_SNAPNAME'}; + } } else { warn "could not remove $snap : $?"; } diff --git a/sanoid.conf b/sanoid.conf index 218a492..e684614 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -78,6 +78,8 @@ #no_inconsistent_snapshot = yes ### run post_snapshot_script when pre_snapshot_script is failing #force_post_snapshot_script = yes + ### dataset name will be supplied as an environment variable $SANOID_TARGET + pruning_script = /path/to/script.sh [template_ignore] autoprune = no diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 12a8049..d4dd19e 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -19,6 +19,7 @@ pre_snapshot_script = post_snapshot_script = no_inconsistent_snapshot = force_post_snapshot_script = +pruning_script = # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately # prune any of those type snapshots already present. From a7b7fe8d15adc826d78f914f962b0b6929df88cb Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 16 Oct 2018 11:58:25 +0200 Subject: [PATCH 87/97] let pruning script timeout so it doesn't hang sanoid --- sanoid | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index 866eef0..7c213e4 100755 --- a/sanoid +++ b/sanoid @@ -302,11 +302,12 @@ sub prune_snapshots { my $dataset = (split '@', $snap)[0]; my $snapname = (split '@', $snap)[1]; if ($config{$dataset}{'pruning_script'}) { + my $timeout = 5; $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pruning_script '".$config{$dataset}{'pruning_script'}."' on dataset '$dataset'\n"; } - system($config{$dataset}{'pruning_script'}) == 0 - or warn "WARN: pruning_script failed, $?"; + my $ret = runscript('pruning_script',$dataset,$timeout); + delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; } From a8d5c5652a82c505b117ccd02a39962be240bd10 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 16 Oct 2018 17:54:37 +0200 Subject: [PATCH 88/97] make script timeout configureable --- sanoid | 16 +++++++++------- sanoid.conf | 6 ++++-- sanoid.defaults.conf | 3 ++- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/sanoid b/sanoid index 7c213e4..9c0e54d 100755 --- a/sanoid +++ b/sanoid @@ -302,11 +302,10 @@ sub prune_snapshots { my $dataset = (split '@', $snap)[0]; my $snapname = (split '@', $snap)[1]; if ($config{$dataset}{'pruning_script'}) { - my $timeout = 5; $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pruning_script '".$config{$dataset}{'pruning_script'}."' on dataset '$dataset'\n"; } - my $ret = runscript('pruning_script',$dataset,$timeout); + my $ret = runscript('pruning_script',$dataset); delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; @@ -475,7 +474,7 @@ sub take_snapshots { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; } - my $ret = runscript('pre_snapshot_script',$dataset,$timeout); + my $ret = runscript('pre_snapshot_script',$dataset); delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; @@ -498,7 +497,7 @@ sub take_snapshots { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; } - runscript('post_snapshot_script',$dataset,$timeout); + runscript('post_snapshot_script',$dataset); delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; @@ -1358,12 +1357,15 @@ sub removecachedsnapshots { sub runscript { my $key=shift; my $dataset=shift; - my $timeout=shift; + + my $timeout=$config{$dataset}{'script_timeout'}; my $ret; eval { - local $SIG{ALRM} = sub { die "alarm\n" }; - alarm $timeout; + if ($timeout gt 0) { + local $SIG{ALRM} = sub { die "alarm\n" }; + alarm $timeout; + } $ret = system($config{$dataset}{$key}); alarm 0; }; diff --git a/sanoid.conf b/sanoid.conf index e684614..db468e2 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -74,12 +74,14 @@ ### run script after snapshot ### dataset name will be supplied as an environment variable $SANOID_TARGET post_snapshot_script = /path/to/script.sh + ### dataset name will be supplied as an environment variable $SANOID_TARGET + pruning_script = /path/to/script.sh ### don't take an inconsistent snapshot #no_inconsistent_snapshot = yes ### run post_snapshot_script when pre_snapshot_script is failing #force_post_snapshot_script = yes - ### dataset name will be supplied as an environment variable $SANOID_TARGET - pruning_script = /path/to/script.sh + ### limit allowed execution time of scripts before continuing (<= 0 -> infinite) + script_timeout = 5 [template_ignore] autoprune = no diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index d4dd19e..d8e428a 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -17,9 +17,10 @@ use_template = process_children_only = pre_snapshot_script = post_snapshot_script = +pruning_script = +script_timeout = 5 no_inconsistent_snapshot = force_post_snapshot_script = -pruning_script = # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately # prune any of those type snapshots already present. From 6968e441468355a03bd1d6fdd39c5c93d2409d36 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 16 Oct 2018 18:10:57 +0200 Subject: [PATCH 89/97] updated documentation regarding pre/post/prun scripts --- sanoid | 1 - sanoid.conf | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/sanoid b/sanoid index 9c0e54d..69562f7 100755 --- a/sanoid +++ b/sanoid @@ -469,7 +469,6 @@ sub take_snapshots { my $dataset = (split '@', $snap)[0]; my $snapname = (split '@', $snap)[1]; my $presnapshotfailure = 0; - my $timeout = 5; if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; diff --git a/sanoid.conf b/sanoid.conf index db468e2..feb2237 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -68,19 +68,19 @@ daily_crit = 60 [template_scripts] + ### dataset and snapshot name will be supplied as environment variables + ### for all pre/post/prune scripts ($SANOID_TARGET, $SANOID_SNAPNAME) ### run script before snapshot - ### dataset name will be supplied as an environment variable $SANOID_TARGET pre_snapshot_script = /path/to/script.sh ### run script after snapshot - ### dataset name will be supplied as an environment variable $SANOID_TARGET post_snapshot_script = /path/to/script.sh - ### dataset name will be supplied as an environment variable $SANOID_TARGET + ### run script after pruning snapshot pruning_script = /path/to/script.sh - ### don't take an inconsistent snapshot + ### don't take an inconsistent snapshot (skip if pre script fails) #no_inconsistent_snapshot = yes ### run post_snapshot_script when pre_snapshot_script is failing #force_post_snapshot_script = yes - ### limit allowed execution time of scripts before continuing (<= 0 -> infinite) + ### limit allowed execution time of scripts before continuing (<= 0: infinite) script_timeout = 5 [template_ignore] From fa3c511dc106008bd03066059ed819f231734438 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 14 Nov 2018 18:26:11 +0100 Subject: [PATCH 90/97] split snapshot taking/pruning into seperate units for debian package to prevent pruning blocking snapshot taking --- packages/debian/rules | 12 +++++++++++- packages/debian/sanoid-prune.service | 13 +++++++++++++ packages/debian/sanoid.service | 2 +- 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 packages/debian/sanoid-prune.service diff --git a/packages/debian/rules b/packages/debian/rules index 83eb475..ddd77b0 100755 --- a/packages/debian/rules +++ b/packages/debian/rules @@ -16,4 +16,14 @@ override_dh_auto_install: @mkdir -p $(DESTDIR)/usr/share/doc/sanoid; \ cp sanoid.conf $(DESTDIR)/usr/share/doc/sanoid/sanoid.conf.example; @mkdir -p $(DESTDIR)/lib/systemd/system; \ - cp debian/sanoid.timer $(DESTDIR)/lib/systemd/system; + cp debian/sanoid-prune.service $(DESTDIR)/lib/systemd/system; + +override_dh_installinit: + dh_installinit --noscripts + +override_dh_systemd_enable: + dh_systemd_enable sanoid.timer + dh_systemd_enable sanoid-prune.service + +override_dh_systemd_start: + dh_systemd_start sanoid.timer diff --git a/packages/debian/sanoid-prune.service b/packages/debian/sanoid-prune.service new file mode 100644 index 0000000..c956bd5 --- /dev/null +++ b/packages/debian/sanoid-prune.service @@ -0,0 +1,13 @@ +[Unit] +Description=Cleanup ZFS Pool +Requires=zfs.target +After=zfs.target sanoid.service +ConditionFileNotEmpty=/etc/sanoid/sanoid.conf + +[Service] +Environment=TZ=UTC +Type=oneshot +ExecStart=/usr/sbin/sanoid --prune-snapshots + +[Install] +WantedBy=sanoid.service diff --git a/packages/debian/sanoid.service b/packages/debian/sanoid.service index 2d01bbf..e146354 100644 --- a/packages/debian/sanoid.service +++ b/packages/debian/sanoid.service @@ -7,4 +7,4 @@ ConditionFileNotEmpty=/etc/sanoid/sanoid.conf [Service] Environment=TZ=UTC Type=oneshot -ExecStart=/usr/sbin/sanoid --cron +ExecStart=/usr/sbin/sanoid --take-snapshots From 2796e22dbf8a6eaea8b4d1d27b04f9bea4636ddb Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 21 Nov 2018 00:34:21 +0100 Subject: [PATCH 91/97] added option to defer pruning based on the available pool capacity --- sanoid | 68 +++++++++++++++++++++++++++++++++++++++++--- sanoid.defaults.conf | 4 ++- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/sanoid b/sanoid index 7ae1b5b..34445aa 100755 --- a/sanoid +++ b/sanoid @@ -31,6 +31,7 @@ if (keys %args < 2) { my $pscmd = '/bin/ps'; my $zfs = '/sbin/zfs'; +my $zpool = '/sbin/zpool'; my $conf_file = "$args{'configdir'}/sanoid.conf"; my $default_conf_file = "$args{'configdir'}/sanoid.defaults.conf"; @@ -44,6 +45,7 @@ my $cache = '/var/cache/sanoidsnapshots.txt'; my $cacheTTL = 900; # 15 minutes my %snaps = getsnaps( \%config, $cacheTTL, $forcecacheupdate ); my %pruned; +my %capacitycache; my %snapsbytype = getsnapsbytype( \%config, \%snaps ); @@ -254,6 +256,10 @@ sub prune_snapshots { my $path = $config{$section}{'path'}; my $period = 0; + if (check_prune_defer($config, $section)) { + if ($args{'verbose'}) { print "INFO: deferring snapshot pruning ($section)...\n"; } + next; + } foreach my $type (keys %{ $config{$section} }){ unless ($type =~ /ly$/) { next; } @@ -872,7 +878,7 @@ sub check_zpool() { exit $ERRORS{$state}; } - my $statcommand="/sbin/zpool list -o name,size,cap,health,free $pool"; + my $statcommand="$zpool list -o name,size,cap,health,free $pool"; if (! open STAT, "$statcommand|") { print ("$state '$statcommand' command returns no result! NOTE: This plugin needs OS support for ZFS, and execution with root privileges.\n"); @@ -920,7 +926,7 @@ sub check_zpool() { ## flag to detect section of zpool status involving our zpool my $poolfind=0; - $statcommand="/sbin/zpool status $pool"; + $statcommand="$zpool status $pool"; if (! open STAT, "$statcommand|") { $state = 'CRITICAL'; print ("$state '$statcommand' command returns no result! NOTE: This plugin needs OS support for ZFS, and execution with root privileges.\n"); @@ -1028,7 +1034,7 @@ sub check_zpool() { return ($ERRORS{$state},$msg); } # end check_zpool() -sub check_capacity_limit() { +sub check_capacity_limit { my $value = shift; if (!defined($value) || $value !~ /^\d+\z/) { @@ -1051,7 +1057,7 @@ sub check_zpool_capacity() { my $capacitylimitsref=shift; my %capacitylimits=%$capacitylimitsref; - my $statcommand="/sbin/zpool list -H -o cap $pool"; + my $statcommand="$zpool list -H -o cap $pool"; if (! open STAT, "$statcommand|") { print ("$state '$statcommand' command returns no result!\n"); @@ -1096,6 +1102,60 @@ sub check_zpool_capacity() { return ($ERRORS{$state},$msg); } # end check_zpool_capacity() +sub check_prune_defer { + my ($config, $section) = @_; + + my $limit = $config{$section}{"prune_defer"}; + + if (!check_capacity_limit($limit)) { + die "ERROR: invalid prune_defer limit!\n"; + } + + if ($limit eq 0) { + return 0; + } + + my @parts = split /\//, $section, 2; + my $pool = $parts[0]; + + if (exists $capacitycache{$pool}) { + } else { + $capacitycache{$pool} = get_zpool_capacity($pool); + } + + if ($limit < $capacitycache{$pool}) { + return 0; + } + + return 1; +} + +sub get_zpool_capacity { + my $pool = shift; + + my $statcommand="$zpool list -H -o cap $pool"; + + if (! open STAT, "$statcommand|") { + die "ERROR: '$statcommand' command returns no result!\n"; + } + + my $line = ; + close(STAT); + + chomp $line; + my @row = split(/ +/, $line); + my $cap=$row[0]; + + ## check for valid capacity value + if ($cap !~ m/^[0-9]{1,3}%$/ ) { + die "ERROR: '$statcommand' command returned invalid capacity value ($cap)!\n"; + } + + $cap =~ s/\D//g; + + return $cap; +} + ###################################################################################################### ###################################################################################################### ###################################################################################################### diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index d86cc47..0797fa8 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -26,7 +26,9 @@ hourly = 48 daily = 90 monthly = 6 yearly = 0 -min_percent_free = 10 +# pruning can be skipped based on the used capacity of the pool +# (0: always prune, 1-100: only prune if used capacity is greater than this value) +prune_defer = 0 # We will automatically take snapshots if autosnap is on, at the desired times configured # below (or immediately, if we don't have one since the last preferred time for that type). From c8b880c5e2ebb4771fa845af8e4cd006751d8970 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 21 Nov 2018 18:01:40 +0100 Subject: [PATCH 92/97] implemented clone handling (try to recreate on target instead of full replication) --- syncoid | 116 +++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 93 insertions(+), 23 deletions(-) diff --git a/syncoid b/syncoid index 30aef49..e9e3f27 100755 --- a/syncoid +++ b/syncoid @@ -104,17 +104,59 @@ my $exitcode = 0; ## replication ## if (!defined $args{'recursive'}) { - syncdataset($sourcehost, $sourcefs, $targethost, $targetfs); + syncdataset($sourcehost, $sourcefs, $targethost, $targetfs, undef); } else { if ($debug) { print "DEBUG: recursive sync of $sourcefs.\n"; } my @datasets = getchilddatasets($sourcehost, $sourcefs, $sourceisroot); - foreach my $dataset(@datasets) { + + my @deferred; + + foreach my $datasetProperties(@datasets) { + my $dataset = $datasetProperties->{'name'}; + my $origin = $datasetProperties->{'origin'}; + if ($origin eq "-") { + $origin = undef; + } else { + # check if clone source is replicated too + my @values = split(/@/, $origin, 2); + my $srcdataset = $values[0]; + + my $found = 0; + foreach my $datasetProperties(@datasets) { + if ($datasetProperties->{'name'} eq $srcdataset) { + $found = 1; + last; + } + } + + if ($found == 0) { + # clone source is not replicated, do a full replication + $origin = undef; + } else { + # clone source is replicated, defer until all non clones are replicated + push @deferred, $datasetProperties; + next; + } + } + $dataset =~ s/\Q$sourcefs\E//; chomp $dataset; my $childsourcefs = $sourcefs . $dataset; my $childtargetfs = $targetfs . $dataset; # print "syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs); \n"; - syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs); + syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs, $origin); + } + + # replicate cloned datasets and if this is the initial run, recreate them on the target + foreach my $datasetProperties(@deferred) { + my $dataset = $datasetProperties->{'name'}; + my $origin = $datasetProperties->{'origin'}; + + $dataset =~ s/\Q$sourcefs\E//; + chomp $dataset; + my $childsourcefs = $sourcefs . $dataset; + my $childtargetfs = $targetfs . $dataset; + syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs, $origin); } } @@ -147,37 +189,51 @@ sub getchilddatasets { $fsescaped = escapeshellparam($fsescaped); } - my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name -t filesystem,volume -Hr $fsescaped |"; + my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name,origin -t filesystem,volume -Hr $fsescaped |"; if ($debug) { print "DEBUG: getting list of child datasets on $fs using $getchildrencmd...\n"; } - open FH, $getchildrencmd; - my @children = ; - close FH; - - if (defined $args{'skip-parent'}) { - # parent dataset is the first element - shift @children; + if (! open FH, $getchildrencmd) { + die "ERROR: list command failed!\n"; } - if (defined $args{'exclude'}) { - my $excludes = $args{'exclude'}; - foreach (@$excludes) { - for my $i ( 0 .. $#children ) { - if ($children[$i] =~ /$_/) { - if ($debug) { print "DEBUG: excluded $children[$i] because of $_\n"; } - undef $children[$i] + my @children; + my $first = 1; + + DATASETS: while() { + chomp; + + if (defined $args{'skip-parent'} && $first eq 1) { + # parent dataset is the first element + $first = 0; + next; + } + + my ($dataset, $origin) = /^([^\t]+)\t([^\t]+)/; + + if (defined $args{'exclude'}) { + my $excludes = $args{'exclude'}; + foreach (@$excludes) { + print("$dataset\n"); + if ($dataset =~ /$_/) { + if ($debug) { print "DEBUG: excluded $dataset because of $_\n"; } + next DATASETS; } } - - @children = grep{ defined }@children; } + + my %properties; + $properties{'name'} = $dataset; + $properties{'origin'} = $origin; + + push @children, \%properties; } + close FH; return @children; } sub syncdataset { - my ($sourcehost, $sourcefs, $targethost, $targetfs) = @_; + my ($sourcehost, $sourcefs, $targethost, $targetfs, $origin) = @_; my $sourcefsescaped = escapeshellparam($sourcefs); my $targetfsescaped = escapeshellparam($targetfs); @@ -305,11 +361,25 @@ sub syncdataset { my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped"; my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; - my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); + my $pvsize; + if (defined $origin) { + my $originescaped = escapeshellparam($origin); + $sendcmd = "$sourcesudocmd $zfscmd send -i $originescaped $sourcefsescaped\@$oldestsnapescaped"; + my $streamargBackup = $args{'streamarg'}; + $args{'streamarg'} = "-i"; + $pvsize = getsendsize($sourcehost,$origin,"$sourcefs\@$oldestsnap",$sourceisroot); + $args{'streamarg'} = $streamargBackup; + } else { + $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); + } + my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = 'UNKNOWN'; } my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); if (!$quiet) { + if (defined $origin) { + print "INFO: Clone is recreated on target $targetfs based on $origin\n"; + } if (!defined ($args{'no-stream'}) ) { print "INFO: Sending oldest full snapshot $sourcefs\@$oldestsnap (~ $disp_pvsize) to new target filesystem:\n"; } else { @@ -396,7 +466,7 @@ sub syncdataset { # a resumed transfer will only be done to the next snapshot, # so do an normal sync cycle - return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs); + return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs, undef); } # find most recent matching snapshot and do an -I From 9d6cb42f4d2927c34d9d6344fbadb57f9f72caa1 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 21 Nov 2018 18:08:38 +0100 Subject: [PATCH 93/97] added option to disable smart clone handling --- README.md | 5 +++++ syncoid | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b833dec..ed7a107 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,11 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This argument tells syncoid to not use resumeable zfs send/receive streams. ++ --no-clone-handling + + This argument tells syncoid to not recreate clones on the targe on initial sync and doing a normal replication instead. + + + --dumpsnaps This prints a list of snapshots during the run. diff --git a/syncoid b/syncoid index e9e3f27..c54c915 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,8 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", + "no-clone-handling") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -114,7 +115,7 @@ if (!defined $args{'recursive'}) { foreach my $datasetProperties(@datasets) { my $dataset = $datasetProperties->{'name'}; my $origin = $datasetProperties->{'origin'}; - if ($origin eq "-") { + if ($origin eq "-" || defined $args{'no-clone-handling'}) { $origin = undef; } else { # check if clone source is replicated too @@ -1320,3 +1321,4 @@ Options: --dumpsnaps Dumps a list of snapshots during the run --no-command-checks Do not check command existence before attempting transfer. Not recommended --no-resume Don't use the ZFS resume feature if available + --no-clone-handling Don't try to recreate clones on target From f153810d08f86e804f7e43febb812ddabe810e92 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 21 Nov 2018 23:14:30 +0100 Subject: [PATCH 94/97] check for valid estimated send size to prevent a perl warning on systems which doesn't output size informations --- syncoid | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/syncoid b/syncoid index 30aef49..1f759e2 100755 --- a/syncoid +++ b/syncoid @@ -1154,6 +1154,11 @@ sub getsendsize { } chomp $sendsize; + # check for valid value + if ($sendsize !~ /^\d+$/) { + $sendsize = ''; + } + # to avoid confusion with a zero size pv, give sendsize # a minimum 4K value - or if empty, make sure it reads UNKNOWN if ($debug) { print "DEBUG: sendsize = $sendsize\n"; } From a25ec83812ace12c26ba01296e278a2799d04d77 Mon Sep 17 00:00:00 2001 From: Rodger Donaldson Date: Sun, 25 Nov 2018 07:14:59 +1300 Subject: [PATCH 95/97] Add a dependency for the configini patch The RPM will install but fail to run if the perl-Config-IniFiles rpm is not also installed; this adds as a Requires: so that yum/dnf can find and install. --- packages/rhel/sanoid.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index 3a9412f..7d4995d 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -14,7 +14,7 @@ License: GPLv3 URL: https://github.com/jimsalterjrs/sanoid Source0: https://github.com/jimsalterjrs/%{name}/archive/%{git_tag}/%{name}-%{version}.tar.gz -Requires: perl, mbuffer, lzop, pv +Requires: perl, mbuffer, lzop, pv, perl-Config-IniFiles %if 0%{?_with_systemd} Requires: systemd >= 212 From ea55308dfcdce7614306d8a4b7187ff61d43c4b8 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 4 Dec 2018 15:51:51 +0100 Subject: [PATCH 96/97] implemented support for excluding children of a specific dataset --- sanoid | 18 ++++++++++++++---- sanoid.defaults.conf | 1 + 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/sanoid b/sanoid index 7ae1b5b..8f390ba 100755 --- a/sanoid +++ b/sanoid @@ -661,7 +661,7 @@ sub init { tie my %ini, 'Config::IniFiles', ( -file => $conf_file ) or die "FATAL: cannot load $conf_file - please create a valid local config file before running sanoid!"; # we'll use these later to normalize potentially true and false values on any toggle keys - my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only'); + my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only','skip_children'); my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON"); my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF"); @@ -718,7 +718,7 @@ sub init { # override with any locally set values in the module itself foreach my $key (keys %{$ini{$section}} ) { - if (! ($key =~ /template|recursive/)) { + if (! ($key =~ /template|recursive|skip_children/)) { if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value directly set in module.\n"; } $config{$section}{$key} = $ini{$section}{$key}; } @@ -743,10 +743,17 @@ sub init { # how 'bout some recursion? =) my @datasets; - if ($ini{$section}{'recursive'}) { + if ($ini{$section}{'recursive'} || $ini{$section}{'skip_children'}) { @datasets = getchilddatasets($config{$section}{'path'}); - foreach my $dataset(@datasets) { + DATASETS: foreach my $dataset(@datasets) { chomp $dataset; + + if ($ini{$section}{'skip_children'}) { + if ($args{'debug'}) { print "DEBUG: ignoring $dataset.\n"; } + delete $config{$dataset}; + next DATASETS; + } + foreach my $key (keys %{$config{$section}} ) { if (! ($key =~ /template|recursive|children_only/)) { if ($args{'debug'}) { print "DEBUG: recursively setting $key from $section to $dataset.\n"; } @@ -1257,6 +1264,9 @@ sub getchilddatasets { my @children = ; close FH; + # parent dataset is the first element + shift @children; + return @children; } diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index d86cc47..0c9037a 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -15,6 +15,7 @@ path = recursive = use_template = process_children_only = +skip_children = # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately # prune any of those type snapshots already present. From a0b983ee6ef791449fb631353e26b4f05a5ab71a Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 4 Dec 2018 21:38:15 +0100 Subject: [PATCH 97/97] warn if unknown interval type is used --- sanoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanoid b/sanoid index cb43066..5c81893 100755 --- a/sanoid +++ b/sanoid @@ -471,7 +471,7 @@ sub take_snapshots { $lastpreferred = timelocal(@preferredtime); if ($lastpreferred > time()) { $lastpreferred -= 60*60*24*31*365.25; } # preferred time is later this year - so look at last year } else { - # unknown type + warn "WARN: unknown interval type $type in config!"; next; }