From ee6c1d15e41694c811006e961f3b6862f26bbb05 Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Thu, 10 Aug 2017 14:16:34 +0100 Subject: [PATCH 01/15] Convert sanoid's argument parser to GetOpt::Long and add helptext --- sanoid | 117 ++++++++++++++++++++------------------------------------- 1 file changed, 40 insertions(+), 77 deletions(-) diff --git a/sanoid b/sanoid index 687da03..80de11e 100755 --- a/sanoid +++ b/sanoid @@ -4,22 +4,26 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.17'; +$::VERSION = '1.4.17'; use strict; -use Config::IniFiles; # read samba-style conf file -use File::Path; # for rmtree command in use_prune -use Data::Dumper; # debugging - print contents of hash -use Time::Local; # to parse dates in reverse +use Config::IniFiles; # read samba-style conf file +use Data::Dumper; # debugging - print contents of hash +use File::Path; # for rmtree command in use_prune +use Getopt::Long qw(:config auto_version auto_help); +use Pod::Usage; # pod2usage +use Time::Local; # to parse dates in reverse -# parse CLI arguments -my %args = getargs(@ARGV); +my %args = ("cron" => 1, "verbose" => 1, "configdir" => "/etc/sanoid"); +GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet", + "monitor-health", "force-update", "configdir=s", + "monitor-snapshots", "take-snapshots", "prune-snapshots" + ) or pod2usage(2); my $pscmd = '/bin/ps'; my $zfs = '/sbin/zfs'; -if ($args{'configdir'} eq '') { $args{'configdir'} = '/etc/sanoid'; } my $conf_file = "$args{'configdir'}/sanoid.conf"; my $default_conf_file = "$args{'configdir'}/sanoid.defaults.conf"; @@ -42,11 +46,9 @@ if ($args{'debug'}) { $args{'verbose'}=1; blabber (@params); } if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); } if ($args{'monitor-health'}) { monitor_health(@params); } if ($args{'force-update'}) { my $snaps = getsnaps( \%config, $cacheTTL, 1 ); } -if ($args{'version'}) { print "INFO: Sanoid version: $version\n"; } -if ($args{'cron'} || $args{'noargs'}) { - if ($args{'noargs'}) { print "INFO: No arguments given - assuming --cron and --verbose.\n"; } - if (!$args{'quiet'}) { $args{'verbose'} = 1; } +if ($args{'cron'}) { + if ($args{'quiet'}) { $args{'verbose'} = 0; } take_snapshots (@params); prune_snapshots (@params); } else { @@ -1026,71 +1028,6 @@ sub iszfsbusy { #######################################################################################################################3 #######################################################################################################################3 -sub getargs { - my @args = @_; - my %args; - - my @validargs; - my @novalueargs; - my %validargs; - my %novalueargs; - - push my @validargs, 'verbose','debug','version','monitor-health','monitor-snapshots','force-update','cron','take-snapshots','prune-snapshots','readonly','configdir','quiet'; - push my @novalueargs, 'verbose','debug','version','monitor-health','monitor-snapshots','force-update','cron','take-snapshots','prune-snapshots','readonly','quiet'; - foreach my $item (@validargs) { $validargs{$item}=1; } - foreach my $item (@novalueargs) { $novalueargs{$item}=1; } - - if (! (scalar @args)) { - $args{'noargs'} = 1; - } - - while (my $rawarg = shift(@args)) { - my $argvalue; - my $arg = $rawarg; - if ($rawarg =~ /=/) { - # user specified the value for a CLI argument with = - # instead of with blank space. separate appropriately. - $argvalue = $arg; - $arg =~ s/=.*$//; - $argvalue =~ s/^.*=//; - } - if ($rawarg =~ /^--/) { - # doubledash arg - $arg =~ s/^--//; - if ($novalueargs{$arg}) { - $args{$arg} = 1; - } else { - # if this CLI arg takes a user-specified value and - # we don't already have it, then the user must have - # specified with a space, so pull in the next value - # from the array as this value rather than as the - # next argument. - if ($argvalue eq '') { $argvalue = shift(@args); } - $args{$arg} = $argvalue; - } - } elsif ($rawarg =~ /^-/) { - # singledash arg - $arg =~ s/^-//; - if ($novalueargs{$arg}) { - $args{$arg} = 1; - } else { - # if this CLI arg takes a user-specified value and - # we don't already have it, then the user must have - # specified with a space, so pull in the next value - # from the array as this value rather than as the - # next argument. - if ($argvalue eq '') { $argvalue = shift(@args); } - $args{$arg} = $argvalue; - } - } else { - # bare arg - die "ERROR: don't know what to do with bare argument $rawarg.\n"; - } - if (! ($validargs{$arg})) { die "ERROR: don't understand argument $rawarg.\n"; } - } - return %args; -} - sub getchilddatasets { # for later, if we make sanoid itself support sudo use my $fs = shift; @@ -1105,3 +1042,29 @@ sub getchilddatasets { return @children; } +__END__ + +=head1 NAME + +sanoid - ZFS snapshot management and replication tool + +=head1 SYNOPSIS + +sanoid [options] + +Options: + --configdir=DIR Specify a directory to find config file sanoid.conf + + --cron Creates snapshots and purges expired snapshots + --readonly Simulates creation/deletion of snapshots + --quiet Suppresses non-error output + --force-update Clears out sanoid's zfs snapshot cache + + --monitor-health Reports on zpool "health", in a Nagios compatible format + --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format + --take-snapshots Creates snapshots as specified in sanoid.conf + --prune-snapshots Purges expired snapshots as specified in sanoid.conf + + --help Prints this helptext + --verbose Prints the version number + --debug Prints out a lot of additional information during a sanoid run From 36980d4788da2ccaa444266f68e90c3e78849e08 Mon Sep 17 00:00:00 2001 From: jimsalterjrs Date: Fri, 11 Aug 2017 10:14:03 -0400 Subject: [PATCH 02/15] nerfed most of the warnings with if defined checks --- sanoid | 92 +++++++++++++++++++++++++++++++--------------------------- 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/sanoid b/sanoid index 80de11e..83bce62 100755 --- a/sanoid +++ b/sanoid @@ -7,6 +7,7 @@ $::VERSION = '1.4.17'; use strict; +use warnings; use Config::IniFiles; # read samba-style conf file use Data::Dumper; # debugging - print contents of hash use File::Path; # for rmtree command in use_prune @@ -195,45 +196,48 @@ sub prune_snapshots { elsif ($type eq 'daily') { $period = 60*60*24; } elsif ($type eq 'monthly') { $period = 60*60*24*31; } elsif ($type eq 'yearly') { $period = 60*60*24*365.25; } + + # avoid pissing off use warnings by not executing this block if no matching snaps exist + if (defined $snapsbytype{$path}{$type}{'sorted'}) { + my @sorted = split (/\|/,$snapsbytype{$path}{$type}{'sorted'}); - my @sorted = split (/\|/,$snapsbytype{$path}{$type}{'sorted'}); + # if we say "daily=30" we really mean "don't keep any dailies more than 30 days old", etc + my $maxage = ( time() - $config{$section}{$type} * $period ); + # but if we say "daily=30" we ALSO mean "don't get rid of ANY dailies unless we have more than 30". + my $minsnapsthistype = $config{$section}{$type}; - # if we say "daily=30" we really mean "don't keep any dailies more than 30 days old", etc - my $maxage = ( time() - $config{$section}{$type} * $period ); - # but if we say "daily=30" we ALSO mean "don't get rid of ANY dailies unless we have more than 30". - my $minsnapsthistype = $config{$section}{$type}; + # how many total snaps of this type do we currently have? + my $numsnapsthistype = scalar (@sorted); - # how many total snaps of this type do we currently have? - my $numsnapsthistype = scalar (@sorted); - - my @prunesnaps; - foreach my $snap( @sorted ){ - # print "snap $path\@$snap has age $snaps{$path}{$snap}{'ctime'}, maxage is $maxage.\n"; - if ( ($snaps{$path}{$snap}{'ctime'} < $maxage) && ($numsnapsthistype > $minsnapsthistype) ) { - my $fullpath = $path . '@' . $snap; - push(@prunesnaps,$fullpath); - # we just got rid of a snap, so we now have one fewer, duh - $numsnapsthistype--; - } - } - - if ((scalar @prunesnaps) > 0) { - # print "found some snaps to prune!\n" - if (checklock('sanoid_pruning')) { - writelock('sanoid_pruning'); - foreach my $snap( @prunesnaps ){ - if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; } - if (iszfsbusy($path)) { - print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; - } else { - if (! $args{'readonly'}) { system($zfs, "destroy",$snap) == 0 or warn "could not remove $snap : $?"; } - } + my @prunesnaps; + foreach my $snap( @sorted ){ + # print "snap $path\@$snap has age $snaps{$path}{$snap}{'ctime'}, maxage is $maxage.\n"; + if ( ($snaps{$path}{$snap}{'ctime'} < $maxage) && ($numsnapsthistype > $minsnapsthistype) ) { + my $fullpath = $path . '@' . $snap; + push(@prunesnaps,$fullpath); + # we just got rid of a snap, so we now have one fewer, duh + $numsnapsthistype--; + } + } + + if ((scalar @prunesnaps) > 0) { + # print "found some snaps to prune!\n" + if (checklock('sanoid_pruning')) { + writelock('sanoid_pruning'); + foreach my $snap( @prunesnaps ){ + if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; } + if (iszfsbusy($path)) { + print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; + } else { + if (! $args{'readonly'}) { system($zfs, "destroy",$snap) == 0 or warn "could not remove $snap : $?"; } + } + } + removelock('sanoid_pruning'); + $forcecacheupdate = 1; + %snaps = getsnaps(%config,$cacheTTL,$forcecacheupdate); + } else { + print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n"; } - removelock('sanoid_pruning'); - $forcecacheupdate = 1; - %snaps = getsnaps(%config,$cacheTTL,$forcecacheupdate); - } else { - print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n"; } } } @@ -512,10 +516,14 @@ sub getsnaps { foreach my $snap (@rawsnaps) { my ($fs,$snapname,$snapdate) = ($snap =~ m/(.*)\@(.*ly)\s*creation\s*(\d*)/); - my ($snaptype) = ($snapname =~ m/.*_(\w*ly)/); - if ($snapname =~ /^autosnap/) { - $snaps{$fs}{$snapname}{'ctime'}=$snapdate; - $snaps{$fs}{$snapname}{'type'}=$snaptype; + + # avoid pissing off use warnings + if (defined $snapname) { + my ($snaptype) = ($snapname =~ m/.*_(\w*ly)/); + if ($snapname =~ /^autosnap/) { + $snaps{$fs}{$snapname}{'ctime'}=$snapdate; + $snaps{$fs}{$snapname}{'type'}=$snaptype; + } } } @@ -603,10 +611,10 @@ sub init { # make sure that true values are true and false values are false for any toggled values foreach my $toggle(@toggles) { foreach my $true (@istrue) { - if ($config{$section}{$toggle} eq $true) { $config{$section}{$toggle} = 1; } + if (defined $config{$section}{$toggle} && $config{$section}{$toggle} eq $true) { $config{$section}{$toggle} = 1; } } foreach my $false (@isfalse) { - if ($config{$section}{$toggle} eq $false) { $config{$section}{$toggle} = 0; } + if (defined $config{$section}{$toggle} && $config{$section}{$toggle} eq $false) { $config{$section}{$toggle} = 0; } } } @@ -1031,7 +1039,7 @@ sub iszfsbusy { sub getchilddatasets { # for later, if we make sanoid itself support sudo use my $fs = shift; - my $mysudocmd; + my $mysudocmd = ''; my $getchildrencmd = "$mysudocmd $zfs list -o name -Hr $fs |"; if ($args{'debug'}) { print "DEBUG: getting list of child datasets on $fs using $getchildrencmd...\n"; } From f8ea8f907dc104393c206266789173562f8cc955 Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Tue, 15 Aug 2017 15:56:04 +0100 Subject: [PATCH 03/15] Add GetOptions helptext to syncoid Lots of changes here Notably: nocommandcheck is now no-command-check sshoptions can be specified multiple times sshcipher now defaults to whatever your ssh client wants to compress no longer accepts "no" or "0" --- syncoid | 319 ++++++++++++++++++++++++-------------------------------- 1 file changed, 139 insertions(+), 180 deletions(-) diff --git a/syncoid b/syncoid index 2989ab5..8fa0453 100755 --- a/syncoid +++ b/syncoid @@ -4,25 +4,43 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.16'; +$::VERSION = '1.4.16'; use strict; use warnings; use Data::Dumper; +use Getopt::Long qw(:config auto_version auto_help); +use Pod::Usage; use Time::Local; use Sys::Hostname; -my %args = getargs(@ARGV); +# Blank defaults to use ssh client's default +# TODO: Merge into a single "sshflags" option? +my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); +GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", + "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", + "debug", "quiet", "no-stream", "no-sync-snap") or pod2usage(2); -if ($args{'version'}) { - print "Syncoid version: $version\n"; - exit 0; +$args{'compress'} = compressargset($args{'compress'} || 'default'); # Can't be done with GetOptions arg, as default still needs to be set + +# TODO Expand to accept multiple sources? +if (scalar(@ARGV) != 2) { + print("Source or target not found!\n"); + pod2usage(2); + exit 127; +} else { + $args{'source'} = $ARGV[0]; + $args{'target'} = $ARGV[1]; } -if (!(defined $args{'source'} && defined $args{'target'})) { - print 'usage: syncoid [src_user@src_host:]src_pool/src_dataset [dst_user@dst_host:]dst_pool/dst_dataset'."\n"; - exit 127; +# Could possibly merge these into an options function +if (length $args{'source-bwlimit'}) { + $args{'source-bwlimit'} = "-R $args{'source-bwlimit'}"; } +if (length $args{'target-bwlimit'}) { + $args{'target-bwlimit'} = "-r $args{'target-bwlimit'}"; +} +$args{'streamarg'} = (defined $args{'no-stream'} ? '-i' : '-I'); my $rawsourcefs = $args{'source'}; my $rawtargetfs = $args{'target'}; @@ -32,25 +50,8 @@ my $quiet = $args{'quiet'}; my $zfscmd = '/sbin/zfs'; my $sshcmd = '/usr/bin/ssh'; my $pscmd = '/bin/ps'; -my $sshcipher; -if (defined $args{'c'}) { - $sshcipher = "-c $args{'c'}"; -} else { - $sshcipher = '-c chacha20-poly1305@openssh.com,arcfour'; -} -my $sshport = '-p 22'; -my $sshoption; -if (defined $args{'o'}) { - my @options = split(',', $args{'o'}); - foreach my $option (@options) { - $sshoption .= " -o $option"; - if ($option eq "NoneSwitch=yes") { - $sshcipher = ""; - } - } -} else { - $sshoption = ""; -} + + my $pvcmd = '/usr/bin/pv'; my $mbuffercmd = '/usr/bin/mbuffer'; my $sudocmd = '/usr/bin/sudo'; @@ -59,23 +60,24 @@ my $mbufferoptions = '-q -s 128k -m 16M 2>/dev/null'; # being present on remote machines. my $lscmd = '/bin/ls'; -if ( $args{'sshport'} ) { - $sshport = "-p $args{'sshport'}"; +if (length $args{'sshcipher'}) { + $args{'sshcipher'} = "-c $args{'sshcipher'}"; } +if (length $args{'sshport'}) { + $args{'sshport'} = "-p $args{'sshport'}"; +} +if (length $args{'sshkey'}) { + $args{'sshkey'} = "-i $args{'sshkey'}"; +} +my $sshoptions = join " ", map { "-o " . $_ } $args{'sshoption'}; + # figure out if source and/or target are remote. -if ( $args{'sshkey'} ) { - $sshcmd = "$sshcmd $sshoption $sshcipher $sshport -i $args{'sshkey'}"; -} -else { - $sshcmd = "$sshcmd $sshoption $sshcipher $sshport"; -} +$sshcmd = "$sshcmd $args{'sshcipher'} $sshoptions $args{'sshport'} $args{'sshkey'}"; my ($sourcehost,$sourcefs,$sourceisroot) = getssh($rawsourcefs); my ($targethost,$targetfs,$targetisroot) = getssh($rawtargetfs); -my $sourcesudocmd; -my $targetsudocmd; -if ($sourceisroot) { $sourcesudocmd = ''; } else { $sourcesudocmd = $sudocmd; } -if ($targetisroot) { $targetsudocmd = ''; } else { $targetsudocmd = $sudocmd; } +my $sourcesudocmd = $sourceisroot ? '' : $sudocmd; +my $targetsudocmd = $targetisroot ? '' : $sudocmd; # figure out whether compression, mbuffering, pv # are available on source, target, local machines. @@ -88,7 +90,7 @@ my %snaps; ## can loop across children separately, for recursive ## ## replication ## -if (! $args{'recursive'}) { +if (defined $args{'recursive'}) { syncdataset($sourcehost, $sourcefs, $targethost, $targetfs); } else { if ($debug) { print "DEBUG: recursive sync of $sourcefs.\n"; } @@ -161,11 +163,15 @@ sub syncdataset { %snaps = (%sourcesnaps, %targetsnaps); } - if ($args{'dumpsnaps'}) { print "merged snapshot list of $targetfs: \n"; dumphash(\%snaps); print "\n\n\n"; } + if (defined $args{'dumpsnaps'}) { + print "merged snapshot list of $targetfs: \n"; + dumphash(\%snaps); + print "\n\n\n"; + } # create a new syncoid snapshot on the source filesystem. my $newsyncsnap; - if (!defined ($args{'no-sync-snap'}) ) { + if (!defined $args{'no-sync-snap'}) { $newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot); } else { # we don't want sync snapshots created, so use the newest snapshot we can find. @@ -334,111 +340,38 @@ sub syncdataset { } # end syncdataset() - -sub getargs { - my @args = @_; - my %args; - - my %novaluearg; - my %validarg; - push my @validargs, ('debug','nocommandchecks','version','monitor-version','compress','c','o','source-bwlimit','target-bwlimit','dumpsnaps','recursive','r','sshkey','sshport','quiet','no-stream','no-sync-snap'); - foreach my $item (@validargs) { $validarg{$item} = 1; } - push my @novalueargs, ('debug','nocommandchecks','version','monitor-version','dumpsnaps','recursive','r','quiet','no-stream','no-sync-snap'); - foreach my $item (@novalueargs) { $novaluearg{$item} = 1; } - - while (my $rawarg = shift(@args)) { - my $arg = $rawarg; - my $argvalue = ''; - if ($rawarg =~ /=/) { - # user specified the value for a CLI argument with = - # instead of with blank space. separate appropriately. - $argvalue = $arg; - $arg =~ s/=.*$//; - $argvalue =~ s/^.*=//; - } - if ($rawarg =~ /^--/) { - # doubledash arg - $arg =~ s/^--//; - if (! $validarg{$arg}) { die "ERROR: don't understand argument $rawarg.\n"; } - if ($novaluearg{$arg}) { - $args{$arg} = 1; - } else { - # if this CLI arg takes a user-specified value and - # we don't already have it, then the user must have - # specified with a space, so pull in the next value - # from the array as this value rather than as the - # next argument. - if ($argvalue eq '') { $argvalue = shift(@args); } - $args{$arg} = $argvalue; - } - } elsif ($arg =~ /^-/) { - # singledash arg - $arg =~ s/^-//; - if (! $validarg{$arg}) { die "ERROR: don't understand argument $rawarg.\n"; } - if ($novaluearg{$arg}) { - $args{$arg} = 1; - } else { - # if this CLI arg takes a user-specified value and - # we don't already have it, then the user must have - # specified with a space, so pull in the next value - # from the array as this value rather than as the - # next argument. - if ($argvalue eq '') { $argvalue = shift(@args); } - $args{$arg} = $argvalue; - } - } else { - # bare arg - if (defined $args{'source'}) { - if (! defined $args{'target'}) { - $args{'target'} = $arg; - } else { - die "ERROR: don't know what to do with third bare argument $rawarg.\n"; - } - } else { - $args{'source'} = $arg; - } - } +sub compressargset { + my ($value) = @_; + my %comargs = ('rawcmd' => '', 'args' => '', 'decomrawcmd' => '', 'decomargs' => ''); + if (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'lzo', 'default', 'none'))) { + warn "Unrecognised compression value $value, defaulting to lzo"; + $value = 'default'; } - - if (defined $args{'source-bwlimit'}) { $args{'source-bwlimit'} = "-R $args{'source-bwlimit'}"; } else { $args{'source-bwlimit'} = ''; } - if (defined $args{'target-bwlimit'}) { $args{'target-bwlimit'} = "-r $args{'target-bwlimit'}"; } else { $args{'target-bwlimit'} = ''; } - - if (defined $args{'no-stream'}) { $args{'streamarg'} = '-i'; } else { $args{'streamarg'} = '-I'; } - - if ($args{'r'}) { $args{'recursive'} = $args{'r'}; } - - if (!defined $args{'compress'}) { $args{'compress'} = 'default'; } - - if ($args{'compress'} eq 'gzip') { - $args{'rawcompresscmd'} = '/bin/gzip'; - $args{'compressargs'} = '-3'; - $args{'rawdecompresscmd'} = '/bin/zcat'; - $args{'decompressargs'} = ''; - } elsif ( ($args{'compress'} eq 'pigz-fast')) { - $args{'rawcompresscmd'} = '/usr/bin/pigz'; - $args{'compressargs'} = '-3'; - $args{'rawdecompresscmd'} = '/usr/bin/pigz'; - $args{'decompressargs'} = '-dc'; - } elsif ( ($args{'compress'} eq 'pigz-slow')) { - $args{'rawcompresscmd'} = '/usr/bin/pigz'; - $args{'compressargs'} = '-9'; - $args{'rawdecompresscmd'} = '/usr/bin/pigz'; - $args{'decompressargs'} = '-dc'; - } elsif ( ($args{'compress'} eq 'lzo') || ($args{'compress'} eq 'default') ) { - $args{'rawcompresscmd'} = '/usr/bin/lzop'; - $args{'compressargs'} = ''; - $args{'rawdecompresscmd'} = '/usr/bin/lzop'; - $args{'decompressargs'} = '-dfc'; - } else { - $args{'rawcompresscmd'} = ''; - $args{'compressargs'} = ''; - $args{'rawdecompresscmd'} = ''; - $args{'decompressargs'} = ''; + if ($value eq 'gzip') { + $comargs{'rawcmd'} = '/bin/gzip'; + $comargs{'args'} = '-3'; + $comargs{'decomrawcmd'} = '/bin/zcat'; + $comargs{'decomargs'} = ''; + } elsif ($value eq 'pigz-fast') { + $comargs{'rawcmd'} = '/usr/bin/pigz'; + $comargs{'args'} = '-3'; + $comargs{'decomrawcmd'} = '/usr/bin/pigz'; + $comargs{'decomargs'} = '-dc'; + } elsif ($value eq 'pigz-slow') { + $comargs{'rawcmd'} = '/usr/bin/pigz'; + $comargs{'args'} = '-9'; + $comargs{'decomrawcmd'} = '/usr/bin/pigz'; + $comargs{'decomargs'} = '-dc'; + } elsif (($value eq 'lzo') || ($value eq 'default') ) { + $comargs{'rawcmd'} = '/usr/bin/lzop'; + $comargs{'args'} = ''; + $comargs{'decomrawcmd'} = '/usr/bin/lzop'; + $comargs{'decomargs'} = '-dfc'; } - $args{'compresscmd'} = "$args{'rawcompresscmd'} $args{'compressargs'}"; - $args{'decompresscmd'} = "$args{'rawdecompresscmd'} $args{'decompressargs'}"; - - return %args; + $comargs{'compress'} = $value; + $comargs{'cmd'} = "$comargs{'rawcmd'} $comargs{'args'}"; + $comargs{'decomcmd'} = "$comargs{'decomrawcmd'} $comargs{'decomargs'}"; + return \%comargs; } sub checkcommands { @@ -468,24 +401,15 @@ sub checkcommands { # if raw compress command is null, we must have specified no compression. otherwise, # make sure that compression is available everywhere we need it - if ($args{'rawcompresscmd'} eq '') { - $avail{'sourcecompress'} = 0; - $avail{'sourcecompress'} = 0; - $avail{'localcompress'} = 0; - if ($args{'compress'} eq 'none' || - $args{'compress'} eq 'no' || - $args{'compress'} eq '0') { - if ($debug) { print "DEBUG: compression forced off from command line arguments.\n"; } - } else { - print "WARN: value $args{'compress'} for argument --compress not understood, proceeding without compression.\n"; - } + if ($args{'compress'}{'compress'} eq 'none') { + if ($debug) { print "DEBUG: compression forced off from command line arguments.\n"; } } else { - if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on source...\n"; } - $avail{'sourcecompress'} = `$sourcessh $lscmd $args{'rawcompresscmd'} 2>/dev/null`; - if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on target...\n"; } - $avail{'targetcompress'} = `$targetssh $lscmd $args{'rawcompresscmd'} 2>/dev/null`; - if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on local machine...\n"; } - $avail{'localcompress'} = `$lscmd $args{'rawcompresscmd'} 2>/dev/null`; + if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on source...\n"; } + $avail{'sourcecompress'} = `$sourcessh $lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`; + if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on target...\n"; } + $avail{'targetcompress'} = `$targetssh $lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`; + if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on local machine...\n"; } + $avail{'localcompress'} = `$lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`; } my ($s,$t); @@ -511,14 +435,14 @@ sub checkcommands { if ($avail{'sourcecompress'} eq '') { - if ($args{'rawcompresscmd'} ne '') { - print "WARN: $args{'compresscmd'} not available on source $s- sync will continue without compression.\n"; + if ($args{'compress'}{'rawcmd'} ne '') { + print "WARN: $args{'compress'}{'rawcmd'} not available on source $s- sync will continue without compression.\n"; } $avail{'compress'} = 0; } if ($avail{'targetcompress'} eq '') { - if ($args{'rawcompresscmd'} ne '') { - print "WARN: $args{'compresscmd'} not available on target $t - sync will continue without compression.\n"; + if ($args{'compress'}{'rawcmd'} ne '') { + print "WARN: $args{'compress'}{'rawcmd'} not available on target $t - sync will continue without compression.\n"; } $avail{'compress'} = 0; } @@ -530,8 +454,8 @@ sub checkcommands { # corner case - if source AND target are BOTH remote, we have to check for local compress too if ($sourcehost ne '' && $targethost ne '' && $avail{'localcompress'} eq '') { - if ($args{'rawcompresscmd'} ne '') { - print "WARN: $args{'compresscmd'} not available on local machine - sync will continue without compression.\n"; + if ($args{'compress'}{'rawcmd'} ne '') { + print "WARN: $args{'compress'}{'rawcmd'} not available on local machine - sync will continue without compression.\n"; } $avail{'compress'} = 0; } @@ -687,9 +611,9 @@ sub buildsynccmd { $synccmd = "$sendcmd |"; # avoid confusion - accept either source-bwlimit or target-bwlimit as the bandwidth limiting option here my $bwlimit = ''; - if (defined $args{'source-bwlimit'}) { + if (length $args{'bwlimit'}) { $bwlimit = $args{'source-bwlimit'}; - } elsif (defined $args{'target-bwlimit'}) { + } elsif (length $args{'target-bwlimit'}) { $bwlimit = $args{'target-bwlimit'}; } @@ -698,18 +622,18 @@ sub buildsynccmd { $synccmd .= " $recvcmd"; } elsif ($sourcehost eq '') { # local source, remote target. - #$synccmd = "$sendcmd | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'"; + #$synccmd = "$sendcmd | $pvcmd | $args{'compress'}{'cmd'} | $mbuffercmd | $sshcmd $targethost '$args{'compress'}{'decomcmd'} | $mbuffercmd | $recvcmd'"; $synccmd = "$sendcmd |"; if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; } - if ($avail{'compress'}) { $synccmd .= " $args{'compresscmd'} |"; } + if ($avail{'compress'}) { $synccmd .= " $args{'compress'}{'cmd'} |"; } if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; } $synccmd .= " $sshcmd $targethost '"; if ($avail{'targetmbuffer'}) { $synccmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; } - if ($avail{'compress'}) { $synccmd .= " $args{'decompresscmd'} |"; } + if ($avail{'compress'}) { $synccmd .= " $args{'compress'}{'decomcmd'} |"; } $synccmd .= " $recvcmd'"; } elsif ($targethost eq '') { # remote source, local target. - #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $mbuffercmd | $pvcmd | $recvcmd"; + #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compress'}{'cmd'} | $mbuffercmd' | $args{'decompress'}{'cmd'} | $mbuffercmd | $pvcmd | $recvcmd"; $synccmd = "$sshcmd $sourcehost '$sendcmd"; if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; } if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } @@ -720,18 +644,18 @@ sub buildsynccmd { $synccmd .= "$recvcmd"; } else { #remote source, remote target... weird, but whatever, I'm not here to judge you. - #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'"; + #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compress'}{'cmd'} | $mbuffercmd' | $args{'compress'}{'decomcmd'} | $pvcmd | $args{'compress'}{'cmd'} | $mbuffercmd | $sshcmd $targethost '$args{'compress'}{'decomcmd'} | $mbuffercmd | $recvcmd'"; $synccmd = "$sshcmd $sourcehost '$sendcmd"; - if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; } + if ($avail{'compress'}) { $synccmd .= " | $args{'compress'}{'cmd'}"; } if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } $synccmd .= "' | "; - if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'decomcmd'} | "; } if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } - if ($avail{'compress'}) { $synccmd .= "$args{'compresscmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'cmd'} | "; } if ($avail{'localmbuffer'}) { $synccmd .= "$mbuffercmd $mbufferoptions | "; } $synccmd .= "$sshcmd $targethost '"; if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } - if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'decomcmd'} | "; } $synccmd .= "$recvcmd'"; } return $synccmd; @@ -865,7 +789,7 @@ sub getssh { if ($remoteuser eq 'root') { $isroot = 1; } else { $isroot = 0; } # now we need to establish a persistent master SSH connection $socket = "/tmp/syncoid-$remoteuser-$rhost-" . time(); - open FH, "$sshcmd -M -S $socket -o ControlPersist=1m $sshport $rhost exit |"; + open FH, "$sshcmd -M -S $socket -o ControlPersist=1m $args{'sshport'} $rhost exit |"; close FH; $rhost = "-S $socket $rhost"; } else { @@ -985,4 +909,39 @@ sub getdate { return %date; } +__END__ + +=head1 NAME + +syncoid - ZFS snapshot replication tool + +=head1 SYNOPSIS + + syncoid [options]... SOURCE TARGET + or syncoid [options]... SOURCE [USER@]HOST:TARGET + or syncoid [options]... [USER@]HOST:SOURCE [TARGET] + or syncoid [options]... [USER@]HOST:SOURCE [USER@]HOST:TARGET + + SOURCE Source ZFS dataset. Can be either local or remote + TARGET Target ZFS dataset. Can be either local or remote + +Options: + + --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none + --recursive|r Also transfers child datasets + --source-bwlimit= Bandwidth limit on the source transfer + --target-bwlimit= Bandwidth limit on the target transfer + + --sshkey=FILE Specifies a ssh public key to use to connect + --sshport=PORT Connects to remote on a particular port + --sshcipher|c=CIPHER Passes CIPHER to ssh to use a particular cipher set + --sshoption|o=OPTION Passes OPTION to ssh for remote usage. Can be specified multiple times + + --help Prints this helptext + --verbose Prints the version number + --debug Prints out a lot of additional information during a syncoid run + --monitor-version Currently does nothing + --quiet Suppresses non-error output + --dumpsnaps Dumps a list of snapshots during the run + --no-command-checks Do not check command existence before attempting transfer. Not recommended From 5849285815a7590515b1e75e76f98133dd21a891 Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Tue, 15 Aug 2017 16:05:06 +0100 Subject: [PATCH 04/15] Fix sanoid default state Fixes #134 --- sanoid | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 83bce62..0904c73 100755 --- a/sanoid +++ b/sanoid @@ -15,12 +15,18 @@ use Getopt::Long qw(:config auto_version auto_help); use Pod::Usage; # pod2usage use Time::Local; # to parse dates in reverse -my %args = ("cron" => 1, "verbose" => 1, "configdir" => "/etc/sanoid"); +my %args = ("configdir" => "/etc/sanoid"); GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet", "monitor-health", "force-update", "configdir=s", "monitor-snapshots", "take-snapshots", "prune-snapshots" ) or pod2usage(2); +# If only config directory (or nothing) has been specified, default to --cron --verbose +if (keys %args < 2) { + $args{'cron'} = 1; + $args{'verbose'} = 1; +} + my $pscmd = '/bin/ps'; my $zfs = '/sbin/zfs'; @@ -1060,6 +1066,8 @@ sanoid - ZFS snapshot management and replication tool sanoid [options] +Assumes --cron --verbose if no other arguments (other than configdir) are specified + Options: --configdir=DIR Specify a directory to find config file sanoid.conf From ecde5acf4ca7afaa84cc0106911e4cac7c8cd795 Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Tue, 5 Sep 2017 15:10:10 +0100 Subject: [PATCH 05/15] An array needed a deref, and an if condition got inverted --- syncoid | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index 8fa0453..c8d915f 100755 --- a/syncoid +++ b/syncoid @@ -69,10 +69,11 @@ if (length $args{'sshport'}) { if (length $args{'sshkey'}) { $args{'sshkey'} = "-i $args{'sshkey'}"; } -my $sshoptions = join " ", map { "-o " . $_ } $args{'sshoption'}; +my $sshoptions = join " ", map { "-o " . $_ } @{$args{'sshoption'}}; # deref required # figure out if source and/or target are remote. $sshcmd = "$sshcmd $args{'sshcipher'} $sshoptions $args{'sshport'} $args{'sshkey'}"; +if ($debug) { print "DEBUG: SSHCMD: $sshcmd\n"; } my ($sourcehost,$sourcefs,$sourceisroot) = getssh($rawsourcefs); my ($targethost,$targetfs,$targetisroot) = getssh($rawtargetfs); @@ -90,7 +91,7 @@ my %snaps; ## can loop across children separately, for recursive ## ## replication ## -if (defined $args{'recursive'}) { +if (!defined $args{'recursive'}) { syncdataset($sourcehost, $sourcefs, $targethost, $targetfs); } else { if ($debug) { print "DEBUG: recursive sync of $sourcefs.\n"; } From 8b7dee9ee8a5192666db2729c0d8e08c80727105 Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Wed, 6 Sep 2017 12:48:56 +0100 Subject: [PATCH 06/15] Typo in helptext --- sanoid | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 0904c73..aec0fab 100755 --- a/sanoid +++ b/sanoid @@ -1072,6 +1072,7 @@ Options: --configdir=DIR Specify a directory to find config file sanoid.conf --cron Creates snapshots and purges expired snapshots + --verbose Prints out additional information during a sanoid run --readonly Simulates creation/deletion of snapshots --quiet Suppresses non-error output --force-update Clears out sanoid's zfs snapshot cache @@ -1082,5 +1083,5 @@ Options: --prune-snapshots Purges expired snapshots as specified in sanoid.conf --help Prints this helptext - --verbose Prints the version number + --version Prints the version number --debug Prints out a lot of additional information during a sanoid run From 871bf6ebd2d3482b56473e9f18a4c9c54c609eb1 Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Wed, 6 Sep 2017 12:52:53 +0100 Subject: [PATCH 07/15] Fix undefined variable --- sanoid | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sanoid b/sanoid index aec0fab..51caf8a 100755 --- a/sanoid +++ b/sanoid @@ -136,8 +136,10 @@ sub monitor_snapshots() { my $warn = $config{$section}{$typewarn} * $smallerperiod; my $crit = $config{$section}{$typecrit} * $smallerperiod; my $elapsed = -1; - if (defined $snapsbytype{$path}{$type}{'newest'}) { $elapsed = $snapsbytype{$path}{$type}{'newest'}; } - my $dispelapsed = displaytime($snapsbytype{$path}{$type}{'newest'}); + if (defined $snapsbytype{$path}{$type}{'newest'}) { + $elapsed = $snapsbytype{$path}{$type}{'newest'}; + } + my $dispelapsed = displaytime($elapsed); my $dispwarn = displaytime($warn); my $dispcrit = displaytime($crit); if ( $elapsed > $crit || $elapsed == -1) { @@ -681,7 +683,7 @@ sub get_date { sub displaytime { # take a time in seconds, return it in human readable form - my $elapsed = shift; + my ($elapsed) = @_; my $days = int ($elapsed / 60 / 60 / 24); $elapsed -= $days * 60 * 60 * 24; From 5487c3835027a0b448cd1edc272d9f52d7b97c16 Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Thu, 7 Sep 2017 17:08:10 +0100 Subject: [PATCH 08/15] Remove semicolons from defaults file --- sanoid.defaults.conf | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 15d1a0c..74ab6b0 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -34,21 +34,21 @@ min_percent_free = 10 # Note that we will not take snapshots for a given type if that type is set to 0 above, # regardless of the autosnap setting - for example, if yearly=0 we will not take yearlies # even if we've defined a preferred time for yearlies and autosnap is on. -autosnap = 1; +autosnap = 1 # hourly - top of the hour -hourly_min = 0; +hourly_min = 0 # daily - at 23:59 (most people expect a daily to contain everything done DURING that day) -daily_hour = 23; -daily_min = 59; +daily_hour = 23 +daily_min = 59 # monthly - immediately at the beginning of the month (ie 00:00 of day 1) -monthly_mday = 1; -monthly_hour = 0; -monthly_min = 0; +monthly_mday = 1 +monthly_hour = 0 +monthly_min = 0 # yearly - immediately at the beginning of the year (ie 00:00 on Jan 1) -yearly_mon = 1; -yearly_mday = 1; -yearly_hour = 0; -yearly_min = 0; +yearly_mon = 1 +yearly_mday = 1 +yearly_hour = 0 +yearly_min = 0 # monitoring plugin - define warn / crit levels for each snapshot type by age, in units of one period down # example hourly_warn = 90 means issue WARNING if most recent hourly snapshot is not less than 90 minutes old, From 8896059fe433d652a9b8a3579a563709b232c4b9 Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Thu, 7 Sep 2017 17:43:33 +0100 Subject: [PATCH 09/15] Remove unnecessary (and warning-causing) prototypes --- sanoid | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sanoid b/sanoid index 51caf8a..1a8f51e 100755 --- a/sanoid +++ b/sanoid @@ -70,7 +70,7 @@ exit 0; #################################################################################### #################################################################################### -sub monitor_health() { +sub monitor_health { my ($config, $snaps, $snapsbytype, $snapsbypath) = @_; my %pools; my @messages; @@ -93,13 +93,13 @@ sub monitor_health() { print "$message\n"; exit $errlevel; -} # end monitor_health() +} #################################################################################### #################################################################################### #################################################################################### -sub monitor_snapshots() { +sub monitor_snapshots { # nagios plugin format: exit 0,1,2,3 for OK, WARN, CRITICAL, or ERROR. @@ -172,7 +172,7 @@ sub monitor_snapshots() { print "$msg\n"; exit $errorlevel; -} # end monitor() +} #################################################################################### #################################################################################### From f04c71604d5a9b4309c7218ddaeba487a3770897 Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Thu, 12 Oct 2017 14:54:36 +0100 Subject: [PATCH 10/15] Missed a couple of syncoid options --- syncoid | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/syncoid b/syncoid index c8d915f..e52958e 100755 --- a/syncoid +++ b/syncoid @@ -932,6 +932,8 @@ Options: --recursive|r Also transfers child datasets --source-bwlimit= Bandwidth limit on the source transfer --target-bwlimit= Bandwidth limit on the target transfer + --no-stream Replicates using newest snapshot instead of intermediates + --no-sync-snap Does not create new snapshot, only transfers existing --sshkey=FILE Specifies a ssh public key to use to connect --sshport=PORT Connects to remote on a particular port @@ -945,4 +947,3 @@ Options: --quiet Suppresses non-error output --dumpsnaps Dumps a list of snapshots during the run --no-command-checks Do not check command existence before attempting transfer. Not recommended - From d09c361e420b473d1e0ff45c82a116be828a032f Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Thu, 12 Oct 2017 14:54:58 +0100 Subject: [PATCH 11/15] Added newline to fix sanoid helptext formatting --- sanoid | 1 + 1 file changed, 1 insertion(+) diff --git a/sanoid b/sanoid index 1a8f51e..f29cca7 100755 --- a/sanoid +++ b/sanoid @@ -1071,6 +1071,7 @@ sanoid [options] Assumes --cron --verbose if no other arguments (other than configdir) are specified Options: + --configdir=DIR Specify a directory to find config file sanoid.conf --cron Creates snapshots and purges expired snapshots From dd098b806a9bccef2dfd756e0f4b75fb35394e8d Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Thu, 12 Oct 2017 14:55:17 +0100 Subject: [PATCH 12/15] Updated readme as appropriate --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5618c0c..78cf7de 100644 --- a/README.md +++ b/README.md @@ -111,11 +111,11 @@ Syncoid supports recursive replication (replication of a dataset and all its chi ##### Syncoid Command Line Options -+ --[source] ++ [source] This is the source dataset. It can be either local or remote. -+ --[destination] ++ [destination] This is the destination dataset. It can be either local or remote. @@ -125,7 +125,7 @@ Syncoid supports recursive replication (replication of a dataset and all its chi + --compress - Currently accepts gzip and lzo. lzo is fast and light on the processsor and is the default. If the selected compression method is unavailable on the source and destination, no compression will be used. + Currently accepted options: gzip, pigz-fast, pigz-slow, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. + --source-bwlimit @@ -135,7 +135,7 @@ Syncoid supports recursive replication (replication of a dataset and all its chi This is the bandwidth limit imposed upon the target. This is mainly used if the source does not have mbuffer installed, but bandwidth limites are desired. -+ --nocommandchecks ++ --no-command-checks Do not check the existance of commands before attempting the transfer. It assumes all programs are available. This should never be used. From 7756dff623d830316b4e5d0936da9babaedf50e0 Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Thu, 12 Oct 2017 15:14:03 +0100 Subject: [PATCH 13/15] Format compression as a hash of hashes, rather than a large if block --- syncoid | 62 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/syncoid b/syncoid index e52958e..e64b50d 100755 --- a/syncoid +++ b/syncoid @@ -343,32 +343,46 @@ sub syncdataset { sub compressargset { my ($value) = @_; - my %comargs = ('rawcmd' => '', 'args' => '', 'decomrawcmd' => '', 'decomargs' => ''); + my $DEFAULT_COMPRESSION = 'lzo'; + my %COMPRESS_ARGS = ( + 'none' => { + rawcmd => '', + args => '', + decomrawcmd => '', + decomargs => '', + }, + 'gzip' => { + rawcmd => '/bin/gzip', + args => '-3', + decomrawcmd => '/bin/zcat', + decomargs => '', + }, + 'pigz-fast' => { + rawcmd => '/usr/bin/pigz', + args => '-3', + decomrawcmd => '/usr/bin/pigz', + decomargs => '-dc', + }, + 'pigz-slow' => { + rawcmd => '/usr/bin/pigz', + args => '-9', + decomrawcmd => '/usr/bin/pigz', + decomargs => '-dc', + }, + 'lzo' => { + rawcmd => '/usr/bin/lzop', + args => '', + decomrawcmd => '/usr/bin/lzop', + decomargs => '-dfc', + }, + ); + if (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'lzo', 'default', 'none'))) { - warn "Unrecognised compression value $value, defaulting to lzo"; - $value = 'default'; - } - if ($value eq 'gzip') { - $comargs{'rawcmd'} = '/bin/gzip'; - $comargs{'args'} = '-3'; - $comargs{'decomrawcmd'} = '/bin/zcat'; - $comargs{'decomargs'} = ''; - } elsif ($value eq 'pigz-fast') { - $comargs{'rawcmd'} = '/usr/bin/pigz'; - $comargs{'args'} = '-3'; - $comargs{'decomrawcmd'} = '/usr/bin/pigz'; - $comargs{'decomargs'} = '-dc'; - } elsif ($value eq 'pigz-slow') { - $comargs{'rawcmd'} = '/usr/bin/pigz'; - $comargs{'args'} = '-9'; - $comargs{'decomrawcmd'} = '/usr/bin/pigz'; - $comargs{'decomargs'} = '-dc'; - } elsif (($value eq 'lzo') || ($value eq 'default') ) { - $comargs{'rawcmd'} = '/usr/bin/lzop'; - $comargs{'args'} = ''; - $comargs{'decomrawcmd'} = '/usr/bin/lzop'; - $comargs{'decomargs'} = '-dfc'; + warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION"; + $value = $DEFAULT_COMPRESSION; } + + my %comargs = %{$COMPRESS_ARGS{$value}}; # copy $comargs{'compress'} = $value; $comargs{'cmd'} = "$comargs{'rawcmd'} $comargs{'args'}"; $comargs{'decomcmd'} = "$comargs{'decomrawcmd'} $comargs{'decomargs'}"; From c416be3eab42ad890d28f7f392e57bfb3b843d1d Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Thu, 12 Oct 2017 15:31:12 +0100 Subject: [PATCH 14/15] Remember to set default compression properly --- syncoid | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/syncoid b/syncoid index e64b50d..30bf712 100755 --- a/syncoid +++ b/syncoid @@ -377,7 +377,9 @@ sub compressargset { }, ); - if (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'lzo', 'default', 'none'))) { + if ($value eq 'default') { + $value = $DEFAULT_COMPRESSION; + } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'lzo', 'default', 'none'))) { warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION"; $value = $DEFAULT_COMPRESSION; } From 49758fe8b07b4e8a8118c8c9f6323231b2b76ebc Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Mon, 16 Oct 2017 10:20:03 +0100 Subject: [PATCH 15/15] Fix warning when cache file doesn't exist --- sanoid | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sanoid b/sanoid index f29cca7..acc460d 100755 --- a/sanoid +++ b/sanoid @@ -487,11 +487,9 @@ sub getsnaps { my $cache = '/var/cache/sanoidsnapshots.txt'; my @rawsnaps; - my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, - $atime,$mtime,$ctime,$blksize,$blocks) - = stat($cache); + my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, $mtime, $ctime, $blksize, $blocks) = stat($cache); - if ( $forcecacheupdate || (time() - $mtime) > $cacheTTL ) { + if ( $forcecacheupdate || ! -f $cache || (time() - $mtime) > $cacheTTL ) { if (checklock('sanoid_cacheupdate')) { writelock('sanoid_cacheupdate'); if ($args{'verbose'}) {