Merge pull request #131 from LordAro/helptext

Rewrite argument parser with GetOpt::Long and add helptext
This commit is contained in:
Jim Salter 2017-11-08 10:04:59 -05:00 committed by GitHub
commit 06e327e1f3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 282 additions and 324 deletions

View File

@ -111,11 +111,11 @@ Syncoid supports recursive replication (replication of a dataset and all its chi
##### Syncoid Command Line Options
+ --[source]
+ [source]
This is the source dataset. It can be either local or remote.
+ --[destination]
+ [destination]
This is the destination dataset. It can be either local or remote.
@ -125,7 +125,7 @@ Syncoid supports recursive replication (replication of a dataset and all its chi
+ --compress <compression type>
Currently accepts gzip and lzo. lzo is fast and light on the processsor and is the default. If the selected compression method is unavailable on the source and destination, no compression will be used.
Currently accepted options: gzip, pigz-fast, pigz-slow, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used.
+ --source-bwlimit <limit t|g|m|k>
@ -135,7 +135,7 @@ Syncoid supports recursive replication (replication of a dataset and all its chi
This is the bandwidth limit imposed upon the target. This is mainly used if the source does not have mbuffer installed, but bandwidth limites are desired.
+ --nocommandchecks
+ --no-command-checks
Do not check the existance of commands before attempting the transfer. It assumes all programs are available. This should never be used.

241
sanoid
View File

@ -4,22 +4,33 @@
# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this
# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
my $version = '1.4.17';
$::VERSION = '1.4.17';
use strict;
use Config::IniFiles; # read samba-style conf file
use File::Path; # for rmtree command in use_prune
use Data::Dumper; # debugging - print contents of hash
use Time::Local; # to parse dates in reverse
use warnings;
use Config::IniFiles; # read samba-style conf file
use Data::Dumper; # debugging - print contents of hash
use File::Path; # for rmtree command in use_prune
use Getopt::Long qw(:config auto_version auto_help);
use Pod::Usage; # pod2usage
use Time::Local; # to parse dates in reverse
# parse CLI arguments
my %args = getargs(@ARGV);
my %args = ("configdir" => "/etc/sanoid");
GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet",
"monitor-health", "force-update", "configdir=s",
"monitor-snapshots", "take-snapshots", "prune-snapshots"
) or pod2usage(2);
# If only config directory (or nothing) has been specified, default to --cron --verbose
if (keys %args < 2) {
$args{'cron'} = 1;
$args{'verbose'} = 1;
}
my $pscmd = '/bin/ps';
my $zfs = '/sbin/zfs';
if ($args{'configdir'} eq '') { $args{'configdir'} = '/etc/sanoid'; }
my $conf_file = "$args{'configdir'}/sanoid.conf";
my $default_conf_file = "$args{'configdir'}/sanoid.defaults.conf";
@ -42,11 +53,9 @@ if ($args{'debug'}) { $args{'verbose'}=1; blabber (@params); }
if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); }
if ($args{'monitor-health'}) { monitor_health(@params); }
if ($args{'force-update'}) { my $snaps = getsnaps( \%config, $cacheTTL, 1 ); }
if ($args{'version'}) { print "INFO: Sanoid version: $version\n"; }
if ($args{'cron'} || $args{'noargs'}) {
if ($args{'noargs'}) { print "INFO: No arguments given - assuming --cron and --verbose.\n"; }
if (!$args{'quiet'}) { $args{'verbose'} = 1; }
if ($args{'cron'}) {
if ($args{'quiet'}) { $args{'verbose'} = 0; }
take_snapshots (@params);
prune_snapshots (@params);
} else {
@ -61,7 +70,7 @@ exit 0;
####################################################################################
####################################################################################
sub monitor_health() {
sub monitor_health {
my ($config, $snaps, $snapsbytype, $snapsbypath) = @_;
my %pools;
my @messages;
@ -84,13 +93,13 @@ sub monitor_health() {
print "$message\n";
exit $errlevel;
} # end monitor_health()
}
####################################################################################
####################################################################################
####################################################################################
sub monitor_snapshots() {
sub monitor_snapshots {
# nagios plugin format: exit 0,1,2,3 for OK, WARN, CRITICAL, or ERROR.
@ -127,8 +136,10 @@ sub monitor_snapshots() {
my $warn = $config{$section}{$typewarn} * $smallerperiod;
my $crit = $config{$section}{$typecrit} * $smallerperiod;
my $elapsed = -1;
if (defined $snapsbytype{$path}{$type}{'newest'}) { $elapsed = $snapsbytype{$path}{$type}{'newest'}; }
my $dispelapsed = displaytime($snapsbytype{$path}{$type}{'newest'});
if (defined $snapsbytype{$path}{$type}{'newest'}) {
$elapsed = $snapsbytype{$path}{$type}{'newest'};
}
my $dispelapsed = displaytime($elapsed);
my $dispwarn = displaytime($warn);
my $dispcrit = displaytime($crit);
if ( $elapsed > $crit || $elapsed == -1) {
@ -161,7 +172,7 @@ sub monitor_snapshots() {
print "$msg\n";
exit $errorlevel;
} # end monitor()
}
####################################################################################
####################################################################################
@ -193,45 +204,48 @@ sub prune_snapshots {
elsif ($type eq 'daily') { $period = 60*60*24; }
elsif ($type eq 'monthly') { $period = 60*60*24*31; }
elsif ($type eq 'yearly') { $period = 60*60*24*365.25; }
# avoid pissing off use warnings by not executing this block if no matching snaps exist
if (defined $snapsbytype{$path}{$type}{'sorted'}) {
my @sorted = split (/\|/,$snapsbytype{$path}{$type}{'sorted'});
my @sorted = split (/\|/,$snapsbytype{$path}{$type}{'sorted'});
# if we say "daily=30" we really mean "don't keep any dailies more than 30 days old", etc
my $maxage = ( time() - $config{$section}{$type} * $period );
# but if we say "daily=30" we ALSO mean "don't get rid of ANY dailies unless we have more than 30".
my $minsnapsthistype = $config{$section}{$type};
# if we say "daily=30" we really mean "don't keep any dailies more than 30 days old", etc
my $maxage = ( time() - $config{$section}{$type} * $period );
# but if we say "daily=30" we ALSO mean "don't get rid of ANY dailies unless we have more than 30".
my $minsnapsthistype = $config{$section}{$type};
# how many total snaps of this type do we currently have?
my $numsnapsthistype = scalar (@sorted);
# how many total snaps of this type do we currently have?
my $numsnapsthistype = scalar (@sorted);
my @prunesnaps;
foreach my $snap( @sorted ){
# print "snap $path\@$snap has age $snaps{$path}{$snap}{'ctime'}, maxage is $maxage.\n";
if ( ($snaps{$path}{$snap}{'ctime'} < $maxage) && ($numsnapsthistype > $minsnapsthistype) ) {
my $fullpath = $path . '@' . $snap;
push(@prunesnaps,$fullpath);
# we just got rid of a snap, so we now have one fewer, duh
$numsnapsthistype--;
}
}
if ((scalar @prunesnaps) > 0) {
# print "found some snaps to prune!\n"
if (checklock('sanoid_pruning')) {
writelock('sanoid_pruning');
foreach my $snap( @prunesnaps ){
if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; }
if (iszfsbusy($path)) {
print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n";
} else {
if (! $args{'readonly'}) { system($zfs, "destroy",$snap) == 0 or warn "could not remove $snap : $?"; }
}
my @prunesnaps;
foreach my $snap( @sorted ){
# print "snap $path\@$snap has age $snaps{$path}{$snap}{'ctime'}, maxage is $maxage.\n";
if ( ($snaps{$path}{$snap}{'ctime'} < $maxage) && ($numsnapsthistype > $minsnapsthistype) ) {
my $fullpath = $path . '@' . $snap;
push(@prunesnaps,$fullpath);
# we just got rid of a snap, so we now have one fewer, duh
$numsnapsthistype--;
}
}
if ((scalar @prunesnaps) > 0) {
# print "found some snaps to prune!\n"
if (checklock('sanoid_pruning')) {
writelock('sanoid_pruning');
foreach my $snap( @prunesnaps ){
if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; }
if (iszfsbusy($path)) {
print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n";
} else {
if (! $args{'readonly'}) { system($zfs, "destroy",$snap) == 0 or warn "could not remove $snap : $?"; }
}
}
removelock('sanoid_pruning');
$forcecacheupdate = 1;
%snaps = getsnaps(%config,$cacheTTL,$forcecacheupdate);
} else {
print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n";
}
removelock('sanoid_pruning');
$forcecacheupdate = 1;
%snaps = getsnaps(%config,$cacheTTL,$forcecacheupdate);
} else {
print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n";
}
}
}
@ -473,11 +487,9 @@ sub getsnaps {
my $cache = '/var/cache/sanoidsnapshots.txt';
my @rawsnaps;
my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
$atime,$mtime,$ctime,$blksize,$blocks)
= stat($cache);
my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, $mtime, $ctime, $blksize, $blocks) = stat($cache);
if ( $forcecacheupdate || (time() - $mtime) > $cacheTTL ) {
if ( $forcecacheupdate || ! -f $cache || (time() - $mtime) > $cacheTTL ) {
if (checklock('sanoid_cacheupdate')) {
writelock('sanoid_cacheupdate');
if ($args{'verbose'}) {
@ -510,10 +522,14 @@ sub getsnaps {
foreach my $snap (@rawsnaps) {
my ($fs,$snapname,$snapdate) = ($snap =~ m/(.*)\@(.*ly)\s*creation\s*(\d*)/);
my ($snaptype) = ($snapname =~ m/.*_(\w*ly)/);
if ($snapname =~ /^autosnap/) {
$snaps{$fs}{$snapname}{'ctime'}=$snapdate;
$snaps{$fs}{$snapname}{'type'}=$snaptype;
# avoid pissing off use warnings
if (defined $snapname) {
my ($snaptype) = ($snapname =~ m/.*_(\w*ly)/);
if ($snapname =~ /^autosnap/) {
$snaps{$fs}{$snapname}{'ctime'}=$snapdate;
$snaps{$fs}{$snapname}{'type'}=$snaptype;
}
}
}
@ -601,10 +617,10 @@ sub init {
# make sure that true values are true and false values are false for any toggled values
foreach my $toggle(@toggles) {
foreach my $true (@istrue) {
if ($config{$section}{$toggle} eq $true) { $config{$section}{$toggle} = 1; }
if (defined $config{$section}{$toggle} && $config{$section}{$toggle} eq $true) { $config{$section}{$toggle} = 1; }
}
foreach my $false (@isfalse) {
if ($config{$section}{$toggle} eq $false) { $config{$section}{$toggle} = 0; }
if (defined $config{$section}{$toggle} && $config{$section}{$toggle} eq $false) { $config{$section}{$toggle} = 0; }
}
}
@ -665,7 +681,7 @@ sub get_date {
sub displaytime {
# take a time in seconds, return it in human readable form
my $elapsed = shift;
my ($elapsed) = @_;
my $days = int ($elapsed / 60 / 60 / 24);
$elapsed -= $days * 60 * 60 * 24;
@ -1026,75 +1042,10 @@ sub iszfsbusy {
#######################################################################################################################3
#######################################################################################################################3
sub getargs {
my @args = @_;
my %args;
my @validargs;
my @novalueargs;
my %validargs;
my %novalueargs;
push my @validargs, 'verbose','debug','version','monitor-health','monitor-snapshots','force-update','cron','take-snapshots','prune-snapshots','readonly','configdir','quiet';
push my @novalueargs, 'verbose','debug','version','monitor-health','monitor-snapshots','force-update','cron','take-snapshots','prune-snapshots','readonly','quiet';
foreach my $item (@validargs) { $validargs{$item}=1; }
foreach my $item (@novalueargs) { $novalueargs{$item}=1; }
if (! (scalar @args)) {
$args{'noargs'} = 1;
}
while (my $rawarg = shift(@args)) {
my $argvalue;
my $arg = $rawarg;
if ($rawarg =~ /=/) {
# user specified the value for a CLI argument with =
# instead of with blank space. separate appropriately.
$argvalue = $arg;
$arg =~ s/=.*$//;
$argvalue =~ s/^.*=//;
}
if ($rawarg =~ /^--/) {
# doubledash arg
$arg =~ s/^--//;
if ($novalueargs{$arg}) {
$args{$arg} = 1;
} else {
# if this CLI arg takes a user-specified value and
# we don't already have it, then the user must have
# specified with a space, so pull in the next value
# from the array as this value rather than as the
# next argument.
if ($argvalue eq '') { $argvalue = shift(@args); }
$args{$arg} = $argvalue;
}
} elsif ($rawarg =~ /^-/) {
# singledash arg
$arg =~ s/^-//;
if ($novalueargs{$arg}) {
$args{$arg} = 1;
} else {
# if this CLI arg takes a user-specified value and
# we don't already have it, then the user must have
# specified with a space, so pull in the next value
# from the array as this value rather than as the
# next argument.
if ($argvalue eq '') { $argvalue = shift(@args); }
$args{$arg} = $argvalue;
}
} else {
# bare arg
die "ERROR: don't know what to do with bare argument $rawarg.\n";
}
if (! ($validargs{$arg})) { die "ERROR: don't understand argument $rawarg.\n"; }
}
return %args;
}
sub getchilddatasets {
# for later, if we make sanoid itself support sudo use
my $fs = shift;
my $mysudocmd;
my $mysudocmd = '';
my $getchildrencmd = "$mysudocmd $zfs list -o name -Hr $fs |";
if ($args{'debug'}) { print "DEBUG: getting list of child datasets on $fs using $getchildrencmd...\n"; }
@ -1105,3 +1056,33 @@ sub getchilddatasets {
return @children;
}
__END__
=head1 NAME
sanoid - ZFS snapshot management and replication tool
=head1 SYNOPSIS
sanoid [options]
Assumes --cron --verbose if no other arguments (other than configdir) are specified
Options:
--configdir=DIR Specify a directory to find config file sanoid.conf
--cron Creates snapshots and purges expired snapshots
--verbose Prints out additional information during a sanoid run
--readonly Simulates creation/deletion of snapshots
--quiet Suppresses non-error output
--force-update Clears out sanoid's zfs snapshot cache
--monitor-health Reports on zpool "health", in a Nagios compatible format
--monitor-snapshots Reports on snapshot "health", in a Nagios compatible format
--take-snapshots Creates snapshots as specified in sanoid.conf
--prune-snapshots Purges expired snapshots as specified in sanoid.conf
--help Prints this helptext
--version Prints the version number
--debug Prints out a lot of additional information during a sanoid run

View File

@ -34,21 +34,21 @@ min_percent_free = 10
# Note that we will not take snapshots for a given type if that type is set to 0 above,
# regardless of the autosnap setting - for example, if yearly=0 we will not take yearlies
# even if we've defined a preferred time for yearlies and autosnap is on.
autosnap = 1;
autosnap = 1
# hourly - top of the hour
hourly_min = 0;
hourly_min = 0
# daily - at 23:59 (most people expect a daily to contain everything done DURING that day)
daily_hour = 23;
daily_min = 59;
daily_hour = 23
daily_min = 59
# monthly - immediately at the beginning of the month (ie 00:00 of day 1)
monthly_mday = 1;
monthly_hour = 0;
monthly_min = 0;
monthly_mday = 1
monthly_hour = 0
monthly_min = 0
# yearly - immediately at the beginning of the year (ie 00:00 on Jan 1)
yearly_mon = 1;
yearly_mday = 1;
yearly_hour = 0;
yearly_min = 0;
yearly_mon = 1
yearly_mday = 1
yearly_hour = 0
yearly_min = 0
# monitoring plugin - define warn / crit levels for each snapshot type by age, in units of one period down
# example hourly_warn = 90 means issue WARNING if most recent hourly snapshot is not less than 90 minutes old,

335
syncoid
View File

@ -4,25 +4,43 @@
# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this
# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
my $version = '1.4.16';
$::VERSION = '1.4.16';
use strict;
use warnings;
use Data::Dumper;
use Getopt::Long qw(:config auto_version auto_help);
use Pod::Usage;
use Time::Local;
use Sys::Hostname;
my %args = getargs(@ARGV);
# Blank defaults to use ssh client's default
# TODO: Merge into a single "sshflags" option?
my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => '');
GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r",
"source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
"debug", "quiet", "no-stream", "no-sync-snap") or pod2usage(2);
if ($args{'version'}) {
print "Syncoid version: $version\n";
exit 0;
$args{'compress'} = compressargset($args{'compress'} || 'default'); # Can't be done with GetOptions arg, as default still needs to be set
# TODO Expand to accept multiple sources?
if (scalar(@ARGV) != 2) {
print("Source or target not found!\n");
pod2usage(2);
exit 127;
} else {
$args{'source'} = $ARGV[0];
$args{'target'} = $ARGV[1];
}
if (!(defined $args{'source'} && defined $args{'target'})) {
print 'usage: syncoid [src_user@src_host:]src_pool/src_dataset [dst_user@dst_host:]dst_pool/dst_dataset'."\n";
exit 127;
# Could possibly merge these into an options function
if (length $args{'source-bwlimit'}) {
$args{'source-bwlimit'} = "-R $args{'source-bwlimit'}";
}
if (length $args{'target-bwlimit'}) {
$args{'target-bwlimit'} = "-r $args{'target-bwlimit'}";
}
$args{'streamarg'} = (defined $args{'no-stream'} ? '-i' : '-I');
my $rawsourcefs = $args{'source'};
my $rawtargetfs = $args{'target'};
@ -32,25 +50,8 @@ my $quiet = $args{'quiet'};
my $zfscmd = '/sbin/zfs';
my $sshcmd = '/usr/bin/ssh';
my $pscmd = '/bin/ps';
my $sshcipher;
if (defined $args{'c'}) {
$sshcipher = "-c $args{'c'}";
} else {
$sshcipher = '-c chacha20-poly1305@openssh.com,arcfour';
}
my $sshport = '-p 22';
my $sshoption;
if (defined $args{'o'}) {
my @options = split(',', $args{'o'});
foreach my $option (@options) {
$sshoption .= " -o $option";
if ($option eq "NoneSwitch=yes") {
$sshcipher = "";
}
}
} else {
$sshoption = "";
}
my $pvcmd = '/usr/bin/pv';
my $mbuffercmd = '/usr/bin/mbuffer';
my $sudocmd = '/usr/bin/sudo';
@ -59,23 +60,25 @@ my $mbufferoptions = '-q -s 128k -m 16M 2>/dev/null';
# being present on remote machines.
my $lscmd = '/bin/ls';
if ( $args{'sshport'} ) {
$sshport = "-p $args{'sshport'}";
if (length $args{'sshcipher'}) {
$args{'sshcipher'} = "-c $args{'sshcipher'}";
}
if (length $args{'sshport'}) {
$args{'sshport'} = "-p $args{'sshport'}";
}
if (length $args{'sshkey'}) {
$args{'sshkey'} = "-i $args{'sshkey'}";
}
my $sshoptions = join " ", map { "-o " . $_ } @{$args{'sshoption'}}; # deref required
# figure out if source and/or target are remote.
if ( $args{'sshkey'} ) {
$sshcmd = "$sshcmd $sshoption $sshcipher $sshport -i $args{'sshkey'}";
}
else {
$sshcmd = "$sshcmd $sshoption $sshcipher $sshport";
}
$sshcmd = "$sshcmd $args{'sshcipher'} $sshoptions $args{'sshport'} $args{'sshkey'}";
if ($debug) { print "DEBUG: SSHCMD: $sshcmd\n"; }
my ($sourcehost,$sourcefs,$sourceisroot) = getssh($rawsourcefs);
my ($targethost,$targetfs,$targetisroot) = getssh($rawtargetfs);
my $sourcesudocmd;
my $targetsudocmd;
if ($sourceisroot) { $sourcesudocmd = ''; } else { $sourcesudocmd = $sudocmd; }
if ($targetisroot) { $targetsudocmd = ''; } else { $targetsudocmd = $sudocmd; }
my $sourcesudocmd = $sourceisroot ? '' : $sudocmd;
my $targetsudocmd = $targetisroot ? '' : $sudocmd;
# figure out whether compression, mbuffering, pv
# are available on source, target, local machines.
@ -88,7 +91,7 @@ my %snaps;
## can loop across children separately, for recursive ##
## replication ##
if (! $args{'recursive'}) {
if (!defined $args{'recursive'}) {
syncdataset($sourcehost, $sourcefs, $targethost, $targetfs);
} else {
if ($debug) { print "DEBUG: recursive sync of $sourcefs.\n"; }
@ -161,11 +164,15 @@ sub syncdataset {
%snaps = (%sourcesnaps, %targetsnaps);
}
if ($args{'dumpsnaps'}) { print "merged snapshot list of $targetfs: \n"; dumphash(\%snaps); print "\n\n\n"; }
if (defined $args{'dumpsnaps'}) {
print "merged snapshot list of $targetfs: \n";
dumphash(\%snaps);
print "\n\n\n";
}
# create a new syncoid snapshot on the source filesystem.
my $newsyncsnap;
if (!defined ($args{'no-sync-snap'}) ) {
if (!defined $args{'no-sync-snap'}) {
$newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot);
} else {
# we don't want sync snapshots created, so use the newest snapshot we can find.
@ -334,111 +341,54 @@ sub syncdataset {
} # end syncdataset()
sub compressargset {
my ($value) = @_;
my $DEFAULT_COMPRESSION = 'lzo';
my %COMPRESS_ARGS = (
'none' => {
rawcmd => '',
args => '',
decomrawcmd => '',
decomargs => '',
},
'gzip' => {
rawcmd => '/bin/gzip',
args => '-3',
decomrawcmd => '/bin/zcat',
decomargs => '',
},
'pigz-fast' => {
rawcmd => '/usr/bin/pigz',
args => '-3',
decomrawcmd => '/usr/bin/pigz',
decomargs => '-dc',
},
'pigz-slow' => {
rawcmd => '/usr/bin/pigz',
args => '-9',
decomrawcmd => '/usr/bin/pigz',
decomargs => '-dc',
},
'lzo' => {
rawcmd => '/usr/bin/lzop',
args => '',
decomrawcmd => '/usr/bin/lzop',
decomargs => '-dfc',
},
);
sub getargs {
my @args = @_;
my %args;
my %novaluearg;
my %validarg;
push my @validargs, ('debug','nocommandchecks','version','monitor-version','compress','c','o','source-bwlimit','target-bwlimit','dumpsnaps','recursive','r','sshkey','sshport','quiet','no-stream','no-sync-snap');
foreach my $item (@validargs) { $validarg{$item} = 1; }
push my @novalueargs, ('debug','nocommandchecks','version','monitor-version','dumpsnaps','recursive','r','quiet','no-stream','no-sync-snap');
foreach my $item (@novalueargs) { $novaluearg{$item} = 1; }
while (my $rawarg = shift(@args)) {
my $arg = $rawarg;
my $argvalue = '';
if ($rawarg =~ /=/) {
# user specified the value for a CLI argument with =
# instead of with blank space. separate appropriately.
$argvalue = $arg;
$arg =~ s/=.*$//;
$argvalue =~ s/^.*=//;
}
if ($rawarg =~ /^--/) {
# doubledash arg
$arg =~ s/^--//;
if (! $validarg{$arg}) { die "ERROR: don't understand argument $rawarg.\n"; }
if ($novaluearg{$arg}) {
$args{$arg} = 1;
} else {
# if this CLI arg takes a user-specified value and
# we don't already have it, then the user must have
# specified with a space, so pull in the next value
# from the array as this value rather than as the
# next argument.
if ($argvalue eq '') { $argvalue = shift(@args); }
$args{$arg} = $argvalue;
}
} elsif ($arg =~ /^-/) {
# singledash arg
$arg =~ s/^-//;
if (! $validarg{$arg}) { die "ERROR: don't understand argument $rawarg.\n"; }
if ($novaluearg{$arg}) {
$args{$arg} = 1;
} else {
# if this CLI arg takes a user-specified value and
# we don't already have it, then the user must have
# specified with a space, so pull in the next value
# from the array as this value rather than as the
# next argument.
if ($argvalue eq '') { $argvalue = shift(@args); }
$args{$arg} = $argvalue;
}
} else {
# bare arg
if (defined $args{'source'}) {
if (! defined $args{'target'}) {
$args{'target'} = $arg;
} else {
die "ERROR: don't know what to do with third bare argument $rawarg.\n";
}
} else {
$args{'source'} = $arg;
}
}
if ($value eq 'default') {
$value = $DEFAULT_COMPRESSION;
} elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'lzo', 'default', 'none'))) {
warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION";
$value = $DEFAULT_COMPRESSION;
}
if (defined $args{'source-bwlimit'}) { $args{'source-bwlimit'} = "-R $args{'source-bwlimit'}"; } else { $args{'source-bwlimit'} = ''; }
if (defined $args{'target-bwlimit'}) { $args{'target-bwlimit'} = "-r $args{'target-bwlimit'}"; } else { $args{'target-bwlimit'} = ''; }
if (defined $args{'no-stream'}) { $args{'streamarg'} = '-i'; } else { $args{'streamarg'} = '-I'; }
if ($args{'r'}) { $args{'recursive'} = $args{'r'}; }
if (!defined $args{'compress'}) { $args{'compress'} = 'default'; }
if ($args{'compress'} eq 'gzip') {
$args{'rawcompresscmd'} = '/bin/gzip';
$args{'compressargs'} = '-3';
$args{'rawdecompresscmd'} = '/bin/zcat';
$args{'decompressargs'} = '';
} elsif ( ($args{'compress'} eq 'pigz-fast')) {
$args{'rawcompresscmd'} = '/usr/bin/pigz';
$args{'compressargs'} = '-3';
$args{'rawdecompresscmd'} = '/usr/bin/pigz';
$args{'decompressargs'} = '-dc';
} elsif ( ($args{'compress'} eq 'pigz-slow')) {
$args{'rawcompresscmd'} = '/usr/bin/pigz';
$args{'compressargs'} = '-9';
$args{'rawdecompresscmd'} = '/usr/bin/pigz';
$args{'decompressargs'} = '-dc';
} elsif ( ($args{'compress'} eq 'lzo') || ($args{'compress'} eq 'default') ) {
$args{'rawcompresscmd'} = '/usr/bin/lzop';
$args{'compressargs'} = '';
$args{'rawdecompresscmd'} = '/usr/bin/lzop';
$args{'decompressargs'} = '-dfc';
} else {
$args{'rawcompresscmd'} = '';
$args{'compressargs'} = '';
$args{'rawdecompresscmd'} = '';
$args{'decompressargs'} = '';
}
$args{'compresscmd'} = "$args{'rawcompresscmd'} $args{'compressargs'}";
$args{'decompresscmd'} = "$args{'rawdecompresscmd'} $args{'decompressargs'}";
return %args;
my %comargs = %{$COMPRESS_ARGS{$value}}; # copy
$comargs{'compress'} = $value;
$comargs{'cmd'} = "$comargs{'rawcmd'} $comargs{'args'}";
$comargs{'decomcmd'} = "$comargs{'decomrawcmd'} $comargs{'decomargs'}";
return \%comargs;
}
sub checkcommands {
@ -468,24 +418,15 @@ sub checkcommands {
# if raw compress command is null, we must have specified no compression. otherwise,
# make sure that compression is available everywhere we need it
if ($args{'rawcompresscmd'} eq '') {
$avail{'sourcecompress'} = 0;
$avail{'sourcecompress'} = 0;
$avail{'localcompress'} = 0;
if ($args{'compress'} eq 'none' ||
$args{'compress'} eq 'no' ||
$args{'compress'} eq '0') {
if ($debug) { print "DEBUG: compression forced off from command line arguments.\n"; }
} else {
print "WARN: value $args{'compress'} for argument --compress not understood, proceeding without compression.\n";
}
if ($args{'compress'}{'compress'} eq 'none') {
if ($debug) { print "DEBUG: compression forced off from command line arguments.\n"; }
} else {
if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on source...\n"; }
$avail{'sourcecompress'} = `$sourcessh $lscmd $args{'rawcompresscmd'} 2>/dev/null`;
if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on target...\n"; }
$avail{'targetcompress'} = `$targetssh $lscmd $args{'rawcompresscmd'} 2>/dev/null`;
if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on local machine...\n"; }
$avail{'localcompress'} = `$lscmd $args{'rawcompresscmd'} 2>/dev/null`;
if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on source...\n"; }
$avail{'sourcecompress'} = `$sourcessh $lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`;
if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on target...\n"; }
$avail{'targetcompress'} = `$targetssh $lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`;
if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on local machine...\n"; }
$avail{'localcompress'} = `$lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`;
}
my ($s,$t);
@ -511,14 +452,14 @@ sub checkcommands {
if ($avail{'sourcecompress'} eq '') {
if ($args{'rawcompresscmd'} ne '') {
print "WARN: $args{'compresscmd'} not available on source $s- sync will continue without compression.\n";
if ($args{'compress'}{'rawcmd'} ne '') {
print "WARN: $args{'compress'}{'rawcmd'} not available on source $s- sync will continue without compression.\n";
}
$avail{'compress'} = 0;
}
if ($avail{'targetcompress'} eq '') {
if ($args{'rawcompresscmd'} ne '') {
print "WARN: $args{'compresscmd'} not available on target $t - sync will continue without compression.\n";
if ($args{'compress'}{'rawcmd'} ne '') {
print "WARN: $args{'compress'}{'rawcmd'} not available on target $t - sync will continue without compression.\n";
}
$avail{'compress'} = 0;
}
@ -530,8 +471,8 @@ sub checkcommands {
# corner case - if source AND target are BOTH remote, we have to check for local compress too
if ($sourcehost ne '' && $targethost ne '' && $avail{'localcompress'} eq '') {
if ($args{'rawcompresscmd'} ne '') {
print "WARN: $args{'compresscmd'} not available on local machine - sync will continue without compression.\n";
if ($args{'compress'}{'rawcmd'} ne '') {
print "WARN: $args{'compress'}{'rawcmd'} not available on local machine - sync will continue without compression.\n";
}
$avail{'compress'} = 0;
}
@ -687,9 +628,9 @@ sub buildsynccmd {
$synccmd = "$sendcmd |";
# avoid confusion - accept either source-bwlimit or target-bwlimit as the bandwidth limiting option here
my $bwlimit = '';
if (defined $args{'source-bwlimit'}) {
if (length $args{'bwlimit'}) {
$bwlimit = $args{'source-bwlimit'};
} elsif (defined $args{'target-bwlimit'}) {
} elsif (length $args{'target-bwlimit'}) {
$bwlimit = $args{'target-bwlimit'};
}
@ -698,18 +639,18 @@ sub buildsynccmd {
$synccmd .= " $recvcmd";
} elsif ($sourcehost eq '') {
# local source, remote target.
#$synccmd = "$sendcmd | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'";
#$synccmd = "$sendcmd | $pvcmd | $args{'compress'}{'cmd'} | $mbuffercmd | $sshcmd $targethost '$args{'compress'}{'decomcmd'} | $mbuffercmd | $recvcmd'";
$synccmd = "$sendcmd |";
if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; }
if ($avail{'compress'}) { $synccmd .= " $args{'compresscmd'} |"; }
if ($avail{'compress'}) { $synccmd .= " $args{'compress'}{'cmd'} |"; }
if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; }
$synccmd .= " $sshcmd $targethost '";
if ($avail{'targetmbuffer'}) { $synccmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; }
if ($avail{'compress'}) { $synccmd .= " $args{'decompresscmd'} |"; }
if ($avail{'compress'}) { $synccmd .= " $args{'compress'}{'decomcmd'} |"; }
$synccmd .= " $recvcmd'";
} elsif ($targethost eq '') {
# remote source, local target.
#$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $mbuffercmd | $pvcmd | $recvcmd";
#$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compress'}{'cmd'} | $mbuffercmd' | $args{'decompress'}{'cmd'} | $mbuffercmd | $pvcmd | $recvcmd";
$synccmd = "$sshcmd $sourcehost '$sendcmd";
if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; }
if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; }
@ -720,18 +661,18 @@ sub buildsynccmd {
$synccmd .= "$recvcmd";
} else {
#remote source, remote target... weird, but whatever, I'm not here to judge you.
#$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'";
#$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compress'}{'cmd'} | $mbuffercmd' | $args{'compress'}{'decomcmd'} | $pvcmd | $args{'compress'}{'cmd'} | $mbuffercmd | $sshcmd $targethost '$args{'compress'}{'decomcmd'} | $mbuffercmd | $recvcmd'";
$synccmd = "$sshcmd $sourcehost '$sendcmd";
if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; }
if ($avail{'compress'}) { $synccmd .= " | $args{'compress'}{'cmd'}"; }
if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; }
$synccmd .= "' | ";
if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; }
if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'decomcmd'} | "; }
if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; }
if ($avail{'compress'}) { $synccmd .= "$args{'compresscmd'} | "; }
if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'cmd'} | "; }
if ($avail{'localmbuffer'}) { $synccmd .= "$mbuffercmd $mbufferoptions | "; }
$synccmd .= "$sshcmd $targethost '";
if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; }
if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; }
if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'decomcmd'} | "; }
$synccmd .= "$recvcmd'";
}
return $synccmd;
@ -865,7 +806,7 @@ sub getssh {
if ($remoteuser eq 'root') { $isroot = 1; } else { $isroot = 0; }
# now we need to establish a persistent master SSH connection
$socket = "/tmp/syncoid-$remoteuser-$rhost-" . time();
open FH, "$sshcmd -M -S $socket -o ControlPersist=1m $sshport $rhost exit |";
open FH, "$sshcmd -M -S $socket -o ControlPersist=1m $args{'sshport'} $rhost exit |";
close FH;
$rhost = "-S $socket $rhost";
} else {
@ -985,4 +926,40 @@ sub getdate {
return %date;
}
__END__
=head1 NAME
syncoid - ZFS snapshot replication tool
=head1 SYNOPSIS
syncoid [options]... SOURCE TARGET
or syncoid [options]... SOURCE [USER@]HOST:TARGET
or syncoid [options]... [USER@]HOST:SOURCE [TARGET]
or syncoid [options]... [USER@]HOST:SOURCE [USER@]HOST:TARGET
SOURCE Source ZFS dataset. Can be either local or remote
TARGET Target ZFS dataset. Can be either local or remote
Options:
--compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none
--recursive|r Also transfers child datasets
--source-bwlimit=<limit k|m|g|t> Bandwidth limit on the source transfer
--target-bwlimit=<limit k|m|g|t> Bandwidth limit on the target transfer
--no-stream Replicates using newest snapshot instead of intermediates
--no-sync-snap Does not create new snapshot, only transfers existing
--sshkey=FILE Specifies a ssh public key to use to connect
--sshport=PORT Connects to remote on a particular port
--sshcipher|c=CIPHER Passes CIPHER to ssh to use a particular cipher set
--sshoption|o=OPTION Passes OPTION to ssh for remote usage. Can be specified multiple times
--help Prints this helptext
--verbose Prints the version number
--debug Prints out a lot of additional information during a syncoid run
--monitor-version Currently does nothing
--quiet Suppresses non-error output
--dumpsnaps Dumps a list of snapshots during the run
--no-command-checks Do not check command existence before attempting transfer. Not recommended