From 21a6881c4bbf97ebd21c3da44bdb7dd8b5192eb3 Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Wed, 8 Nov 2017 16:19:41 +0000 Subject: [PATCH 01/14] Unrevert syncoid --- syncoid | 349 ++++++++++++++++++++++++++------------------------------ 1 file changed, 162 insertions(+), 187 deletions(-) diff --git a/syncoid b/syncoid index d927cba..8694561 100755 --- a/syncoid +++ b/syncoid @@ -4,25 +4,43 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.16'; +$::VERSION = '1.4.16'; use strict; use warnings; use Data::Dumper; +use Getopt::Long qw(:config auto_version auto_help); +use Pod::Usage; use Time::Local; use Sys::Hostname; -my %args = getargs(@ARGV); +# Blank defaults to use ssh client's default +# TODO: Merge into a single "sshflags" option? +my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); +GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", + "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", + "debug", "quiet", "no-stream", "no-sync-snap") or pod2usage(2); -if ($args{'version'}) { - print "Syncoid version: $version\n"; - exit 0; +$args{'compress'} = compressargset($args{'compress'} || 'default'); # Can't be done with GetOptions arg, as default still needs to be set + +# TODO Expand to accept multiple sources? +if (scalar(@ARGV) != 2) { + print("Source or target not found!\n"); + pod2usage(2); + exit 127; +} else { + $args{'source'} = $ARGV[0]; + $args{'target'} = $ARGV[1]; } -if (!(defined $args{'source'} && defined $args{'target'})) { - print 'usage: syncoid [src_user@src_host:]src_pool/src_dataset [dst_user@dst_host:]dst_pool/dst_dataset'."\n"; - exit 127; +# Could possibly merge these into an options function +if (length $args{'source-bwlimit'}) { + $args{'source-bwlimit'} = "-R $args{'source-bwlimit'}"; } +if (length $args{'target-bwlimit'}) { + $args{'target-bwlimit'} = "-r $args{'target-bwlimit'}"; +} +$args{'streamarg'} = (defined $args{'no-stream'} ? '-i' : '-I'); my $rawsourcefs = $args{'source'}; my $rawtargetfs = $args{'target'}; @@ -32,28 +50,7 @@ my $quiet = $args{'quiet'}; my $zfscmd = '/sbin/zfs'; my $sshcmd = '/usr/bin/ssh'; my $pscmd = '/bin/ps'; -my $sshcipher; -if (defined $args{'c'}) { - $sshcipher = "-c $args{'c'}"; -} else { - # default to no cipher specified now that SSH - # has not defaulted to a cripplingly slow cipher - # in a very long time - $sshcipher = ''; -} -my $sshport = '-p 22'; -my $sshoption; -if (defined $args{'o'}) { - my @options = split(',', $args{'o'}); - foreach my $option (@options) { - $sshoption .= " -o $option"; - if ($option eq "NoneSwitch=yes") { - $sshcipher = ""; - } - } -} else { - $sshoption = ""; -} + my $pvcmd = '/usr/bin/pv'; my $mbuffercmd = '/usr/bin/mbuffer'; my $sudocmd = '/usr/bin/sudo'; @@ -62,23 +59,25 @@ my $mbufferoptions = '-q -s 128k -m 16M 2>/dev/null'; # being present on remote machines. my $lscmd = '/bin/ls'; -if ( $args{'sshport'} ) { - $sshport = "-p $args{'sshport'}"; +if (length $args{'sshcipher'}) { + $args{'sshcipher'} = "-c $args{'sshcipher'}"; } +if (length $args{'sshport'}) { + $args{'sshport'} = "-p $args{'sshport'}"; +} +if (length $args{'sshkey'}) { + $args{'sshkey'} = "-i $args{'sshkey'}"; +} +my $sshoptions = join " ", map { "-o " . $_ } @{$args{'sshoption'}}; # deref required + # figure out if source and/or target are remote. -if ( $args{'sshkey'} ) { - $sshcmd = "$sshcmd $sshoption $sshcipher $sshport -i $args{'sshkey'}"; -} -else { - $sshcmd = "$sshcmd $sshoption $sshcipher $sshport"; -} +$sshcmd = "$sshcmd $args{'sshcipher'} $sshoptions $args{'sshport'} $args{'sshkey'}"; +if ($debug) { print "DEBUG: SSHCMD: $sshcmd\n"; } my ($sourcehost,$sourcefs,$sourceisroot) = getssh($rawsourcefs); my ($targethost,$targetfs,$targetisroot) = getssh($rawtargetfs); -my $sourcesudocmd; -my $targetsudocmd; -if ($sourceisroot) { $sourcesudocmd = ''; } else { $sourcesudocmd = $sudocmd; } -if ($targetisroot) { $targetsudocmd = ''; } else { $targetsudocmd = $sudocmd; } +my $sourcesudocmd = $sourceisroot ? '' : $sudocmd; +my $targetsudocmd = $targetisroot ? '' : $sudocmd; # figure out whether compression, mbuffering, pv # are available on source, target, local machines. @@ -91,7 +90,7 @@ my %snaps; ## can loop across children separately, for recursive ## ## replication ## -if (! $args{'recursive'}) { +if (!defined $args{'recursive'}) { syncdataset($sourcehost, $sourcefs, $targethost, $targetfs); } else { if ($debug) { print "DEBUG: recursive sync of $sourcefs.\n"; } @@ -164,17 +163,21 @@ sub syncdataset { %snaps = (%sourcesnaps, %targetsnaps); } - if ($args{'dumpsnaps'}) { print "merged snapshot list: \n"; dumphash(\%snaps); print "\n\n\n"; } + if (defined $args{'dumpsnaps'}) { + print "merged snapshot list of $targetfs: \n"; + dumphash(\%snaps); + print "\n\n\n"; + } # create a new syncoid snapshot on the source filesystem. my $newsyncsnap; - if (!defined ($args{'no-sync-snap'}) ) { + if (!defined $args{'no-sync-snap'}) { $newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot); } else { # we don't want sync snapshots created, so use the newest snapshot we can find. $newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot); if ($newsyncsnap eq 0) { - warn "CRITICAL: no snapshots exist on source, and you asked for --no-sync-snap.\n"; + warn "CRITICAL: no snapshots exist on source $sourcefs, and you asked for --no-sync-snap.\n"; return 0; } } @@ -285,7 +288,7 @@ sub syncdataset { my $targetsize = getzfsvalue($targethost,$targetfs,$targetisroot,'-p used'); - my $matchingsnap = getmatchingsnapshot($targetsize, \%snaps); + my $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, $targetsize, \%snaps); if (! $matchingsnap) { # no matching snapshot; we whined piteously already, but let's go ahead and return false # now in case more child datasets need replication. @@ -337,111 +340,54 @@ sub syncdataset { } # end syncdataset() +sub compressargset { + my ($value) = @_; + my $DEFAULT_COMPRESSION = 'lzo'; + my %COMPRESS_ARGS = ( + 'none' => { + rawcmd => '', + args => '', + decomrawcmd => '', + decomargs => '', + }, + 'gzip' => { + rawcmd => '/bin/gzip', + args => '-3', + decomrawcmd => '/bin/zcat', + decomargs => '', + }, + 'pigz-fast' => { + rawcmd => '/usr/bin/pigz', + args => '-3', + decomrawcmd => '/usr/bin/pigz', + decomargs => '-dc', + }, + 'pigz-slow' => { + rawcmd => '/usr/bin/pigz', + args => '-9', + decomrawcmd => '/usr/bin/pigz', + decomargs => '-dc', + }, + 'lzo' => { + rawcmd => '/usr/bin/lzop', + args => '', + decomrawcmd => '/usr/bin/lzop', + decomargs => '-dfc', + }, + ); -sub getargs { - my @args = @_; - my %args; - - my %novaluearg; - my %validarg; - push my @validargs, ('debug','nocommandchecks','version','monitor-version','compress','c','o','source-bwlimit','target-bwlimit','dumpsnaps','recursive','r','sshkey','sshport','quiet','no-stream','no-sync-snap'); - foreach my $item (@validargs) { $validarg{$item} = 1; } - push my @novalueargs, ('debug','nocommandchecks','version','monitor-version','dumpsnaps','recursive','r','quiet','no-stream','no-sync-snap'); - foreach my $item (@novalueargs) { $novaluearg{$item} = 1; } - - while (my $rawarg = shift(@args)) { - my $arg = $rawarg; - my $argvalue = ''; - if ($rawarg =~ /=/) { - # user specified the value for a CLI argument with = - # instead of with blank space. separate appropriately. - $argvalue = $arg; - $arg =~ s/=.*$//; - $argvalue =~ s/^.*=//; - } - if ($rawarg =~ /^--/) { - # doubledash arg - $arg =~ s/^--//; - if (! $validarg{$arg}) { die "ERROR: don't understand argument $rawarg.\n"; } - if ($novaluearg{$arg}) { - $args{$arg} = 1; - } else { - # if this CLI arg takes a user-specified value and - # we don't already have it, then the user must have - # specified with a space, so pull in the next value - # from the array as this value rather than as the - # next argument. - if ($argvalue eq '') { $argvalue = shift(@args); } - $args{$arg} = $argvalue; - } - } elsif ($arg =~ /^-/) { - # singledash arg - $arg =~ s/^-//; - if (! $validarg{$arg}) { die "ERROR: don't understand argument $rawarg.\n"; } - if ($novaluearg{$arg}) { - $args{$arg} = 1; - } else { - # if this CLI arg takes a user-specified value and - # we don't already have it, then the user must have - # specified with a space, so pull in the next value - # from the array as this value rather than as the - # next argument. - if ($argvalue eq '') { $argvalue = shift(@args); } - $args{$arg} = $argvalue; - } - } else { - # bare arg - if (defined $args{'source'}) { - if (! defined $args{'target'}) { - $args{'target'} = $arg; - } else { - die "ERROR: don't know what to do with third bare argument $rawarg.\n"; - } - } else { - $args{'source'} = $arg; - } - } + if ($value eq 'default') { + $value = $DEFAULT_COMPRESSION; + } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'lzo', 'default', 'none'))) { + warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION"; + $value = $DEFAULT_COMPRESSION; } - if (defined $args{'source-bwlimit'}) { $args{'source-bwlimit'} = "-R $args{'source-bwlimit'}"; } else { $args{'source-bwlimit'} = ''; } - if (defined $args{'target-bwlimit'}) { $args{'target-bwlimit'} = "-r $args{'target-bwlimit'}"; } else { $args{'target-bwlimit'} = ''; } - - if (defined $args{'no-stream'}) { $args{'streamarg'} = '-i'; } else { $args{'streamarg'} = '-I'; } - - if ($args{'r'}) { $args{'recursive'} = $args{'r'}; } - - if (!defined $args{'compress'}) { $args{'compress'} = 'default'; } - - if ($args{'compress'} eq 'gzip') { - $args{'rawcompresscmd'} = '/bin/gzip'; - $args{'compressargs'} = '-3'; - $args{'rawdecompresscmd'} = '/bin/zcat'; - $args{'decompressargs'} = ''; - } elsif ( ($args{'compress'} eq 'pigz-fast')) { - $args{'rawcompresscmd'} = '/usr/bin/pigz'; - $args{'compressargs'} = '-3'; - $args{'rawdecompresscmd'} = '/usr/bin/pigz'; - $args{'decompressargs'} = '-dc'; - } elsif ( ($args{'compress'} eq 'pigz-slow')) { - $args{'rawcompresscmd'} = '/usr/bin/pigz'; - $args{'compressargs'} = '-9'; - $args{'rawdecompresscmd'} = '/usr/bin/pigz'; - $args{'decompressargs'} = '-dc'; - } elsif ( ($args{'compress'} eq 'lzo') || ($args{'compress'} eq 'default') ) { - $args{'rawcompresscmd'} = '/usr/bin/lzop'; - $args{'compressargs'} = ''; - $args{'rawdecompresscmd'} = '/usr/bin/lzop'; - $args{'decompressargs'} = '-dfc'; - } else { - $args{'rawcompresscmd'} = ''; - $args{'compressargs'} = ''; - $args{'rawdecompresscmd'} = ''; - $args{'decompressargs'} = ''; - } - $args{'compresscmd'} = "$args{'rawcompresscmd'} $args{'compressargs'}"; - $args{'decompresscmd'} = "$args{'rawdecompresscmd'} $args{'decompressargs'}"; - - return %args; + my %comargs = %{$COMPRESS_ARGS{$value}}; # copy + $comargs{'compress'} = $value; + $comargs{'cmd'} = "$comargs{'rawcmd'} $comargs{'args'}"; + $comargs{'decomcmd'} = "$comargs{'decomrawcmd'} $comargs{'decomargs'}"; + return \%comargs; } sub checkcommands { @@ -471,24 +417,15 @@ sub checkcommands { # if raw compress command is null, we must have specified no compression. otherwise, # make sure that compression is available everywhere we need it - if ($args{'rawcompresscmd'} eq '') { - $avail{'sourcecompress'} = 0; - $avail{'sourcecompress'} = 0; - $avail{'localcompress'} = 0; - if ($args{'compress'} eq 'none' || - $args{'compress'} eq 'no' || - $args{'compress'} eq '0') { - if ($debug) { print "DEBUG: compression forced off from command line arguments.\n"; } - } else { - print "WARN: value $args{'compress'} for argument --compress not understood, proceeding without compression.\n"; - } + if ($args{'compress'}{'compress'} eq 'none') { + if ($debug) { print "DEBUG: compression forced off from command line arguments.\n"; } } else { - if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on source...\n"; } - $avail{'sourcecompress'} = `$sourcessh $lscmd $args{'rawcompresscmd'} 2>/dev/null`; - if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on target...\n"; } - $avail{'targetcompress'} = `$targetssh $lscmd $args{'rawcompresscmd'} 2>/dev/null`; - if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on local machine...\n"; } - $avail{'localcompress'} = `$lscmd $args{'rawcompresscmd'} 2>/dev/null`; + if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on source...\n"; } + $avail{'sourcecompress'} = `$sourcessh $lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`; + if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on target...\n"; } + $avail{'targetcompress'} = `$targetssh $lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`; + if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on local machine...\n"; } + $avail{'localcompress'} = `$lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`; } my ($s,$t); @@ -514,14 +451,14 @@ sub checkcommands { if ($avail{'sourcecompress'} eq '') { - if ($args{'rawcompresscmd'} ne '') { - print "WARN: $args{'compresscmd'} not available on source $s- sync will continue without compression.\n"; + if ($args{'compress'}{'rawcmd'} ne '') { + print "WARN: $args{'compress'}{'rawcmd'} not available on source $s- sync will continue without compression.\n"; } $avail{'compress'} = 0; } if ($avail{'targetcompress'} eq '') { - if ($args{'rawcompresscmd'} ne '') { - print "WARN: $args{'compresscmd'} not available on target $t - sync will continue without compression.\n"; + if ($args{'compress'}{'rawcmd'} ne '') { + print "WARN: $args{'compress'}{'rawcmd'} not available on target $t - sync will continue without compression.\n"; } $avail{'compress'} = 0; } @@ -533,8 +470,8 @@ sub checkcommands { # corner case - if source AND target are BOTH remote, we have to check for local compress too if ($sourcehost ne '' && $targethost ne '' && $avail{'localcompress'} eq '') { - if ($args{'rawcompresscmd'} ne '') { - print "WARN: $args{'compresscmd'} not available on local machine - sync will continue without compression.\n"; + if ($args{'compress'}{'rawcmd'} ne '') { + print "WARN: $args{'compress'}{'rawcmd'} not available on local machine - sync will continue without compression.\n"; } $avail{'compress'} = 0; } @@ -690,9 +627,9 @@ sub buildsynccmd { $synccmd = "$sendcmd |"; # avoid confusion - accept either source-bwlimit or target-bwlimit as the bandwidth limiting option here my $bwlimit = ''; - if (defined $args{'source-bwlimit'}) { + if (length $args{'bwlimit'}) { $bwlimit = $args{'source-bwlimit'}; - } elsif (defined $args{'target-bwlimit'}) { + } elsif (length $args{'target-bwlimit'}) { $bwlimit = $args{'target-bwlimit'}; } @@ -701,18 +638,18 @@ sub buildsynccmd { $synccmd .= " $recvcmd"; } elsif ($sourcehost eq '') { # local source, remote target. - #$synccmd = "$sendcmd | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'"; + #$synccmd = "$sendcmd | $pvcmd | $args{'compress'}{'cmd'} | $mbuffercmd | $sshcmd $targethost '$args{'compress'}{'decomcmd'} | $mbuffercmd | $recvcmd'"; $synccmd = "$sendcmd |"; if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; } - if ($avail{'compress'}) { $synccmd .= " $args{'compresscmd'} |"; } + if ($avail{'compress'}) { $synccmd .= " $args{'compress'}{'cmd'} |"; } if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; } $synccmd .= " $sshcmd $targethost '"; if ($avail{'targetmbuffer'}) { $synccmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; } - if ($avail{'compress'}) { $synccmd .= " $args{'decompresscmd'} |"; } + if ($avail{'compress'}) { $synccmd .= " $args{'compress'}{'decomcmd'} |"; } $synccmd .= " $recvcmd'"; } elsif ($targethost eq '') { # remote source, local target. - #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $mbuffercmd | $pvcmd | $recvcmd"; + #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compress'}{'cmd'} | $mbuffercmd' | $args{'decompress'}{'cmd'} | $mbuffercmd | $pvcmd | $recvcmd"; $synccmd = "$sshcmd $sourcehost '$sendcmd"; if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; } if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } @@ -723,18 +660,18 @@ sub buildsynccmd { $synccmd .= "$recvcmd"; } else { #remote source, remote target... weird, but whatever, I'm not here to judge you. - #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'"; + #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compress'}{'cmd'} | $mbuffercmd' | $args{'compress'}{'decomcmd'} | $pvcmd | $args{'compress'}{'cmd'} | $mbuffercmd | $sshcmd $targethost '$args{'compress'}{'decomcmd'} | $mbuffercmd | $recvcmd'"; $synccmd = "$sshcmd $sourcehost '$sendcmd"; - if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; } + if ($avail{'compress'}) { $synccmd .= " | $args{'compress'}{'cmd'}"; } if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } $synccmd .= "' | "; - if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'decomcmd'} | "; } if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } - if ($avail{'compress'}) { $synccmd .= "$args{'compresscmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'cmd'} | "; } if ($avail{'localmbuffer'}) { $synccmd .= "$mbuffercmd $mbufferoptions | "; } $synccmd .= "$sshcmd $targethost '"; if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } - if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'decomcmd'} | "; } $synccmd .= "$recvcmd'"; } return $synccmd; @@ -794,7 +731,7 @@ sub pruneoldsyncsnaps { } sub getmatchingsnapshot { - my ($targetsize, $snaps) = shift; + my ($sourcefs, $targetfs, $targetsize, $snaps) = @_; foreach my $snap ( sort { $snaps{'source'}{$b}{'creation'}<=>$snaps{'source'}{$a}{'creation'} } keys %{ $snaps{'source'} }) { if (defined $snaps{'target'}{$snap}{'guid'}) { if ($snaps{'source'}{$snap}{'guid'} == $snaps{'target'}{$snap}{'guid'}) { @@ -806,7 +743,7 @@ sub getmatchingsnapshot { # if we got this far, we failed to find a matching snapshot. print "\n"; - print "CRITICAL ERROR: Target exists but has no matching snapshots!\n"; + print "CRITICAL ERROR: Target $targetfs exists but has no snapshots matching with $sourcefs!\n"; print " Replication to target would require destroying existing\n"; print " target. Cowardly refusing to destroy your existing target.\n\n"; @@ -814,7 +751,7 @@ sub getmatchingsnapshot { # zfs create targetpool/targetsnap ; syncoid sourcepool/sourcesnap targetpool/targetsnap ... if ( $targetsize < (64*1024*1024) ) { - print " NOTE: Target dataset is < 64MB used - did you mistakenly run\n"; + print " NOTE: Target $targetfs dataset is < 64MB used - did you mistakenly run\n"; print " \`zfs create $args{'target'}\` on the target? ZFS initial\n"; print " replication must be to a NON EXISTENT DATASET, which will\n"; print " then be CREATED BY the initial replication process.\n\n"; @@ -868,7 +805,7 @@ sub getssh { if ($remoteuser eq 'root') { $isroot = 1; } else { $isroot = 0; } # now we need to establish a persistent master SSH connection $socket = "/tmp/syncoid-$remoteuser-$rhost-" . time(); - open FH, "$sshcmd -M -S $socket -o ControlPersist=1m $sshport $rhost exit |"; + open FH, "$sshcmd -M -S $socket -o ControlPersist=1m $args{'sshport'} $rhost exit |"; close FH; $rhost = "-S $socket $rhost"; } else { @@ -987,3 +924,41 @@ sub getdate { $date{'stamp'} = "$date{'year'}-$date{'mon'}-$date{'mday'}:$date{'hour'}:$date{'min'}:$date{'sec'}"; return %date; } + +__END__ + +=head1 NAME + +syncoid - ZFS snapshot replication tool + +=head1 SYNOPSIS + + syncoid [options]... SOURCE TARGET + or syncoid [options]... SOURCE [USER@]HOST:TARGET + or syncoid [options]... [USER@]HOST:SOURCE [TARGET] + or syncoid [options]... [USER@]HOST:SOURCE [USER@]HOST:TARGET + + SOURCE Source ZFS dataset. Can be either local or remote + TARGET Target ZFS dataset. Can be either local or remote + +Options: + + --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none + --recursive|r Also transfers child datasets + --source-bwlimit= Bandwidth limit on the source transfer + --target-bwlimit= Bandwidth limit on the target transfer + --no-stream Replicates using newest snapshot instead of intermediates + --no-sync-snap Does not create new snapshot, only transfers existing + + --sshkey=FILE Specifies a ssh public key to use to connect + --sshport=PORT Connects to remote on a particular port + --sshcipher|c=CIPHER Passes CIPHER to ssh to use a particular cipher set + --sshoption|o=OPTION Passes OPTION to ssh for remote usage. Can be specified multiple times + + --help Prints this helptext + --verbose Prints the version number + --debug Prints out a lot of additional information during a syncoid run + --monitor-version Currently does nothing + --quiet Suppresses non-error output + --dumpsnaps Dumps a list of snapshots during the run + --no-command-checks Do not check command existence before attempting transfer. Not recommended From 11da17fa4a82f0eb668131fe7d4587c25cfbeb05 Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Wed, 8 Nov 2017 16:19:59 +0000 Subject: [PATCH 02/14] Fix compress arg handling --- syncoid | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/syncoid b/syncoid index 8694561..3f07990 100755 --- a/syncoid +++ b/syncoid @@ -21,7 +21,7 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", "debug", "quiet", "no-stream", "no-sync-snap") or pod2usage(2); -$args{'compress'} = compressargset($args{'compress'} || 'default'); # Can't be done with GetOptions arg, as default still needs to be set +my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set # TODO Expand to accept multiple sources? if (scalar(@ARGV) != 2) { @@ -417,15 +417,15 @@ sub checkcommands { # if raw compress command is null, we must have specified no compression. otherwise, # make sure that compression is available everywhere we need it - if ($args{'compress'}{'compress'} eq 'none') { + if ($compressargs{'compress'} eq 'none') { if ($debug) { print "DEBUG: compression forced off from command line arguments.\n"; } } else { - if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on source...\n"; } - $avail{'sourcecompress'} = `$sourcessh $lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`; - if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on target...\n"; } - $avail{'targetcompress'} = `$targetssh $lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`; - if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on local machine...\n"; } - $avail{'localcompress'} = `$lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`; + if ($debug) { print "DEBUG: checking availability of $compressargs{'rawcmd'} on source...\n"; } + $avail{'sourcecompress'} = `$sourcessh $lscmd $compressargs{'rawcmd'} 2>/dev/null`; + if ($debug) { print "DEBUG: checking availability of $compressargs{'rawcmd'} on target...\n"; } + $avail{'targetcompress'} = `$targetssh $lscmd $compressargs{'rawcmd'} 2>/dev/null`; + if ($debug) { print "DEBUG: checking availability of $compressargs{'rawcmd'} on local machine...\n"; } + $avail{'localcompress'} = `$lscmd $compressargs{'rawcmd'} 2>/dev/null`; } my ($s,$t); @@ -451,14 +451,14 @@ sub checkcommands { if ($avail{'sourcecompress'} eq '') { - if ($args{'compress'}{'rawcmd'} ne '') { - print "WARN: $args{'compress'}{'rawcmd'} not available on source $s- sync will continue without compression.\n"; + if ($compressargs{'rawcmd'} ne '') { + print "WARN: $compressargs{'rawcmd'} not available on source $s- sync will continue without compression.\n"; } $avail{'compress'} = 0; } if ($avail{'targetcompress'} eq '') { - if ($args{'compress'}{'rawcmd'} ne '') { - print "WARN: $args{'compress'}{'rawcmd'} not available on target $t - sync will continue without compression.\n"; + if ($compressargs{'rawcmd'} ne '') { + print "WARN: $compressargs{'rawcmd'} not available on target $t - sync will continue without compression.\n"; } $avail{'compress'} = 0; } @@ -470,8 +470,8 @@ sub checkcommands { # corner case - if source AND target are BOTH remote, we have to check for local compress too if ($sourcehost ne '' && $targethost ne '' && $avail{'localcompress'} eq '') { - if ($args{'compress'}{'rawcmd'} ne '') { - print "WARN: $args{'compress'}{'rawcmd'} not available on local machine - sync will continue without compression.\n"; + if ($compressargs{'rawcmd'} ne '') { + print "WARN: $compressargs{'rawcmd'} not available on local machine - sync will continue without compression.\n"; } $avail{'compress'} = 0; } @@ -638,40 +638,40 @@ sub buildsynccmd { $synccmd .= " $recvcmd"; } elsif ($sourcehost eq '') { # local source, remote target. - #$synccmd = "$sendcmd | $pvcmd | $args{'compress'}{'cmd'} | $mbuffercmd | $sshcmd $targethost '$args{'compress'}{'decomcmd'} | $mbuffercmd | $recvcmd'"; + #$synccmd = "$sendcmd | $pvcmd | $compressargs{'cmd'} | $mbuffercmd | $sshcmd $targethost '$compressargs{'decomcmd'} | $mbuffercmd | $recvcmd'"; $synccmd = "$sendcmd |"; if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; } - if ($avail{'compress'}) { $synccmd .= " $args{'compress'}{'cmd'} |"; } + if ($avail{'compress'}) { $synccmd .= " $compressargs{'cmd'} |"; } if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; } $synccmd .= " $sshcmd $targethost '"; if ($avail{'targetmbuffer'}) { $synccmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; } - if ($avail{'compress'}) { $synccmd .= " $args{'compress'}{'decomcmd'} |"; } + if ($avail{'compress'}) { $synccmd .= " $compressargs{'decomcmd'} |"; } $synccmd .= " $recvcmd'"; } elsif ($targethost eq '') { # remote source, local target. - #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compress'}{'cmd'} | $mbuffercmd' | $args{'decompress'}{'cmd'} | $mbuffercmd | $pvcmd | $recvcmd"; + #$synccmd = "$sshcmd $sourcehost '$sendcmd | $compressargs{'cmd'} | $mbuffercmd' | $args{'decompress'}{'cmd'} | $mbuffercmd | $pvcmd | $recvcmd"; $synccmd = "$sshcmd $sourcehost '$sendcmd"; - if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; } + if ($avail{'compress'}) { $synccmd .= " | $compressargs{'compresscmd'}"; } if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } $synccmd .= "' | "; if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } - if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$compressargs{'decompresscmd'} | "; } if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } $synccmd .= "$recvcmd"; } else { #remote source, remote target... weird, but whatever, I'm not here to judge you. - #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compress'}{'cmd'} | $mbuffercmd' | $args{'compress'}{'decomcmd'} | $pvcmd | $args{'compress'}{'cmd'} | $mbuffercmd | $sshcmd $targethost '$args{'compress'}{'decomcmd'} | $mbuffercmd | $recvcmd'"; + #$synccmd = "$sshcmd $sourcehost '$sendcmd | $compressargs{'cmd'} | $mbuffercmd' | $compressargs{'decomcmd'} | $pvcmd | $compressargs{'cmd'} | $mbuffercmd | $sshcmd $targethost '$compressargs{'decomcmd'} | $mbuffercmd | $recvcmd'"; $synccmd = "$sshcmd $sourcehost '$sendcmd"; - if ($avail{'compress'}) { $synccmd .= " | $args{'compress'}{'cmd'}"; } + if ($avail{'compress'}) { $synccmd .= " | $compressargs{'cmd'}"; } if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } $synccmd .= "' | "; - if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'decomcmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$compressargs{'decomcmd'} | "; } if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } - if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'cmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$compressargs{'cmd'} | "; } if ($avail{'localmbuffer'}) { $synccmd .= "$mbuffercmd $mbufferoptions | "; } $synccmd .= "$sshcmd $targethost '"; if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } - if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'decomcmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$compressargs{'decomcmd'} | "; } $synccmd .= "$recvcmd'"; } return $synccmd; From 90e6a401da6eeaffa49223e9a6729095d564e2ee Mon Sep 17 00:00:00 2001 From: Charles Pigott Date: Wed, 8 Nov 2017 16:32:39 +0000 Subject: [PATCH 03/14] Missed some compression variable renames --- syncoid | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index 3f07990..7337f5b 100755 --- a/syncoid +++ b/syncoid @@ -651,11 +651,11 @@ sub buildsynccmd { # remote source, local target. #$synccmd = "$sshcmd $sourcehost '$sendcmd | $compressargs{'cmd'} | $mbuffercmd' | $args{'decompress'}{'cmd'} | $mbuffercmd | $pvcmd | $recvcmd"; $synccmd = "$sshcmd $sourcehost '$sendcmd"; - if ($avail{'compress'}) { $synccmd .= " | $compressargs{'compresscmd'}"; } + if ($avail{'compress'}) { $synccmd .= " | $compressargs{'cmd'}"; } if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } $synccmd .= "' | "; if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } - if ($avail{'compress'}) { $synccmd .= "$compressargs{'decompresscmd'} | "; } + if ($avail{'compress'}) { $synccmd .= "$compressargs{'decomcmd'} | "; } if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } $synccmd .= "$recvcmd"; } else { From e260f9095f7ac6d6666c22e9b8b312a0ac7ff6e6 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 10 Dec 2017 16:05:08 +0100 Subject: [PATCH 04/14] escape filesystem names as needed to avoid interpreting special characters like whitespace and stop interpreting metacharacters in fs names for some regular expressions, fixes #40 --- syncoid | 191 +++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 135 insertions(+), 56 deletions(-) diff --git a/syncoid b/syncoid index d927cba..3d58863 100755 --- a/syncoid +++ b/syncoid @@ -97,7 +97,7 @@ if (! $args{'recursive'}) { if ($debug) { print "DEBUG: recursive sync of $sourcefs.\n"; } my @datasets = getchilddatasets($sourcehost, $sourcefs, $sourceisroot); foreach my $dataset(@datasets) { - $dataset =~ s/$sourcefs//; + $dataset =~ s/\Q$sourcefs\E//; chomp $dataset; my $childsourcefs = $sourcefs . $dataset; my $childtargetfs = $targetfs . $dataset; @@ -126,11 +126,16 @@ exit 0; sub getchilddatasets { my ($rhost,$fs,$isroot,%snaps) = @_; my $mysudocmd; + my $fsescaped = escapeshellparam($fs); if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } - my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name -t filesystem,volume -Hr $fs |"; + my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name -t filesystem,volume -Hr $fsescaped |"; if ($debug) { print "DEBUG: getting list of child datasets on $fs using $getchildrencmd...\n"; } open FH, $getchildrencmd; my @children = ; @@ -143,6 +148,9 @@ sub syncdataset { my ($sourcehost, $sourcefs, $targethost, $targetfs) = @_; + my $sourcefsescaped = escapeshellparam($sourcefs); + my $targetfsescaped = escapeshellparam($targetfs); + if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; } # make sure target is not currently in receive. @@ -209,8 +217,8 @@ sub syncdataset { # if --no-stream is specified, our full needs to be the newest snapshot, not the oldest. if (defined $args{'no-stream'}) { $oldestsnap = getnewestsnapshot(\%snaps); } - my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefs\@$oldestsnap"; - my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfs"; + my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnap"; + my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); my $disp_pvsize = readablebytes($pvsize); @@ -245,7 +253,7 @@ sub syncdataset { # $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); - $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefs\@$oldestsnap $sourcefs\@$newsyncsnap"; + $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnap $sourcefsescaped\@$newsyncsnap"; $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap","$sourcefs\@$newsyncsnap",$sourceisroot); $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -305,15 +313,15 @@ sub syncdataset { # rollback target to matchingsnap if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; } if ($targethost ne '') { - if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap\n"; } - system ("$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap"); + if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap\n"; } + system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap")); } else { - if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap\n"; } - system ("$targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap"); + if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap\n"; } + system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap"); } - my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefs\@$matchingsnap $sourcefs\@$newsyncsnap"; - my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfs"; + my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnap $sourcefsescaped\@$newsyncsnap"; + my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -590,7 +598,7 @@ sub iszfsbusy { foreach my $process (@processes) { # if ($debug) { print "DEBUG: checking process $process...\n"; } - if ($process =~ /zfs *(receive|recv).*$fs/) { + if ($process =~ /zfs *(receive|recv).*\Q$fs\E/) { # there's already a zfs receive process for our target filesystem - return true if ($debug) { print "DEBUG: process $process matches target $fs!\n"; } return 1; @@ -603,24 +611,40 @@ sub iszfsbusy { sub setzfsvalue { my ($rhost,$fs,$isroot,$property,$value) = @_; - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + + my $fsescaped = escapeshellparam($fs); + + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } + if ($debug) { print "DEBUG: setting $property to $value on $fs...\n"; } my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } - if ($debug) { print "$rhost $mysudocmd $zfscmd set $property=$value $fs\n"; } - system("$rhost $mysudocmd $zfscmd set $property=$value $fs") == 0 - or warn "WARNING: $rhost $mysudocmd $zfscmd set $property=$value $fs died: $?, proceeding anyway.\n"; + if ($debug) { print "$rhost $mysudocmd $zfscmd set $property=$value $fsescaped\n"; } + system("$rhost $mysudocmd $zfscmd set $property=$value $fsescaped") == 0 + or warn "WARNING: $rhost $mysudocmd $zfscmd set $property=$value $fsescaped died: $?, proceeding anyway.\n"; return; } sub getzfsvalue { my ($rhost,$fs,$isroot,$property) = @_; - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + + my $fsescaped = escapeshellparam($fs); + + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } + if ($debug) { print "DEBUG: getting current value of $property on $fs...\n"; } my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } - if ($debug) { print "$rhost $mysudocmd $zfscmd get -H $property $fs\n"; } - open FH, "$rhost $mysudocmd $zfscmd get -H $property $fs |"; + if ($debug) { print "$rhost $mysudocmd $zfscmd get -H $property $fsescaped\n"; } + open FH, "$rhost $mysudocmd $zfscmd get -H $property $fsescaped |"; my $value = ; close FH; my @values = split(/\s/,$value); @@ -706,17 +730,24 @@ sub buildsynccmd { if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; } if ($avail{'compress'}) { $synccmd .= " $args{'compresscmd'} |"; } if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; } - $synccmd .= " $sshcmd $targethost '"; - if ($avail{'targetmbuffer'}) { $synccmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; } - if ($avail{'compress'}) { $synccmd .= " $args{'decompresscmd'} |"; } - $synccmd .= " $recvcmd'"; + $synccmd .= " $sshcmd $targethost "; + + my $remotecmd = ""; + if ($avail{'targetmbuffer'}) { $remotecmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; } + if ($avail{'compress'}) { $remotecmd .= " $args{'decompresscmd'} |"; } + $remotecmd .= " $recvcmd"; + + $synccmd .= escapeshellparam($remotecmd); } elsif ($targethost eq '') { # remote source, local target. #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $mbuffercmd | $pvcmd | $recvcmd"; - $synccmd = "$sshcmd $sourcehost '$sendcmd"; - if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; } - if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } - $synccmd .= "' | "; + + my $remotecmd = $sendcmd; + if ($avail{'compress'}) { $remotecmd .= " | $args{'compresscmd'}"; } + if ($avail{'sourcembuffer'}) { $remotecmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } + + $synccmd = "$sshcmd $sourcehost " . escapeshellparam($remotecmd); + $synccmd .= " | "; if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } @@ -724,25 +755,37 @@ sub buildsynccmd { } else { #remote source, remote target... weird, but whatever, I'm not here to judge you. #$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'"; - $synccmd = "$sshcmd $sourcehost '$sendcmd"; - if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; } - if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } - $synccmd .= "' | "; + + my $remotecmd = $sendcmd; + if ($avail{'compress'}) { $remotecmd .= " | $args{'compresscmd'}"; } + if ($avail{'sourcembuffer'}) { $remotecmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } + + $synccmd = "$sshcmd $sourcehost " . escapeshellparam($remotecmd); + $synccmd .= " | "; + if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } if ($avail{'compress'}) { $synccmd .= "$args{'compresscmd'} | "; } if ($avail{'localmbuffer'}) { $synccmd .= "$mbuffercmd $mbufferoptions | "; } - $synccmd .= "$sshcmd $targethost '"; - if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } - if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } - $synccmd .= "$recvcmd'"; + $synccmd .= "$sshcmd $targethost "; + + $remotecmd = ""; + if ($avail{'targetmbuffer'}) { $remotecmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; } + if ($avail{'compress'}) { $remotecmd .= " $args{'decompresscmd'} |"; } + $remotecmd .= " $recvcmd"; + + $synccmd .= escapeshellparam($remotecmd); } return $synccmd; } sub pruneoldsyncsnaps { my ($rhost,$fs,$newsyncsnap,$isroot,@snaps) = @_; + + my $fsescaped = escapeshellparam($fs); + if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + my $hostid = hostname(); my $mysudocmd; @@ -752,7 +795,7 @@ sub pruneoldsyncsnaps { # only prune snaps beginning with syncoid and our own hostname foreach my $snap(@snaps) { - if ($snap =~ /^syncoid_$hostid/) { + if ($snap =~ /^syncoid_\Q$hostid\E/) { # no matter what, we categorically refuse to # prune the new sync snap we created for this run if ($snap ne $newsyncsnap) { @@ -768,12 +811,14 @@ sub pruneoldsyncsnaps { my $prunecmd; foreach my $snap(@prunesnaps) { $counter ++; - $prunecmd .= "$mysudocmd $zfscmd destroy $fs\@$snap; "; + $prunecmd .= "$mysudocmd $zfscmd destroy $fsescaped\@$snap; "; if ($counter > $maxsnapspercmd) { $prunecmd =~ s/\; $//; - if ($rhost ne '') { $prunecmd = '"' . $prunecmd . '"'; } if ($debug) { print "DEBUG: pruning up to $maxsnapspercmd obsolete sync snapshots...\n"; } if ($debug) { print "DEBUG: $rhost $prunecmd\n"; } + if ($rhost ne '') { + $prunecmd = escapeshellparam($prunecmd); + } system("$rhost $prunecmd") == 0 or warn "CRITICAL ERROR: $rhost $prunecmd failed: $?"; $prunecmd = ''; @@ -784,9 +829,11 @@ sub pruneoldsyncsnaps { # the loop, commit 'em now if ($counter) { $prunecmd =~ s/\; $//; - if ($rhost ne '') { $prunecmd = '"' . $prunecmd . '"'; } if ($debug) { print "DEBUG: pruning up to $maxsnapspercmd obsolete sync snapshots...\n"; } if ($debug) { print "DEBUG: $rhost $prunecmd\n"; } + if ($rhost ne '') { + $prunecmd = escapeshellparam($prunecmd); + } system("$rhost $prunecmd") == 0 or warn "WARNING: $rhost $prunecmd failed: $?"; } @@ -824,13 +871,18 @@ sub getmatchingsnapshot { sub newsyncsnap { my ($rhost,$fs,$isroot) = @_; - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + my $fsescaped = escapeshellparam($fs); + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } my $hostid = hostname(); my %date = getdate(); my $snapname = "syncoid\_$hostid\_$date{'stamp'}"; - my $snapcmd = "$rhost $mysudocmd $zfscmd snapshot $fs\@$snapname\n"; + my $snapcmd = "$rhost $mysudocmd $zfscmd snapshot $fsescaped\@$snapname\n"; system($snapcmd) == 0 or die "CRITICAL ERROR: $snapcmd failed: $?"; return $snapname; @@ -838,16 +890,21 @@ sub newsyncsnap { sub targetexists { my ($rhost,$fs,$isroot) = @_; - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + my $fsescaped = escapeshellparam($fs); + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } - my $checktargetcmd = "$rhost $mysudocmd $zfscmd get -H name $fs"; + my $checktargetcmd = "$rhost $mysudocmd $zfscmd get -H name $fsescaped"; if ($debug) { print "DEBUG: checking to see if target filesystem exists using \"$checktargetcmd 2>&1 |\"...\n"; } open FH, "$checktargetcmd 2>&1 |"; my $targetexists = ; close FH; my $exit = $?; - $targetexists = ( $targetexists =~ /^$fs/ && $exit == 0 ); + $targetexists = ( $targetexists =~ /^\Q$fs\E/ && $exit == 0 ); return $targetexists; } @@ -888,11 +945,16 @@ sub dumphash() { sub getsnaps() { my ($type,$rhost,$fs,$isroot,%snaps) = @_; my $mysudocmd; + my $fsescaped = escapeshellparam($fs); if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } - if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } - my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t snapshot guid,creation $fs |"; + my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t snapshot guid,creation $fsescaped |"; if ($debug) { print "DEBUG: getting list of snapshots on $fs using $getsnapcmd...\n"; } open FH, $getsnapcmd; my @rawsnaps = ; @@ -903,24 +965,24 @@ sub getsnaps() { foreach my $line (@rawsnaps) { # only import snap guids from the specified filesystem - if ($line =~ /$fs\@.*guid/) { + if ($line =~ /\Q$fs\E\@.*guid/) { chomp $line; my $guid = $line; $guid =~ s/^.*\sguid\s*(\d*).*/$1/; my $snap = $line; - $snap =~ s/^\S*\@(\S*)\s*guid.*$/$1/; + $snap =~ s/^.*\@(\S*)\s*guid.*$/$1/; $snaps{$type}{$snap}{'guid'}=$guid; } } foreach my $line (@rawsnaps) { # only import snap creations from the specified filesystem - if ($line =~ /$fs\@.*creation/) { + if ($line =~ /\Q$fs\E\@.*creation/) { chomp $line; my $creation = $line; $creation =~ s/^.*\screation\s*(\d*).*/$1/; my $snap = $line; - $snap =~ s/^\S*\@(\S*)\s*creation.*$/$1/; + $snap =~ s/^.*\@(\S*)\s*creation.*$/$1/; $snaps{$type}{$snap}{'creation'}=$creation; } } @@ -932,21 +994,30 @@ sub getsnaps() { sub getsendsize { my ($sourcehost,$snap1,$snap2,$isroot) = @_; + my $snap1escaped = escapeshellparam($snap1); + my $snap2escaped = escapeshellparam($snap2); + my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } + my $sourcessh; + if ($sourcehost ne '') { + $sourcessh = "$sshcmd $sourcehost"; + $snap1escaped = escapeshellparam($snap1escaped); + $snap2escaped = escapeshellparam($snap2escaped); + } else { + $sourcessh = ''; + } + my $snaps; if ($snap2) { # if we got a $snap2 argument, we want an incremental send estimate from $snap1 to $snap2. - $snaps = "$args{'streamarg'} $snap1 $snap2"; + $snaps = "$args{'streamarg'} $snap1escaped $snap2escaped"; } else { # if we didn't get a $snap2 arg, we want a full send estimate for $snap1. - $snaps = "$snap1"; + $snaps = "$snap1escaped"; } - my $sourcessh; - if ($sourcehost ne '') { $sourcessh = "$sshcmd $sourcehost"; } else { $sourcessh = ''; } - my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send -nP $snaps"; if ($debug) { print "DEBUG: getting estimated transfer size from source $sourcehost using \"$getsendsizecmd 2>&1 |\"...\n"; } @@ -987,3 +1058,11 @@ sub getdate { $date{'stamp'} = "$date{'year'}-$date{'mon'}-$date{'mday'}:$date{'hour'}:$date{'min'}:$date{'sec'}"; return %date; } + +sub escapeshellparam { + my ($par) = @_; + # "escape" all single quotes + $par =~ s/'/'"'"'/g; + # single-quote entire string + return "'$par'"; + } From adca6230b735b973d34988068c0a3643aff9a760 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 10 Dec 2017 16:10:49 +0100 Subject: [PATCH 05/14] missed one regular expression --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 3d58863..ed5cce9 100755 --- a/syncoid +++ b/syncoid @@ -919,7 +919,7 @@ sub getssh { if ($fs =~ /\@/) { $rhost = $fs; $fs =~ s/^\S*\@\S*://; - $rhost =~ s/:$fs$//; + $rhost =~ s/:\Q$fs\E$//; my $remoteuser = $rhost; $remoteuser =~ s/\@.*$//; if ($remoteuser eq 'root') { $isroot = 1; } else { $isroot = 0; } From 3027831095d51e4e53f28bb82e337d8c11606cac Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 10 Dec 2017 16:32:49 +0100 Subject: [PATCH 06/14] escape all remaining snapshot names which could have special characters --- syncoid | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/syncoid b/syncoid index ed5cce9..797b5c0 100755 --- a/syncoid +++ b/syncoid @@ -216,8 +216,9 @@ sub syncdataset { # if --no-stream is specified, our full needs to be the newest snapshot, not the oldest. if (defined $args{'no-stream'}) { $oldestsnap = getnewestsnapshot(\%snaps); } + my $oldestsnapescaped = escapeshellparam($oldestsnap); - my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnap"; + my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped"; my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); @@ -253,7 +254,7 @@ sub syncdataset { # $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); - $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnap $sourcefsescaped\@$newsyncsnap"; + $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnapescaped $sourcefsescaped\@$newsyncsnap"; $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap","$sourcefs\@$newsyncsnap",$sourceisroot); $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -310,17 +311,18 @@ sub syncdataset { # barf some text but don't touch the filesystem if (!$quiet) { print "INFO: no snapshots on source newer than $newsyncsnap on target. Nothing to do, not syncing.\n"; } } else { + my $matchingsnapescaped = escapeshellparam($matchingsnap); # rollback target to matchingsnap if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; } if ($targethost ne '') { - if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap\n"; } - system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap")); + if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped\n"; } + system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped")); } else { - if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap\n"; } - system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnap"); + if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped\n"; } + system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped"); } - my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnap $sourcefsescaped\@$newsyncsnap"; + my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnap"; my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); From 85cc99c9e6b4c507a1559903ecc7718685b0a77c Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 10 Dec 2017 16:56:56 +0100 Subject: [PATCH 07/14] accidentally removed helptext --- syncoid | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/syncoid b/syncoid index 86810d4..8825dcb 100755 --- a/syncoid +++ b/syncoid @@ -1005,3 +1005,41 @@ sub escapeshellparam { # single-quote entire string return "'$par'"; } + +__END__ + +=head1 NAME + +syncoid - ZFS snapshot replication tool + +=head1 SYNOPSIS + + syncoid [options]... SOURCE TARGET + or syncoid [options]... SOURCE [USER@]HOST:TARGET + or syncoid [options]... [USER@]HOST:SOURCE [TARGET] + or syncoid [options]... [USER@]HOST:SOURCE [USER@]HOST:TARGET + + SOURCE Source ZFS dataset. Can be either local or remote + TARGET Target ZFS dataset. Can be either local or remote + +Options: + + --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none + --recursive|r Also transfers child datasets + --source-bwlimit= Bandwidth limit on the source transfer + --target-bwlimit= Bandwidth limit on the target transfer + --no-stream Replicates using newest snapshot instead of intermediates + --no-sync-snap Does not create new snapshot, only transfers existing + + --sshkey=FILE Specifies a ssh public key to use to connect + --sshport=PORT Connects to remote on a particular port + --sshcipher|c=CIPHER Passes CIPHER to ssh to use a particular cipher set + --sshoption|o=OPTION Passes OPTION to ssh for remote usage. Can be specified multiple times + + --help Prints this helptext + --verbose Prints the version number + --debug Prints out a lot of additional information during a syncoid run + --monitor-version Currently does nothing + --quiet Suppresses non-error output + --dumpsnaps Dumps a list of snapshots during the run + --no-command-checks Do not check command existence before attempting transfer. Not recommended From d5f4b5abba08a597108538cadbd09961afa80363 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 13 Feb 2018 18:47:55 +0100 Subject: [PATCH 08/14] support resumable zfs send/receive --- syncoid | 127 +++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 98 insertions(+), 29 deletions(-) diff --git a/syncoid b/syncoid index 8825dcb..8a31f37 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "resume") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -161,35 +161,55 @@ sub syncdataset { # does the target filesystem exist yet? my $targetexists = targetexists($targethost,$targetfs,$targetisroot); - # build hashes of the snaps on the source and target filesystems. + my $receiveextraargs = ""; + my $receivetoken; + if (defined $args{'resume'}) { + # save state of interrupted receive stream + $receiveextraargs = "-s"; - %snaps = getsnaps('source',$sourcehost,$sourcefs,$sourceisroot); + if ($targetexists) { + # check remote dataset for receive resume token (interrupted receive) + $receivetoken = getreceivetoken($targethost,$targetfs,$targetisroot); - if ($targetexists) { - my %targetsnaps = getsnaps('target',$targethost,$targetfs,$targetisroot); - my %sourcesnaps = %snaps; - %snaps = (%sourcesnaps, %targetsnaps); - } - - if (defined $args{'dumpsnaps'}) { - print "merged snapshot list of $targetfs: \n"; - dumphash(\%snaps); - print "\n\n\n"; - } - - # create a new syncoid snapshot on the source filesystem. - my $newsyncsnap; - if (!defined $args{'no-sync-snap'}) { - $newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot); - } else { - # we don't want sync snapshots created, so use the newest snapshot we can find. - $newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot); - if ($newsyncsnap eq 0) { - warn "CRITICAL: no snapshots exist on source $sourcefs, and you asked for --no-sync-snap.\n"; - return 0; + if ($debug && defined($receivetoken)) { + print "DEBUG: got receive resume token: $receivetoken: \n"; + } } } + my $newsyncsnap; + + # skip snapshot checking/creation in case of resumed receive + if (!defined($receivetoken)) { + # build hashes of the snaps on the source and target filesystems. + + %snaps = getsnaps('source',$sourcehost,$sourcefs,$sourceisroot); + + if ($targetexists) { + my %targetsnaps = getsnaps('target',$targethost,$targetfs,$targetisroot); + my %sourcesnaps = %snaps; + %snaps = (%sourcesnaps, %targetsnaps); + } + + if (defined $args{'dumpsnaps'}) { + print "merged snapshot list of $targetfs: \n"; + dumphash(\%snaps); + print "\n\n\n"; + } + + if (!defined $args{'no-sync-snap'}) { + # create a new syncoid snapshot on the source filesystem. + $newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot); + } else { + # we don't want sync snapshots created, so use the newest snapshot we can find. + $newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot); + if ($newsyncsnap eq 0) { + warn "CRITICAL: no snapshots exist on source $sourcefs, and you asked for --no-sync-snap.\n"; + return 0; + } + } + } + # there is currently (2014-09-01) a bug in ZFS on Linux # that causes readonly to always show on if it's EVER # been turned on... even when it's off... unless and @@ -222,7 +242,7 @@ sub syncdataset { my $oldestsnapescaped = escapeshellparam($oldestsnap); my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); my $disp_pvsize = readablebytes($pvsize); @@ -286,6 +306,27 @@ sub syncdataset { # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly); } } else { + # resume interrupted receive if there is a valid resume $token + # and because this will ony resume the receive to the next + # snapshot, do a normal sync after that + if (defined($receivetoken)) { + my $sendcmd = "$sourcesudocmd $zfscmd send -t $receivetoken"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $pvsize = getsendsize($sourcehost,"","",$sourceisroot,$receivetoken); + my $disp_pvsize = readablebytes($pvsize); + if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } + my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + + if (!$quiet) { print "Resuming interrupted zfs send/receive from $sourcefs to $targetfs (~ $disp_pvsize remaining):\n"; } + if ($debug) { print "DEBUG: $synccmd\n"; } + system("$synccmd") == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; + + # a resumed transfer will only be done to the next snapshot, + # so do an normal sync cycle + return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs); + } + # find most recent matching snapshot and do an -I # to the new snapshot @@ -326,7 +367,7 @@ sub syncdataset { } my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnap"; - my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -931,7 +972,7 @@ sub getsnaps() { sub getsendsize { - my ($sourcehost,$snap1,$snap2,$isroot) = @_; + my ($sourcehost,$snap1,$snap2,$isroot,$receivetoken) = @_; my $snap1escaped = escapeshellparam($snap1); my $snap2escaped = escapeshellparam($snap2); @@ -957,6 +998,12 @@ sub getsendsize { $snaps = "$snap1escaped"; } + # in case of a resumed receive, get the remaining + # size based on the resume token + if (defined($receivetoken)) { + $snaps = "-t $receivetoken"; + } + my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send -nP $snaps"; if ($debug) { print "DEBUG: getting estimated transfer size from source $sourcehost using \"$getsendsizecmd 2>&1 |\"...\n"; } @@ -969,7 +1016,13 @@ sub getsendsize { # size of proposed xfer in bytes, but we need to remove # human-readable crap from it my $sendsize = pop(@rawsize); - $sendsize =~ s/^size\s*//; + # the output format is different in case of + # a resumed receive + if (defined($receivetoken)) { + $sendsize =~ s/.*\s([0-9]+)$/$1/; + } else { + $sendsize =~ s/^size\s*//; + } chomp $sendsize; # to avoid confusion with a zero size pv, give sendsize @@ -1006,6 +1059,21 @@ sub escapeshellparam { return "'$par'"; } +sub getreceivetoken() { + my ($rhost,$fs,$isroot) = @_; + my $token = getzfsvalue($rhost,$fs,$isroot,"receive_resume_token"); + + if ($token ne '-' && $token ne '') { + return $token; + } + + if ($debug) { + print "DEBUG: no receive token found \n"; + } + + return +} + __END__ =head1 NAME @@ -1043,3 +1111,4 @@ Options: --quiet Suppresses non-error output --dumpsnaps Dumps a list of snapshots during the run --no-command-checks Do not check command existence before attempting transfer. Not recommended + --resume Save the state of unfinished receive streams and resume interrupted ones if available From 1b5ab20b0d9fd83be65ab7266e294dc914a78cf0 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 25 Apr 2018 14:58:01 +0200 Subject: [PATCH 09/14] use resumeable zfs send/receive as default if supported by source and target --- README.md | 5 +++++ syncoid | 40 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9c4e6ba..b66deac 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,7 @@ syncoid root@remotehost:data/images/vm backup/images/vm Which would pull-replicate the filesystem from the remote host to the local system over an SSH tunnel. Syncoid supports recursive replication (replication of a dataset and all its child datasets) and uses mbuffer buffering, lzop compression, and pv progress bars if the utilities are available on the systems used. +If ZFS supports resumeable send/receive streams on both the source and target those will be enabled as default. ##### Syncoid Command Line Options @@ -147,6 +148,10 @@ Syncoid supports recursive replication (replication of a dataset and all its chi This argument tells syncoid to restrict itself to existing snapshots, instead of creating a semi-ephemeral syncoid snapshot at execution time. Especially useful in multi-target (A->B, A->C) replication schemes, where you might otherwise accumulate a large number of foreign syncoid snapshots. ++ --no-resume + + This argument tells syncoid to not use resumeable zfs send/receive streams. + + --dumpsnaps This prints a list of snapshots during the run. diff --git a/syncoid b/syncoid index 8a31f37..170799b 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "resume") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -46,6 +46,7 @@ my $rawsourcefs = $args{'source'}; my $rawtargetfs = $args{'target'}; my $debug = $args{'debug'}; my $quiet = $args{'quiet'}; +my $resume = !$args{'no-resume'}; my $zfscmd = '/sbin/zfs'; my $sshcmd = '/usr/bin/ssh'; @@ -163,7 +164,7 @@ sub syncdataset { my $receiveextraargs = ""; my $receivetoken; - if (defined $args{'resume'}) { + if ($resume) { # save state of interrupted receive stream $receiveextraargs = "-s"; @@ -457,6 +458,8 @@ sub checkcommands { $avail{'localmbuffer'} = 1; $avail{'sourcembuffer'} = 1; $avail{'targetmbuffer'} = 1; + $avail{'sourceresume'} = 1; + $avail{'targetresume'} = 1; return %avail; } @@ -564,6 +567,37 @@ sub checkcommands { $avail{'localpv'} = 1; } + # check for ZFS resume feature support + if ($resume) { + my $resumechkcmd = "$zfscmd get receive_resume_token -d 0"; + + if ($debug) { print "DEBUG: checking availability of zfs resume feature on source...\n"; } + $avail{'sourceresume'} = system("$sourcessh $resumechkcmd >/dev/null 2>&1"); + $avail{'sourceresume'} = $avail{'sourceresume'} == 0 ? 1 : 0; + + if ($debug) { print "DEBUG: checking availability of zfs resume feature on target...\n"; } + $avail{'targetresume'} = system("$targetssh $resumechkcmd >/dev/null 2>&1"); + $avail{'targetresume'} = $avail{'targetresume'} == 0 ? 1 : 0; + + if ($avail{'sourceresume'} == 0 || $avail{'targetresume'} == 0) { + # disable resume + $resume = ''; + + my @hosts = (); + if ($avail{'sourceresume'} == 0) { + push @hosts, 'source'; + } + if ($avail{'targetresume'} == 0) { + push @hosts, 'target'; + } + my $affected = join(" and ", @hosts); + print "WARN: ZFS resume feature not available on $affected machine - sync will continue without resume support.\n"; + } + } else { + $avail{'sourceresume'} = 0; + $avail{'targetresume'} = 0; + } + return %avail; } @@ -1111,4 +1145,4 @@ Options: --quiet Suppresses non-error output --dumpsnaps Dumps a list of snapshots during the run --no-command-checks Do not check command existence before attempting transfer. Not recommended - --resume Save the state of unfinished receive streams and resume interrupted ones if available + --no-resume Don't use the ZFS resume feature if available From e6eec0aca041dc5464f30cdededaa597dc3e062f Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 25 Apr 2018 17:15:55 +0200 Subject: [PATCH 10/14] fix snapshot list generation and incremental sync for snapshots containing whitespaces --- syncoid | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/syncoid b/syncoid index 8825dcb..0654ac3 100755 --- a/syncoid +++ b/syncoid @@ -189,6 +189,7 @@ sub syncdataset { return 0; } } + my $newsyncsnapescaped = escapeshellparam($newsyncsnap); # there is currently (2014-09-01) a bug in ZFS on Linux # that causes readonly to always show on if it's EVER @@ -257,7 +258,7 @@ sub syncdataset { # $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); - $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnapescaped $sourcefsescaped\@$newsyncsnap"; + $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$oldestsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap","$sourcefs\@$newsyncsnap",$sourceisroot); $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -325,7 +326,7 @@ sub syncdataset { system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped"); } - my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnap"; + my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); @@ -909,7 +910,7 @@ sub getsnaps() { my $guid = $line; $guid =~ s/^.*\sguid\s*(\d*).*/$1/; my $snap = $line; - $snap =~ s/^.*\@(\S*)\s*guid.*$/$1/; + $snap =~ s/^.*\@(.*)\tguid.*$/$1/; $snaps{$type}{$snap}{'guid'}=$guid; } } @@ -921,7 +922,7 @@ sub getsnaps() { my $creation = $line; $creation =~ s/^.*\screation\s*(\d*).*/$1/; my $snap = $line; - $snap =~ s/^.*\@(\S*)\s*creation.*$/$1/; + $snap =~ s/^.*\@(.*)\tcreation.*$/$1/; $snaps{$type}{$snap}{'creation'}=$creation; } } From 4ebc0abef5a4999c344e9e7d37d2686e101609f3 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 25 Apr 2018 14:58:35 -0400 Subject: [PATCH 11/14] fix use of uninitialized string in escapeshellparam() --- syncoid | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index 784bd90..10c78c1 100755 --- a/syncoid +++ b/syncoid @@ -1087,9 +1087,12 @@ sub getdate { } sub escapeshellparam { - my ($par) = @_; - # "escape" all single quotes - $par =~ s/'/'"'"'/g; + my $par; + if (scalar @_) { + ($par) = @_; + # "escape" all single quotes + $par =~ s/'/'"'"'/g; + } # single-quote entire string return "'$par'"; } From 2732392088cf713898882e54561a20a84b1b6538 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 25 Apr 2018 15:18:00 -0400 Subject: [PATCH 12/14] fix use of uninitialized value in escapeshellparam() --- syncoid | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index 10c78c1..df08122 100755 --- a/syncoid +++ b/syncoid @@ -1087,11 +1087,14 @@ sub getdate { } sub escapeshellparam { - my $par; - if (scalar @_) { - ($par) = @_; + my ($par) = @_; + # avoid use of uninitialized string in regex + if (length($par)) { # "escape" all single quotes $par =~ s/'/'"'"'/g; + } else { + # avoid use of uninitialized string in concatenation below + $par = ''; } # single-quote entire string return "'$par'"; From 72245338a805b565183f7a0579e98757c3e6cb14 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 25 Apr 2018 15:21:56 -0400 Subject: [PATCH 13/14] special character handling and resumable zfs receive support in syncoid --- CHANGELIST | 3 +++ VERSION | 2 +- sanoid | 2 +- syncoid | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 9af90f6..515d05d 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,6 @@ +1.4.18 implemented special character handling and support of ZFS resume/receive tokens by default in syncoid, + thank you @phreaker0! + 1.4.17 changed die to warn when unexpectedly unable to remove a snapshot - this allows sanoid to continue taking/removing other snapshots not affected by whatever lock prevented the first from being taken or removed diff --git a/VERSION b/VERSION index 04e0d3f..f689e8c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.17 +1.4.18 diff --git a/sanoid b/sanoid index d6e58ce..b6dc9fe 100755 --- a/sanoid +++ b/sanoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '1.4.17'; +$::VERSION = '1.4.18'; use strict; use warnings; diff --git a/syncoid b/syncoid index df08122..e63b68c 100755 --- a/syncoid +++ b/syncoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '1.4.16'; +$::VERSION = '1.4.18'; use strict; use warnings; From 979cb8bc3d154e30e7f0fbc016a1fdccc509941d Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Wed, 25 Apr 2018 15:24:24 -0400 Subject: [PATCH 14/14] resume/receive support in syncoid 1.4.18 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b66deac..cc75bdf 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,8 @@ Which would pull-replicate the filesystem from the remote host to the local syst Syncoid supports recursive replication (replication of a dataset and all its child datasets) and uses mbuffer buffering, lzop compression, and pv progress bars if the utilities are available on the systems used. If ZFS supports resumeable send/receive streams on both the source and target those will be enabled as default. +As of 1.4.18, syncoid also automatically supports and enables resume of interrupted replication when both source and target support this feature. + ##### Syncoid Command Line Options + [source]